Skip to content

Commit cca47cb

Browse files
committed
fix: make leaderelection possible
1 parent 2cb6d9f commit cca47cb

File tree

14 files changed

+603
-124
lines changed

14 files changed

+603
-124
lines changed

cmd/manager/main.go

+82-35
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"errors"
55
"fmt"
6+
"net/url"
67
"os"
78
"runtime"
89
"strconv"
@@ -19,7 +20,6 @@ import (
1920
dvo_prom "github.com/app-sre/deployment-validation-operator/pkg/prometheus"
2021
"github.com/app-sre/deployment-validation-operator/pkg/validations"
2122
"github.com/app-sre/deployment-validation-operator/version"
22-
"github.com/prometheus/client_golang/prometheus"
2323

2424
"github.com/go-logr/logr"
2525
osappsv1 "github.com/openshift/api/apps/v1"
@@ -43,13 +43,18 @@ func main() {
4343
os.Setenv(operatorNameEnvVar, dvconfig.OperatorName)
4444

4545
opts := options.Options{
46-
MetricsPort: 8383,
47-
MetricsPath: "metrics",
48-
ProbeAddr: ":8081",
49-
ConfigFile: "config/deployment-validation-operator-config.yaml",
46+
MetricsBindAddr: ":8383",
47+
MetricsPath: "metrics",
48+
MetricsServiceName: "deployment-validation-operator-metrics",
49+
ProbeAddr: ":8081",
50+
ConfigFile: "config/deployment-validation-operator-config.yaml",
5051
}
5152

52-
opts.Process()
53+
if err := opts.Process(); err != nil {
54+
fmt.Fprintf(os.Stdout, "processing options: %v\n", err)
55+
56+
os.Exit(1)
57+
}
5358

5459
// Use a zap logr.Logger implementation. If none of the zap
5560
// flags are configured (or if the zap flag set is not being
@@ -106,12 +111,8 @@ func setupManager(log logr.Logger, opts options.Options) (manager.Manager, error
106111
return nil, fmt.Errorf("initializing manager: %w", err)
107112
}
108113

109-
if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
110-
return nil, fmt.Errorf("adding healthz check: %w", err)
111-
}
112-
113-
if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
114-
return nil, fmt.Errorf("adding readyz check: %w", err)
114+
if err := setupProbes(mgr, opts); err != nil {
115+
return nil, fmt.Errorf("setting up probes: %w", err)
115116
}
116117

117118
log.Info("Registering Components")
@@ -126,29 +127,12 @@ func setupManager(log logr.Logger, opts options.Options) (manager.Manager, error
126127
return nil, fmt.Errorf("initializing generic reconciler: %w", err)
127128
}
128129

129-
if err = gr.AddToManager(mgr); err != nil {
130+
if err := gr.AddToManager(mgr); err != nil {
130131
return nil, fmt.Errorf("adding generic reconciler to manager: %w", err)
131132
}
132133

133-
log.Info("Initializing Prometheus Registry")
134-
135-
reg := prometheus.NewRegistry()
136-
137-
log.Info(fmt.Sprintf("Initializing Prometheus metrics endpoint on %q", opts.MetricsEndpoint()))
138-
139-
srv, err := dvo_prom.NewServer(reg, opts.MetricsPath, fmt.Sprintf(":%d", opts.MetricsPort))
140-
if err != nil {
141-
return nil, fmt.Errorf("initializing metrics server: %w", err)
142-
}
143-
144-
if err := mgr.Add(srv); err != nil {
145-
return nil, fmt.Errorf("adding metrics server to manager: %w", err)
146-
}
147-
148-
log.Info("Initializing Validation Engine")
149-
150-
if err := validations.InitializeValidationEngine(opts.ConfigFile, reg); err != nil {
151-
return nil, fmt.Errorf("initializing validation engine: %w", err)
134+
if err := setupComponents(log, mgr, opts); err != nil {
135+
return nil, fmt.Errorf("setting up components: %w", err)
152136
}
153137

154138
return mgr, nil
@@ -193,9 +177,13 @@ func getManagerOptions(scheme *k8sruntime.Scheme, opts options.Options) (manager
193177
}
194178

195179
mgrOpts := manager.Options{
196-
Namespace: ns,
197-
HealthProbeBindAddress: opts.ProbeAddr,
198-
MetricsBindAddress: "0", // disable controller-runtime managed prometheus endpoint
180+
LeaderElection: opts.EnableLeaderElection,
181+
LeaderElectionID: "23h85e23.deployment-validation-operator-lock",
182+
LeaderElectionNamespace: opts.LeaderElectionNamespace,
183+
LeaderElectionResourceLock: "leases",
184+
Namespace: ns,
185+
HealthProbeBindAddress: opts.ProbeAddr,
186+
MetricsBindAddress: "0", // disable controller-runtime managed prometheus endpoint
199187
// disable caching of everything
200188
NewClient: newClient,
201189
Scheme: scheme,
@@ -237,3 +225,62 @@ func kubeClientQPS() (float32, error) {
237225
qps = float32(val)
238226
return qps, err
239227
}
228+
229+
func setupProbes(mgr manager.Manager, opts options.Options) error {
230+
if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
231+
return fmt.Errorf("adding healthz check: %w", err)
232+
}
233+
234+
if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
235+
return fmt.Errorf("adding readyz check: %w", err)
236+
}
237+
238+
return nil
239+
}
240+
241+
func setupComponents(log logr.Logger, mgr manager.Manager, opts options.Options) error {
242+
log.Info("Initializing Prometheus Registry")
243+
244+
reg, err := dvo_prom.NewRegistry()
245+
if err != nil {
246+
return fmt.Errorf("initializing prometheus registry: %w", err)
247+
}
248+
249+
log.Info(fmt.Sprintf("Initializing Prometheus metrics endpoint on %q", opts.MetricsEndpoint()))
250+
251+
svcURL := &url.URL{
252+
Scheme: "http",
253+
Host: opts.MetricsServiceName,
254+
}
255+
if parts := strings.Split(opts.MetricsBindAddr, ":"); len(parts) > 0 {
256+
if len(parts) > 1 {
257+
svcURL.Host += parts[len(parts)-1]
258+
}
259+
}
260+
261+
srv, err := dvo_prom.NewServer(reg,
262+
dvo_prom.WithMetricsAddr(opts.MetricsBindAddr),
263+
dvo_prom.WithMetricsPath(opts.MetricsPath),
264+
dvo_prom.WithServiceURL(svcURL.String()),
265+
)
266+
if err != nil {
267+
return fmt.Errorf("initializing metrics server: %w", err)
268+
}
269+
270+
go func() {
271+
<-mgr.Elected()
272+
srv.Ready()
273+
}()
274+
275+
if err := mgr.Add(srv); err != nil {
276+
return fmt.Errorf("adding metrics server to manager: %w", err)
277+
}
278+
279+
log.Info("Initializing Validation Engine")
280+
281+
if err := validations.InitializeValidationEngine(opts.ConfigFile, reg); err != nil {
282+
return fmt.Errorf("initializing validation engine: %w", err)
283+
}
284+
285+
return nil
286+
}

go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ require (
1515
github.com/prometheus/common v0.32.1
1616
github.com/spf13/pflag v1.0.5
1717
github.com/spf13/viper v1.9.0
18-
github.com/stretchr/testify v1.7.0
18+
github.com/stretchr/testify v1.7.1
1919
go.uber.org/multierr v1.6.0
20+
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
2021
golang.stackrox.io/kube-linter v0.0.0-20210928184316-5e1ead387f43
2122
k8s.io/api v0.22.2
2223
k8s.io/apimachinery v0.22.2

go.sum

+5-2
Original file line numberDiff line numberDiff line change
@@ -1104,8 +1104,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
11041104
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
11051105
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
11061106
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
1107-
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
11081107
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
1108+
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
1109+
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
11091110
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
11101111
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
11111112
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1262,8 +1263,9 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
12621263
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
12631264
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
12641265
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
1265-
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
12661266
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
1267+
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
1268+
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
12671269
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
12681270
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
12691271
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1363,6 +1365,7 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
13631365
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
13641366
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
13651367
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
1368+
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
13661369
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
13671370
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
13681371
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=

internal/handler/handler.go

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package handler
2+
3+
import (
4+
"net/http"
5+
"strings"
6+
"sync"
7+
8+
"github.com/go-logr/logr"
9+
)
10+
11+
func NewSwitchableHandler(handA http.Handler, handB http.Handler, opts ...SwitchableHandlerOption) *SwitchableHandler {
12+
var cfg SwitchableHandlerConfig
13+
14+
cfg.Option(opts...)
15+
cfg.Default()
16+
17+
return &SwitchableHandler{
18+
cfg: cfg,
19+
handlerA: handA,
20+
handlerB: handB,
21+
}
22+
}
23+
24+
type SwitchableHandler struct {
25+
cfg SwitchableHandlerConfig
26+
handlerA http.Handler
27+
handlerB http.Handler
28+
29+
lock sync.Mutex
30+
switched bool
31+
}
32+
33+
func (h *SwitchableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
34+
log := h.cfg.Log.WithValues("remote", r.RemoteAddr)
35+
36+
if h.switched {
37+
log.WithValues("handler", "B").Info("serving request")
38+
39+
h.handlerB.ServeHTTP(w, r)
40+
} else {
41+
log.WithValues("handler", "A").Info("serving request")
42+
43+
h.handlerA.ServeHTTP(w, r)
44+
}
45+
}
46+
47+
func (h *SwitchableHandler) Switch() {
48+
h.lock.Lock()
49+
defer h.lock.Unlock()
50+
51+
if h.switched {
52+
h.switched = false
53+
} else {
54+
h.switched = true
55+
}
56+
}
57+
58+
type SwitchableHandlerConfig struct {
59+
Log logr.Logger
60+
}
61+
62+
func (c *SwitchableHandlerConfig) Option(opts ...SwitchableHandlerOption) {
63+
for _, opt := range opts {
64+
opt.ConfigureSwitchableHandler(c)
65+
}
66+
}
67+
68+
func (c *SwitchableHandlerConfig) Default() {
69+
if c.Log == nil {
70+
c.Log = logr.Discard()
71+
}
72+
}
73+
74+
type SwitchableHandlerOption interface {
75+
ConfigureSwitchableHandler(*SwitchableHandlerConfig)
76+
}
77+
78+
func StopAfterNForwards(n uint, h http.Handler) http.Handler {
79+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
80+
rawFwds, ok := r.Header["X-Forwarded-For"]
81+
if !ok {
82+
h.ServeHTTP(w, r)
83+
}
84+
85+
splitFwds := strings.Split(strings.Join(rawFwds, ", "), ", ")
86+
if uint(len(splitFwds)) >= n {
87+
http.Error(w, "", http.StatusLoopDetected)
88+
}
89+
90+
h.ServeHTTP(w, r)
91+
})
92+
}

0 commit comments

Comments
 (0)