implement (optional) mutating webhooks

reduce number of restarts of pods in deployment
wave-k8s · Apr 29, 2024 · 531f331 · 531f331
1 parent 3298233
commit 531f331
Show file tree

Hide file tree

Showing 17 changed files with 487 additions and 13 deletions.
diff --git a/charts/wave/templates/webhook.yaml b/charts/wave/templates/webhook.yaml
@@ -0,0 +1,70 @@
+---
+{{- if .Values.webhooks.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration'
+  annotations:
+    cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert'
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-deployment
+    failurePolicy: Ignore
+    name: deployments.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - deployments
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-statefulset
+    failurePolicy: Ignore
+    name: statefulset.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - statefulsets
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-daemonset
+    failurePolicy: Ignore
+    name: daemonset.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - daemonsets
+    sideEffects: NoneOnDryRun
+{{- end }}
diff --git a/charts/wave/templates/webhook_certificate.yaml b/charts/wave/templates/webhook_certificate.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.webhooks.enabled }}
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "wave-fullname" . }}-serving-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  dnsNames:
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  secretName: {{ template "wave-fullname" . }}-webhook-server-cert
+{{- end }}
diff --git a/charts/wave/templates/webhook_service.yaml b/charts/wave/templates/webhook_service.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.webhooks.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "wave-fullname" . }}-webhook-service
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "wave-labels.chart" . | indent 4 }}
+spec:
+  ports:
+    - port: 443
+      targetPort: 9443
+  selector:
+    {{- include "wave-labels.chart" . | indent 4 }}
+{{- end }}
diff --git a/charts/wave/values.yaml b/charts/wave/values.yaml
@@ -43,6 +43,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+webhooks:
+  enabled: false
+
 # Period for reconciliation
 # syncPeriod: 5m
 

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -23,13 +23,19 @@ import (
 	"runtime"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 
 	"github.com/wave-k8s/wave/pkg/apis"
 	"github.com/wave-k8s/wave/pkg/controller"
+	"github.com/wave-k8s/wave/pkg/controller/daemonset"
+	"github.com/wave-k8s/wave/pkg/controller/deployment"
+	"github.com/wave-k8s/wave/pkg/controller/statefulset"
+
 	"github.com/wave-k8s/wave/pkg/webhook"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -110,4 +116,28 @@ func main() {
 		setupLog.Error(err, "unable to run the manager")
 		os.Exit(1)
 	}
+
+	if err := builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter(
+		&deployment.DeploymentWebhook{
+			Client: mgr.GetClient(),
+		}).Complete(); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "Deployment")
+		os.Exit(1)
+	}
+
+	if err := builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter(
+		&statefulset.StatefulSetWebhook{
+			Client: mgr.GetClient(),
+		}).Complete(); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet")
+		os.Exit(1)
+	}
+
+	if err := builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter(
+		&daemonset.DaemonSetWebhook{
+			Client: mgr.GetClient(),
+		}).Complete(); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet")
+		os.Exit(1)
+	}
 }
diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -0,0 +1,26 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate--v1-pod
+  failurePolicy: Ignore
+  name: mpod-stage.infrastructure.vwn.cloud
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None
diff --git a/pkg/controller/daemonset/daemonset_controller_suite_test.go b/pkg/controller/daemonset/daemonset_controller_suite_test.go
@@ -33,6 +33,9 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	admissionv1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var cfg *rest.Config
@@ -47,8 +50,51 @@ var t *envtest.Environment
 var testCtx, testCancel = context.WithCancel(context.Background())
 
 var _ = BeforeSuite(func() {
+	failurePolicy := admissionv1.Ignore
+	sideEffects := admissionv1.SideEffectClassNone
+	webhookPath := "/mutate-apps-v1-daemonset"
+	webhookInstallOptions := envtest.WebhookInstallOptions{
+		MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "daemonset-operator",
+				},
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "MutatingWebhookConfiguration",
+					APIVersion: "admissionregistration.k8s.io/v1",
+				},
+				Webhooks: []admissionv1.MutatingWebhook{
+					{
+						Name:                    "daemonsets.wave.pusher.com",
+						AdmissionReviewVersions: []string{"v1"},
+						FailurePolicy:           &failurePolicy,
+						ClientConfig: admissionv1.WebhookClientConfig{
+							Service: &admissionv1.ServiceReference{
+								Path: &webhookPath,
+							},
+						},
+						Rules: []admissionv1.RuleWithOperations{
+							{
+								Operations: []admissionv1.OperationType{
+									admissionv1.Create,
+									admissionv1.Update,
+								},
+								Rule: admissionv1.Rule{
+									APIGroups:   []string{"apps"},
+									APIVersions: []string{"v1"},
+									Resources:   []string{"daemonsets"},
+								},
+							},
+						},
+						SideEffects: &sideEffects,
+					},
+				},
+			},
+		},
+	}
 	t = &envtest.Environment{
-		CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")},
+		WebhookInstallOptions: webhookInstallOptions,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crds")},
 	}
 
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go
@@ -29,11 +29,13 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	webhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
 var _ = Describe("DaemonSet controller Suite", func() {
@@ -94,6 +96,11 @@ var _ = Describe("DaemonSet controller Suite", func() {
 			Metrics: metricsserver.Options{
 				BindAddress: "0",
 			},
+			WebhookServer: webhook.NewServer(webhook.Options{
+				Host:    t.WebhookInstallOptions.LocalServingHost,
+				Port:    t.WebhookInstallOptions.LocalServingPort,
+				CertDir: t.WebhookInstallOptions.LocalServingCertDir,
+			}),
 		})
 		Expect(err).NotTo(HaveOccurred())
 		var cerr error
@@ -106,6 +113,14 @@ var _ = Describe("DaemonSet controller Suite", func() {
 		recFn, requestsStart, requests = SetupTestReconcile(r)
 		Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred())
 
+		// register mutating pod webhook
+		err = builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter(
+			&DaemonSetWebhook{
+				Client:  mgr.GetClient(),
+				Handler: r.handler,
+			}).Complete()
+		Expect(err).ToNot(HaveOccurred())
+
 		testCtx, testCancel = context.WithCancel(context.Background())
 		go Run(testCtx, mgr)
 
@@ -163,8 +178,6 @@ var _ = Describe("DaemonSet controller Suite", func() {
 				}
 				clearReconciled()
 				m.Update(daemonset, addAnnotation).Should(Succeed())
-				// Two runs since we the controller retriggers itself by changing the object
-				waitForDaemonSetReconciled(daemonset)
 				waitForDaemonSetReconciled(daemonset)
 
 				// Get the updated DaemonSet
@@ -205,7 +218,6 @@ var _ = Describe("DaemonSet controller Suite", func() {
 					clearReconciled()
 					m.Update(daemonset, removeContainer2).Should(Succeed())
 					waitForDaemonSetReconciled(daemonset)
-					waitForDaemonSetReconciled(daemonset)
 
 					// Get the updated DaemonSet
 					m.Get(daemonset, timeout).Should(Succeed())

diff --git a/pkg/controller/daemonset/daemonset_webhook.go b/pkg/controller/daemonset/daemonset_webhook.go
@@ -0,0 +1,26 @@
+package daemonset
+
+import (
+	"context"
+
+	"github.com/wave-k8s/wave/pkg/core"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// +kubebuilder:webhook:path=/mutate-apps-v1-daemonset,mutating=true,failurePolicy=ignore,groups=apps/v1,resources=daemonset,verbs=create;update,versions=v1,name=daemonset.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun
+type DaemonSetWebhook struct {
+	client.Client
+	Handler *core.Handler
+}
+
+func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	request, err := admission.RequestFromContext(ctx)
+	if err != nil {
+		return err
+	}
+	err = a.Handler.HandleDaemonSetWebhook(obj.(*appsv1.DaemonSet), request.DryRun)
+	return err
+}