implement mutation webhooks

- add (optional) mutating wehhook which will add hash on creation of deployments. this will prevent pod restarts when configmaps/secrets already exist
- fix non-404 error handle when loading children
- fix RBACs
- test minimal and typical production setup
- fixed reconcile loop when secrets/configmaps are missing
- watch not (yet) existing secrets/configmaps for much faster updates when they are created
- when using webhooks: disable scheduling when secrets/configmaps; reenable scheduling once secrets/configmaps; this prevents unneccessary container restarts

.github/workflows/e2e-test.yaml

@@ -10,6 +10,9 @@ jobs:
         version:
           - v1.21
           - v1.29
+        setup:
+          - minimal
+          - production
     runs-on: ubuntu-latest
     name: test on minikube
     steps:
@@ -19,4 +22,4 @@ jobs:
         with:
           kubernetes-version: ${{ matrix.version }}
       - name: Build and run wave
-        run: hack/run-test-in-minikube.sh
+        run: hack/run-test-in-minikube.sh ${{ matrix.setup }}

charts/wave/templates/clusterrole.yaml

@@ -37,4 +37,10 @@ rules:
       - update
       - patch
       - watch
+  - verbs:
+      - '*'
+    apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
 {{- end }}

charts/wave/templates/deployment.yaml

@@ -32,12 +32,36 @@ spec:
           {{- if .Values.syncPeriod }}
             - --sync-period={{ .Values.syncPeriod }}
           {{- end }}
-          volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 12 }}
+          {{- if .Values.webhooks.enabled }}
+            - --enable-webhooks=true
+          {{- end }}
+          volumeMounts:
+          {{- if .Values.webhooks.enabled }}
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+          {{- end }} 
+          {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - containerPort: 9443
+              name: webhook-server
+              protocol: TCP
           resources: {{- toYaml .Values.resources | nindent 12 }}
       securityContext: {{ toYaml .Values.securityContext | nindent 8 }}
       serviceAccountName: {{ .Values.serviceAccount.name | default (include "wave-fullname" .) }}
       nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}
       affinity: {{ toYaml .Values.affinity | nindent 8 }}
       tolerations: {{ toYaml .Values.tolerations | nindent 8 }}
+      volumes:
+      {{- if .Values.webhooks.enabled }}
+        - name: cert
+          secret:
+            defaultMode: 420
+            secretName: {{ template "wave-fullname" . }}-webhook-server-cert
+      {{- end }}
+      {{- with .Values.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       topologySpreadConstraints: {{ toYaml .Values.topologySpreadConstraints | nindent 8 }}
-      volumes: {{ toYaml .Values.extraVolumes | nindent 8 }}

charts/wave/templates/poddisruptionbudget.yaml

@@ -7,6 +7,6 @@ metadata:
 spec:
   selector:
     matchLabels:
-      {{ include "wave.selectorLabels" . | nindent 6}}
+      {{ include "wave-labels.chart" . | nindent 6 }}
   maxUnavailable: 1
 {{- end }}

charts/wave/templates/webhook.yaml

@@ -0,0 +1,70 @@
+---
+{{- if .Values.webhooks.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration'
+  annotations:
+    cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert'
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-deployment
+    failurePolicy: Ignore
+    name: deployments.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - deployments
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-statefulset
+    failurePolicy: Ignore
+    name: statefulsets.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - statefulsets
+    sideEffects: NoneOnDryRun
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "wave-fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-apps-v1-daemonset
+    failurePolicy: Ignore
+    name: daemonsets.wave.pusher.com
+    rules:
+      - apiGroups:
+          - apps
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - daemonsets
+    sideEffects: NoneOnDryRun
+{{- end }}

charts/wave/templates/webhook_certificate.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.webhooks.enabled }}
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "wave-fullname" . }}-serving-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  dnsNames:
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc
+  - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ template "wave-fullname" . }}-selfsigned-issuer
+  secretName: {{ template "wave-fullname" . }}-webhook-server-cert
+{{- end }}

charts/wave/templates/webhook_service.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.webhooks.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "wave-fullname" . }}-webhook-service
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{ include "wave-labels.chart" . | nindent 4 }}
+spec:
+  ports:
+    - port: 443
+      targetPort: 9443
+  selector:
+    {{ include "wave-labels.chart" . | nindent 4 }}
+{{- end }}

charts/wave/values.yaml

@@ -47,6 +47,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+webhooks:
+  enabled: false
+
 # Period for reconciliation
 # syncPeriod: 5m
 

cmd/manager/main.go

@@ -27,7 +27,11 @@ import (
 
 	"github.com/wave-k8s/wave/pkg/apis"
 	"github.com/wave-k8s/wave/pkg/controller"
-	"github.com/wave-k8s/wave/pkg/webhook"
+	"github.com/wave-k8s/wave/pkg/controller/daemonset"
+	"github.com/wave-k8s/wave/pkg/controller/deployment"
+	"github.com/wave-k8s/wave/pkg/controller/statefulset"
+	k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
+
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -42,6 +46,7 @@ var (
 	leaderElectionNamespace = flag.String("leader-election-namespace", "", "Namespace for the configmap used by the leader election system")
 	syncPeriod              = flag.Duration("sync-period", 5*time.Minute, "Reconcile sync period")
 	showVersion             = flag.Bool("version", false, "Show version and exit")
+	enableWebhooks          = flag.Bool("enable-webhooks", false, "Enable webhooks")
 	setupLog                = ctrl.Log.WithName("setup")
 )
 
@@ -69,7 +74,14 @@ func main() {
 
 	// Create a new Cmd to provide shared dependencies and start components
 	setupLog.Info("setting up manager")
+	var webhookServer k8swebhook.Server
+	if *enableWebhooks {
+		webhookServer = k8swebhook.NewServer(k8swebhook.Options{
+			Port: 9443,
+		})
+	}
 	mgr, err := manager.New(cfg, manager.Options{
+		WebhookServer:           webhookServer,
 		LeaderElection:          *leaderElection,
 		LeaderElectionID:        *leaderElectionID,
 		LeaderElectionNamespace: *leaderElectionNamespace,
@@ -97,11 +109,21 @@ func main() {
 		setupLog.Error(err, "unable to register controllers to the manager")
 		os.Exit(1)
 	}
-
-	setupLog.Info("setting up webhooks")
-	if err := webhook.AddToManager(mgr); err != nil {
-		setupLog.Error(err, "unable to register webhooks to the manager")
-		os.Exit(1)
+	if *enableWebhooks {
+		if err := deployment.AddDeploymentWebhook(mgr); err != nil {
+			setupLog.Error(err, "unable to create webhook", "webhook", "Deployment")
+			os.Exit(1)
+		}
+
+		if err := statefulset.AddStatefulSetWebhook(mgr); err != nil {
+			setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet")
+			os.Exit(1)
+		}
+
+		if err := daemonset.AddDaemonSetWebhook(mgr); err != nil {
+			setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet")
+			os.Exit(1)
+		}
 	}
 
 	// Start the Cmd

config/webhook/manifests.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-daemonset
+  failurePolicy: Ignore
+  name: daemonsets.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - daemonsets
+  sideEffects: NoneOnDryRun
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-deployment
+  failurePolicy: Ignore
+  name: deployments.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - deployments
+  sideEffects: NoneOnDryRun
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-statefulset
+  failurePolicy: Ignore
+  name: statefulsets.wave.pusher.com
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - statefulsets
+  sideEffects: NoneOnDryRun

hack/production.yaml

@@ -0,0 +1,16 @@
+# production setup used in tests
+replicas: 2
+
+webhooks:
+  enabled: true
+
+pdb:
+  enabled: true
+
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: topology.kubernetes.io/zone
+    whenUnsatisfiable: ScheduleAnyway
+    labelSelector:
+      matchLabels:
+        app: wave

hack/run-test-in-minikube.sh

@@ -33,7 +33,16 @@ eval $(minikube -p minikube docker-env)
 docker build -f ./Dockerfile -t wave-local:local .
 
 echo Installing wave...
-helm install wave charts/wave --set image.name=wave-local --set image.tag=local
+if [ "$1" = "production" ]; then
+	# Install cert-manager
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
+	while [ "$(kubectl get pods -n cert-manager | grep 'webhook' | grep -c '1/1')" -ne 1 ]; do echo Waiting for \"cert-manager-webhook\" to start; sleep 10; done
+	# Production setup
+	helm install wave charts/wave --set image.name=wave-local --set image.tag=local -f hack/production.yaml
+else
+	# Default install without values
+	helm install wave charts/wave --set image.name=wave-local --set image.tag=local
+fi
 
 while [ "$(kubectl get pods -A | grep -cEv 'Running|Completed')" -gt 1 ]; do echo Waiting for \"cluster\" to start; sleep 10; done