From 531f331c178ca14889f8511beb9adb83679fb55f Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Tue, 30 Apr 2024 01:25:53 +0200 Subject: [PATCH] implement (optional) mutating webhooks reduce number of restarts of pods in deployment --- charts/wave/templates/webhook.yaml | 70 +++++++++++++++++++ .../wave/templates/webhook_certificate.yaml | 25 +++++++ charts/wave/templates/webhook_service.yaml | 15 ++++ charts/wave/values.yaml | 3 + cmd/manager/main.go | 30 ++++++++ config/webhook/manifests.yaml | 26 +++++++ .../daemonset_controller_suite_test.go | 48 ++++++++++++- .../daemonset/daemonset_controller_test.go | 18 ++++- pkg/controller/daemonset/daemonset_webhook.go | 26 +++++++ .../deployment_controller_suite_test.go | 48 ++++++++++++- .../deployment/deployment_controller_test.go | 18 ++++- .../deployment/deployment_webhook.go | 26 +++++++ .../statefulset_controller_suite_test.go | 48 ++++++++++++- .../statefulset_controller_test.go | 18 ++++- .../statefulset/statefulset_webhook.go | 26 +++++++ pkg/core/handler.go | 47 +++++++++++++ pkg/core/hash.go | 8 ++- 17 files changed, 487 insertions(+), 13 deletions(-) create mode 100644 charts/wave/templates/webhook.yaml create mode 100644 charts/wave/templates/webhook_certificate.yaml create mode 100644 charts/wave/templates/webhook_service.yaml create mode 100644 pkg/controller/daemonset/daemonset_webhook.go create mode 100644 pkg/controller/deployment/deployment_webhook.go create mode 100644 pkg/controller/statefulset/statefulset_webhook.go diff --git a/charts/wave/templates/webhook.yaml b/charts/wave/templates/webhook.yaml new file mode 100644 index 00000000..96d3cca2 --- /dev/null +++ b/charts/wave/templates/webhook.yaml @@ -0,0 +1,70 @@ +--- +{{- if .Values.webhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: '{{ template "wave-fullname" . }}-mutating-webhook-configuration' + annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ template "wave-fullname" . }}-serving-cert' +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-deployment + failurePolicy: Ignore + name: deployments.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-statefulset + failurePolicy: Ignore + name: statefulset.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: NoneOnDryRun + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "wave-fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /mutate-apps-v1-daemonset + failurePolicy: Ignore + name: daemonset.wave.pusher.com + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - daemonsets + sideEffects: NoneOnDryRun +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_certificate.yaml b/charts/wave/templates/webhook_certificate.yaml new file mode 100644 index 00000000..5d63f8ba --- /dev/null +++ b/charts/wave/templates/webhook_certificate.yaml @@ -0,0 +1,25 @@ +{{- if .Values.webhooks.enabled }} +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "wave-fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "wave-fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ template "wave-fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ template "wave-fullname" . }}-selfsigned-issuer + secretName: {{ template "wave-fullname" . }}-webhook-server-cert +{{- end }} \ No newline at end of file diff --git a/charts/wave/templates/webhook_service.yaml b/charts/wave/templates/webhook_service.yaml new file mode 100644 index 00000000..9e6109a6 --- /dev/null +++ b/charts/wave/templates/webhook_service.yaml @@ -0,0 +1,15 @@ +{{- if .Values.webhooks.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "wave-fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "wave-labels.chart" . | indent 4 }} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + {{- include "wave-labels.chart" . | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/charts/wave/values.yaml b/charts/wave/values.yaml index e9ee1951..d1b68088 100644 --- a/charts/wave/values.yaml +++ b/charts/wave/values.yaml @@ -43,6 +43,9 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: +webhooks: + enabled: false + # Period for reconciliation # syncPeriod: 5m diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 3065b755..0ae6476b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -23,13 +23,19 @@ import ( "runtime" "time" + appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/cache" "github.com/wave-k8s/wave/pkg/apis" "github.com/wave-k8s/wave/pkg/controller" + "github.com/wave-k8s/wave/pkg/controller/daemonset" + "github.com/wave-k8s/wave/pkg/controller/deployment" + "github.com/wave-k8s/wave/pkg/controller/statefulset" + "github.com/wave-k8s/wave/pkg/webhook" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -110,4 +116,28 @@ func main() { setupLog.Error(err, "unable to run the manager") os.Exit(1) } + + if err := builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter( + &deployment.DeploymentWebhook{ + Client: mgr.GetClient(), + }).Complete(); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Deployment") + os.Exit(1) + } + + if err := builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter( + &statefulset.StatefulSetWebhook{ + Client: mgr.GetClient(), + }).Complete(); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "StatefulSet") + os.Exit(1) + } + + if err := builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter( + &daemonset.DaemonSetWebhook{ + Client: mgr.GetClient(), + }).Complete(); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "DaemonSet") + os.Exit(1) + } } diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index e69de29b..ed4b4f55 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate--v1-pod + failurePolicy: Ignore + name: mpod-stage.infrastructure.vwn.cloud + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None diff --git a/pkg/controller/daemonset/daemonset_controller_suite_test.go b/pkg/controller/daemonset/daemonset_controller_suite_test.go index 54ad0c49..ba864bcd 100644 --- a/pkg/controller/daemonset/daemonset_controller_suite_test.go +++ b/pkg/controller/daemonset/daemonset_controller_suite_test.go @@ -33,6 +33,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -47,8 +50,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-daemonset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daemonset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "daemonsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"daemonsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go index 11fb86f4..0b7e2563 100644 --- a/pkg/controller/daemonset/daemonset_controller_test.go +++ b/pkg/controller/daemonset/daemonset_controller_test.go @@ -29,11 +29,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("DaemonSet controller Suite", func() { @@ -94,6 +96,11 @@ var _ = Describe("DaemonSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -106,6 +113,14 @@ var _ = Describe("DaemonSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter( + &DaemonSetWebhook{ + Client: mgr.GetClient(), + Handler: r.handler, + }).Complete() + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -163,8 +178,6 @@ var _ = Describe("DaemonSet controller Suite", func() { } clearReconciled() m.Update(daemonset, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForDaemonSetReconciled(daemonset) waitForDaemonSetReconciled(daemonset) // Get the updated DaemonSet @@ -205,7 +218,6 @@ var _ = Describe("DaemonSet controller Suite", func() { clearReconciled() m.Update(daemonset, removeContainer2).Should(Succeed()) waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) // Get the updated DaemonSet m.Get(daemonset, timeout).Should(Succeed()) diff --git a/pkg/controller/daemonset/daemonset_webhook.go b/pkg/controller/daemonset/daemonset_webhook.go new file mode 100644 index 00000000..b9717e04 --- /dev/null +++ b/pkg/controller/daemonset/daemonset_webhook.go @@ -0,0 +1,26 @@ +package daemonset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-daemonset,mutating=true,failurePolicy=ignore,groups=apps/v1,resources=daemonset,verbs=create;update,versions=v1,name=daemonset.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun +type DaemonSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDaemonSetWebhook(obj.(*appsv1.DaemonSet), request.DryRun) + return err +} diff --git a/pkg/controller/deployment/deployment_controller_suite_test.go b/pkg/controller/deployment/deployment_controller_suite_test.go index 2e36804b..19f34726 100644 --- a/pkg/controller/deployment/deployment_controller_suite_test.go +++ b/pkg/controller/deployment/deployment_controller_suite_test.go @@ -34,6 +34,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -48,8 +51,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-deployment" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "deployment-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "deployments.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"deployments"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/deployment/deployment_controller_test.go b/pkg/controller/deployment/deployment_controller_test.go index 39c43ec2..034f5331 100644 --- a/pkg/controller/deployment/deployment_controller_test.go +++ b/pkg/controller/deployment/deployment_controller_test.go @@ -31,10 +31,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("Deployment controller Suite", func() { @@ -101,6 +103,11 @@ var _ = Describe("Deployment controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) var cerr error @@ -113,6 +120,14 @@ var _ = Describe("Deployment controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter( + &DeploymentWebhook{ + Client: mgr.GetClient(), + Handler: r.handler, + }).Complete() + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -188,8 +203,6 @@ var _ = Describe("Deployment controller Suite", func() { } clearReconciled() m.Update(deployment, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForDeploymentReconciled(deployment) waitForDeploymentReconciled(deployment) // Get the updated Deployment @@ -232,7 +245,6 @@ var _ = Describe("Deployment controller Suite", func() { clearReconciled() m.Update(deployment, removeContainer2).Should(Succeed()) waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) // Get the updated Deployment m.Get(deployment, timeout).Should(Succeed()) diff --git a/pkg/controller/deployment/deployment_webhook.go b/pkg/controller/deployment/deployment_webhook.go new file mode 100644 index 00000000..0fbf951d --- /dev/null +++ b/pkg/controller/deployment/deployment_webhook.go @@ -0,0 +1,26 @@ +package deployment + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-deployment,mutating=true,failurePolicy=ignore,groups=apps/v1,resources=deployments,verbs=create;update,versions=v1,name=deployments.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun +type DeploymentWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *DeploymentWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleDeploymentWebhook(obj.(*appsv1.Deployment), request.DryRun) + return err +} diff --git a/pkg/controller/statefulset/statefulset_controller_suite_test.go b/pkg/controller/statefulset/statefulset_controller_suite_test.go index 689245de..6c5ba172 100644 --- a/pkg/controller/statefulset/statefulset_controller_suite_test.go +++ b/pkg/controller/statefulset/statefulset_controller_suite_test.go @@ -35,6 +35,9 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + admissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var cfg *rest.Config @@ -49,8 +52,51 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) var _ = BeforeSuite(func() { + failurePolicy := admissionv1.Ignore + sideEffects := admissionv1.SideEffectClassNone + webhookPath := "/mutate-apps-v1-statefulset" + webhookInstallOptions := envtest.WebhookInstallOptions{ + MutatingWebhooks: []*admissionv1.MutatingWebhookConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "statefulset-operator", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + Webhooks: []admissionv1.MutatingWebhook{ + { + Name: "statefulsets.wave.pusher.com", + AdmissionReviewVersions: []string{"v1"}, + FailurePolicy: &failurePolicy, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Path: &webhookPath, + }, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"statefulsets"}, + }, + }, + }, + SideEffects: &sideEffects, + }, + }, + }, + }, + } t = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + WebhookInstallOptions: webhookInstallOptions, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, } apis.AddToScheme(scheme.Scheme) diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index 6e8a2221..07d08346 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -31,10 +31,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("StatefulSet controller Suite", func() { @@ -95,6 +97,11 @@ var _ = Describe("StatefulSet controller Suite", func() { Metrics: metricsserver.Options{ BindAddress: "0", }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: t.WebhookInstallOptions.LocalServingHost, + Port: t.WebhookInstallOptions.LocalServingPort, + CertDir: t.WebhookInstallOptions.LocalServingCertDir, + }), }) Expect(err).NotTo(HaveOccurred()) @@ -109,6 +116,14 @@ var _ = Describe("StatefulSet controller Suite", func() { recFn, requestsStart, requests = SetupTestReconcile(r) Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + // register mutating pod webhook + err = builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter( + &StatefulSetWebhook{ + Client: mgr.GetClient(), + Handler: r.handler, + }).Complete() + Expect(err).ToNot(HaveOccurred()) + testCtx, testCancel = context.WithCancel(context.Background()) go Run(testCtx, mgr) @@ -166,8 +181,6 @@ var _ = Describe("StatefulSet controller Suite", func() { } clearReconciled() m.Update(statefulset, addAnnotation).Should(Succeed()) - // Two runs since we the controller retriggers itself by changing the object - waitForStatefulSetReconciled(statefulset) waitForStatefulSetReconciled(statefulset) // Get the updated StatefulSet @@ -210,7 +223,6 @@ var _ = Describe("StatefulSet controller Suite", func() { clearReconciled() m.Update(statefulset, removeContainer2).Should(Succeed()) waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) // Get the updated StatefulSet m.Get(statefulset, timeout).Should(Succeed()) diff --git a/pkg/controller/statefulset/statefulset_webhook.go b/pkg/controller/statefulset/statefulset_webhook.go new file mode 100644 index 00000000..4fa93dcc --- /dev/null +++ b/pkg/controller/statefulset/statefulset_webhook.go @@ -0,0 +1,26 @@ +package statefulset + +import ( + "context" + + "github.com/wave-k8s/wave/pkg/core" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// +kubebuilder:webhook:path=/mutate-apps-v1-statefulset,mutating=true,failurePolicy=ignore,groups=apps/v1,resources=statefulsets,verbs=create;update,versions=v1,name=statefulsets.wave.pusher.com,admissionReviewVersions=v1,sideEffects=NoneOnDryRun +type StatefulSetWebhook struct { + client.Client + Handler *core.Handler +} + +func (a *StatefulSetWebhook) Default(ctx context.Context, obj runtime.Object) error { + request, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + err = a.Handler.HandleStatefulSetWebhook(obj.(*appsv1.StatefulSet), request.DryRun) + return err +} diff --git a/pkg/core/handler.go b/pkg/core/handler.go index 588eddf6..5248ee6d 100644 --- a/pkg/core/handler.go +++ b/pkg/core/handler.go @@ -48,6 +48,21 @@ func (h *Handler) HandleDeployment(instance *appsv1.Deployment) (reconcile.Resul return h.handlePodController(&deployment{Deployment: instance}) } +// HandleDeploymentWebhook is called by the deployment webhook +func (h *Handler) HandleDeploymentWebhook(instance *appsv1.Deployment, dryRun *bool) error { + return h.updatePodController(&deployment{Deployment: instance}, (dryRun != nil && *dryRun)) +} + +// HandleStatefulSetWebhook is called by the statefulset webhook +func (h *Handler) HandleStatefulSetWebhook(instance *appsv1.StatefulSet, dryRun *bool) error { + return h.updatePodController(&statefulset{StatefulSet: instance}, (dryRun != nil && *dryRun)) +} + +// HandleDaemonSetWebhook is called by the daemonset webhook +func (h *Handler) HandleDaemonSetWebhook(instance *appsv1.DaemonSet, dryRun *bool) error { + return h.updatePodController(&daemonset{DaemonSet: instance}, (dryRun != nil && *dryRun)) +} + // HandleStatefulSet is called by the StatefulSet controller to reconcile StatefulSets func (h *Handler) HandleStatefulSet(instance *appsv1.StatefulSet) (reconcile.Result, error) { return h.handlePodController(&statefulset{StatefulSet: instance}) @@ -104,3 +119,35 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, } return reconcile.Result{}, nil } + +// handlePodController will only update the hash. Everything else is left to the reconciler. +func (h *Handler) updatePodController(instance podController, dryRun bool) error { + log := logf.Log.WithName("wave").WithValues("namespace", instance.GetNamespace(), "name", instance.GetName()) + + // If the required annotation isn't present, ignore the instance + if !hasRequiredAnnotation(instance) { + return nil + } + + // Get all children that the instance currently references + current, err := h.getCurrentChildren(instance) + if err != nil { + return fmt.Errorf("error fetching current children: %v", err) + } + + hash, err := calculateConfigHash(current) + if err != nil { + return fmt.Errorf("error calculating configuration hash: %v", err) + } + + // Update the desired state of the Deployment + oldHash := getConfigHash(instance) + setConfigHash(instance, hash) + + if !dryRun && oldHash != hash { + log.V(0).Info("Updating instance hash", "hash", oldHash) + h.recorder.Eventf(instance.GetApiObject(), corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) + } + + return nil +} diff --git a/pkg/core/hash.go b/pkg/core/hash.go index 85befda8..c179f73b 100644 --- a/pkg/core/hash.go +++ b/pkg/core/hash.go @@ -105,7 +105,7 @@ func getSecretData(child configObject) map[string][]byte { return keyData } -// setConfigHash upates the configuration hash of the given Deployment to the +// setConfigHash updates the configuration hash of the given Deployment to the // given string func setConfigHash(obj podController, hash string) { // Get the existing annotations @@ -120,3 +120,9 @@ func setConfigHash(obj podController, hash string) { podTemplate.SetAnnotations(annotations) obj.SetPodTemplate(podTemplate) } + +// getConfigHash return the config hash string +func getConfigHash(obj podController) string { + podTemplate := obj.GetPodTemplate() + return podTemplate.GetAnnotations()[ConfigHashAnnotation] +}