From 06eaa24e69c192d7f4b642d78c2690f07cfd2c3a Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Thu, 9 May 2024 22:34:13 +0200 Subject: [PATCH 1/8] convert codebase to generics --- .../daemonset/daemonset_controller.go | 8 +- pkg/controller/daemonset/daemonset_webhook.go | 6 +- .../deployment/deployment_controller.go | 8 +- .../deployment/deployment_webhook.go | 6 +- .../statefulset/statefulset_controller.go | 8 +- .../statefulset/statefulset_webhook.go | 6 +- pkg/core/children.go | 18 +-- pkg/core/children_test.go | 8 +- pkg/core/delete.go | 10 +- pkg/core/delete_test.go | 8 +- pkg/core/finalizer.go | 4 +- pkg/core/finalizer_test.go | 4 +- pkg/core/handler.go | 62 ++++------- pkg/core/handler_test.go | 30 ++--- pkg/core/hash.go | 10 +- pkg/core/hash_test.go | 4 +- pkg/core/owner_references.go | 9 +- pkg/core/owner_references_test.go | 8 +- pkg/core/required_annotation.go | 2 +- pkg/core/required_annotation_test.go | 4 +- pkg/core/scheduler.go | 16 +-- pkg/core/scheduler_test.go | 8 +- pkg/core/types.go | 104 +++++------------- pkg/core/watcher.go | 10 +- 24 files changed, 148 insertions(+), 213 deletions(-) diff --git a/pkg/controller/daemonset/daemonset_controller.go b/pkg/controller/daemonset/daemonset_controller.go index 6467042f..c5197049 100644 --- a/pkg/controller/daemonset/daemonset_controller.go +++ b/pkg/controller/daemonset/daemonset_controller.go @@ -48,12 +48,12 @@ func Add(mgr manager.Manager) error { func newReconciler(mgr manager.Manager) *ReconcileDaemonSet { return &ReconcileDaemonSet{ scheme: mgr.GetScheme(), - handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + handler: core.NewHandler[*appsv1.DaemonSet](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), } } // add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler) error { +func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.DaemonSet]) error { // Create a new controller c, err := controller.New("daemonset-controller", mgr, controller.Options{Reconciler: r}) if err != nil { @@ -85,7 +85,7 @@ var _ reconcile.Reconciler = &ReconcileDaemonSet{} // ReconcileDaemonSet reconciles a DaemonSet object type ReconcileDaemonSet struct { scheme *runtime.Scheme - handler *core.Handler + handler *core.Handler[*appsv1.DaemonSet] } // Reconcile reads that state of the cluster for a DaemonSet object and @@ -104,5 +104,5 @@ func (r *ReconcileDaemonSet) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - return r.handler.HandleDaemonSet(instance) + return r.handler.Handle(instance) } diff --git a/pkg/controller/daemonset/daemonset_webhook.go b/pkg/controller/daemonset/daemonset_webhook.go index eff15322..8d6a55a5 100644 --- a/pkg/controller/daemonset/daemonset_webhook.go +++ b/pkg/controller/daemonset/daemonset_webhook.go @@ -16,7 +16,7 @@ import ( type DaemonSetWebhook struct { client.Client - Handler *core.Handler + Handler *core.Handler[*appsv1.DaemonSet] } func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) error { @@ -24,7 +24,7 @@ func (a *DaemonSetWebhook) Default(ctx context.Context, obj runtime.Object) erro if err != nil { return err } - err = a.Handler.HandleDaemonSetWebhook(obj.(*appsv1.DaemonSet), request.DryRun, request.Operation == "CREATE") + err = a.Handler.HandleWebhook(obj.(*appsv1.DaemonSet), request.DryRun, request.Operation == "CREATE") return err } @@ -32,7 +32,7 @@ func AddDaemonSetWebhook(mgr manager.Manager) error { err := builder.WebhookManagedBy(mgr).For(&appsv1.DaemonSet{}).WithDefaulter( &DaemonSetWebhook{ Client: mgr.GetClient(), - Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + Handler: core.NewHandler[*appsv1.DaemonSet](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), }).Complete() return err diff --git a/pkg/controller/deployment/deployment_controller.go b/pkg/controller/deployment/deployment_controller.go index ab5b1690..32b9f7f0 100644 --- a/pkg/controller/deployment/deployment_controller.go +++ b/pkg/controller/deployment/deployment_controller.go @@ -48,12 +48,12 @@ func Add(mgr manager.Manager) error { func newReconciler(mgr manager.Manager) *ReconcileDeployment { return &ReconcileDeployment{ scheme: mgr.GetScheme(), - handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + handler: core.NewHandler[*appsv1.Deployment](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), } } // add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler) error { +func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.Deployment]) error { // Create a new controller c, err := controller.New("deployment-controller", mgr, controller.Options{Reconciler: r}) if err != nil { @@ -86,7 +86,7 @@ var _ reconcile.Reconciler = &ReconcileDeployment{} // ReconcileDeployment reconciles a Deployment object type ReconcileDeployment struct { scheme *runtime.Scheme - handler *core.Handler + handler *core.Handler[*appsv1.Deployment] } // Reconcile reads that state of the cluster for a Deployment object and @@ -105,5 +105,5 @@ func (r *ReconcileDeployment) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - return r.handler.HandleDeployment(instance) + return r.handler.Handle(instance) } diff --git a/pkg/controller/deployment/deployment_webhook.go b/pkg/controller/deployment/deployment_webhook.go index 7ba9e865..277ffa45 100644 --- a/pkg/controller/deployment/deployment_webhook.go +++ b/pkg/controller/deployment/deployment_webhook.go @@ -16,7 +16,7 @@ import ( type DeploymentWebhook struct { client.Client - Handler *core.Handler + Handler *core.Handler[*appsv1.Deployment] } func (a *DeploymentWebhook) Default(ctx context.Context, obj runtime.Object) error { @@ -24,7 +24,7 @@ func (a *DeploymentWebhook) Default(ctx context.Context, obj runtime.Object) err if err != nil { return err } - err = a.Handler.HandleDeploymentWebhook(obj.(*appsv1.Deployment), request.DryRun, request.Operation == "CREATE") + err = a.Handler.HandleWebhook(obj.(*appsv1.Deployment), request.DryRun, request.Operation == "CREATE") return err } @@ -32,7 +32,7 @@ func AddDeploymentWebhook(mgr manager.Manager) error { err := builder.WebhookManagedBy(mgr).For(&appsv1.Deployment{}).WithDefaulter( &DeploymentWebhook{ Client: mgr.GetClient(), - Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + Handler: core.NewHandler[*appsv1.Deployment](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), }).Complete() return err diff --git a/pkg/controller/statefulset/statefulset_controller.go b/pkg/controller/statefulset/statefulset_controller.go index 12ff6aca..6335e1b7 100644 --- a/pkg/controller/statefulset/statefulset_controller.go +++ b/pkg/controller/statefulset/statefulset_controller.go @@ -48,12 +48,12 @@ func Add(mgr manager.Manager) error { func newReconciler(mgr manager.Manager) *ReconcileStatefulSet { return &ReconcileStatefulSet{ scheme: mgr.GetScheme(), - handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + handler: core.NewHandler[*appsv1.StatefulSet](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), } } // add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler) error { +func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.StatefulSet]) error { // Create a new controller c, err := controller.New("statefulset-controller", mgr, controller.Options{Reconciler: r}) if err != nil { @@ -86,7 +86,7 @@ var _ reconcile.Reconciler = &ReconcileStatefulSet{} // ReconcileStatefulSet reconciles a StatefulSet object type ReconcileStatefulSet struct { scheme *runtime.Scheme - handler *core.Handler + handler *core.Handler[*appsv1.StatefulSet] } // Reconcile reads that state of the cluster for a StatefulSet object and @@ -105,5 +105,5 @@ func (r *ReconcileStatefulSet) Reconcile(ctx context.Context, request reconcile. return reconcile.Result{}, err } - return r.handler.HandleStatefulSet(instance) + return r.handler.Handle(instance) } diff --git a/pkg/controller/statefulset/statefulset_webhook.go b/pkg/controller/statefulset/statefulset_webhook.go index ddb6c9ae..d1874436 100644 --- a/pkg/controller/statefulset/statefulset_webhook.go +++ b/pkg/controller/statefulset/statefulset_webhook.go @@ -16,7 +16,7 @@ import ( type StatefulSetWebhook struct { client.Client - Handler *core.Handler + Handler *core.Handler[*appsv1.StatefulSet] } func (a *StatefulSetWebhook) Default(ctx context.Context, obj runtime.Object) error { @@ -24,7 +24,7 @@ func (a *StatefulSetWebhook) Default(ctx context.Context, obj runtime.Object) er if err != nil { return err } - err = a.Handler.HandleStatefulSetWebhook(obj.(*appsv1.StatefulSet), request.DryRun, request.Operation == "CREATE") + err = a.Handler.HandleWebhook(obj.(*appsv1.StatefulSet), request.DryRun, request.Operation == "CREATE") return err } @@ -32,7 +32,7 @@ func AddStatefulSetWebhook(mgr manager.Manager) error { err := builder.WebhookManagedBy(mgr).For(&appsv1.StatefulSet{}).WithDefaulter( &StatefulSetWebhook{ Client: mgr.GetClient(), - Handler: core.NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")), + Handler: core.NewHandler[*appsv1.StatefulSet](mgr.GetClient(), mgr.GetEventRecorderFor("wave")), }).Complete() return err diff --git a/pkg/core/children.go b/pkg/core/children.go index e52246d1..49a8bb3b 100644 --- a/pkg/core/children.go +++ b/pkg/core/children.go @@ -62,7 +62,7 @@ type getResult struct { // referenced in the Deployment's spec. Any reference to a whole ConfigMap or Secret // (i.e. via an EnvFrom or a Volume) will result in one entry in the list, irrespective of // whether individual elements are also references (i.e. via an Env entry). -func (h *Handler) getCurrentChildren(configMaps configMetadataMap, secrets configMetadataMap) ([]configObject, error) { +func (h *Handler[I]) getCurrentChildren(configMaps configMetadataMap, secrets configMetadataMap) ([]configObject, error) { // get all of ConfigMaps and Secrets resultsChan := make(chan getResult) for name, metadata := range configMaps { @@ -116,14 +116,14 @@ func (h *Handler) getCurrentChildren(configMaps configMetadataMap, secrets confi // getChildNamesByType parses the Deployment object and returns two maps, // the first containing ConfigMap metadata for all referenced ConfigMaps, keyed on the name of the ConfigMap, // the second containing Secret metadata for all referenced Secrets, keyed on the name of the Secrets -func getChildNamesByType(obj podController) (configMetadataMap, configMetadataMap) { +func getChildNamesByType[I InstanceType](obj I) (configMetadataMap, configMetadataMap) { // Create sets for storing the names fo the ConfigMaps/Secrets configMaps := make(configMetadataMap) secrets := make(configMetadataMap) // Range through all Volumes and check the VolumeSources for ConfigMaps // and Secrets - for _, vol := range obj.GetPodTemplate().Spec.Volumes { + for _, vol := range GetPodTemplate(obj).Spec.Volumes { if cm := vol.VolumeSource.ConfigMap; cm != nil { configMaps[GetNamespacedName(cm.Name, obj.GetNamespace())] = configMetadata{required: isRequired(cm.Optional), allKeys: true} } @@ -185,7 +185,7 @@ func getChildNamesByType(obj podController) (configMetadataMap, configMetadataMa // Range through all Containers and their respective EnvFrom, // then check the EnvFromSources for ConfigMaps and Secrets - for _, container := range obj.GetPodTemplate().Spec.Containers { + for _, container := range GetPodTemplate(obj).Spec.Containers { for _, env := range container.EnvFrom { if cm := env.ConfigMapRef; cm != nil { configMaps[GetNamespacedName(cm.Name, obj.GetNamespace())] = configMetadata{required: isRequired(cm.Optional), allKeys: true} @@ -197,7 +197,7 @@ func getChildNamesByType(obj podController) (configMetadataMap, configMetadataMa } // Range through all Containers and their respective Env - for _, container := range obj.GetPodTemplate().Spec.Containers { + for _, container := range GetPodTemplate(obj).Spec.Containers { for _, env := range container.Env { if valFrom := env.ValueFrom; valFrom != nil { if cm := valFrom.ConfigMapKeyRef; cm != nil { @@ -247,19 +247,19 @@ func parseSecretKeyRef(metadata configMetadata, s *corev1.SecretKeySelector) con // getConfigMap gets a ConfigMap with the given name and namespace from the // API server. -func (h *Handler) getConfigMap(name types.NamespacedName, metadata configMetadata) getResult { +func (h *Handler[I]) getConfigMap(name types.NamespacedName, metadata configMetadata) getResult { return h.getObject(name, metadata, &corev1.ConfigMap{}) } // getSecret gets a Secret with the given name and namespace from the // API server. -func (h *Handler) getSecret(name types.NamespacedName, metadata configMetadata) getResult { +func (h *Handler[I]) getSecret(name types.NamespacedName, metadata configMetadata) getResult { return h.getObject(name, metadata, &corev1.Secret{}) } // getObject gets the Object with the given name and namespace from the API // server -func (h *Handler) getObject(objectName types.NamespacedName, metadata configMetadata, obj Object) getResult { +func (h *Handler[I]) getObject(objectName types.NamespacedName, metadata configMetadata, obj Object) getResult { err := h.Get(context.TODO(), objectName, obj) if err != nil { if errors.IsNotFound(err) { @@ -276,7 +276,7 @@ func (h *Handler) getObject(objectName types.NamespacedName, metadata configMeta // getExistingChildren returns a list of all Secrets and ConfigMaps that are // owned by the Deployment instance -func (h *Handler) getExistingChildren(obj podController) ([]Object, error) { +func (h *Handler[I]) getExistingChildren(obj I) ([]Object, error) { inNamespace := client.InNamespace(obj.GetNamespace()) // List all ConfigMaps in the Deployment's namespace diff --git a/pkg/core/children_test.go b/pkg/core/children_test.go index 5f1dc174..6e4babdd 100644 --- a/pkg/core/children_test.go +++ b/pkg/core/children_test.go @@ -35,10 +35,10 @@ import ( var _ = Describe("Wave children Suite", func() { var c client.Client - var h *Handler + var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment var currentChildren []configObject var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -70,7 +70,7 @@ var _ = Describe("Wave children Suite", func() { Expect(cerr).NotTo(HaveOccurred()) c = mgr.GetClient() // h = NewHandler(c, mgr.GetEventRecorderFor("wave")) - h = NewHandler(mgr.GetClient(), mgr.GetEventRecorderFor("wave")) + h = NewHandler[*appsv1.Deployment](mgr.GetClient(), mgr.GetEventRecorderFor("wave")) m = utils.Matcher{Client: c} @@ -102,7 +102,7 @@ var _ = Describe("Wave children Suite", func() { m.Create(s6).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) diff --git a/pkg/core/delete.go b/pkg/core/delete.go index f40252bc..b77df077 100644 --- a/pkg/core/delete.go +++ b/pkg/core/delete.go @@ -19,14 +19,13 @@ package core import ( "context" "fmt" - "reflect" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // deleteOwnerReferencesAndFinalizer removes all existing Owner References pointing to the object // before removing the object's Finalizer -func (h *Handler) deleteOwnerReferencesAndFinalizer(obj podController) (reconcile.Result, error) { +func (h *Handler[I]) deleteOwnerReferencesAndFinalizer(obj I) (reconcile.Result, error) { // Fetch all children with an OwnerReference pointing to the object existing, err := h.getExistingChildren(obj) if err != nil { @@ -40,10 +39,9 @@ func (h *Handler) deleteOwnerReferencesAndFinalizer(obj podController) (reconcil } // Remove the object's Finalizer and update if necessary - copy := obj.DeepCopyPodController() - removeFinalizer(copy) - if !reflect.DeepEqual(obj, copy) { - err := h.Update(context.TODO(), copy.GetApiObject()) + if hasFinalizer(obj) { + removeFinalizer(obj) + err := h.Update(context.TODO(), obj) if err != nil { return reconcile.Result{}, fmt.Errorf("error updating Deployment: %v", err) } diff --git a/pkg/core/delete_test.go b/pkg/core/delete_test.go index fee1b95b..96cb9ec0 100644 --- a/pkg/core/delete_test.go +++ b/pkg/core/delete_test.go @@ -35,10 +35,10 @@ import ( var _ = Describe("Wave migration Suite", func() { var c client.Client - var h *Handler + var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -56,7 +56,7 @@ var _ = Describe("Wave migration Suite", func() { var cerr error c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(cerr).NotTo(HaveOccurred()) - h = NewHandler(c, mgr.GetEventRecorderFor("wave")) + h = NewHandler[*appsv1.Deployment](c, mgr.GetEventRecorderFor("wave")) m = utils.Matcher{Client: c} // Create some configmaps and secrets @@ -66,7 +66,7 @@ var _ = Describe("Wave migration Suite", func() { m.Create(utils.ExampleSecret2.DeepCopy()).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) diff --git a/pkg/core/finalizer.go b/pkg/core/finalizer.go index 9b5f3b68..49379c20 100644 --- a/pkg/core/finalizer.go +++ b/pkg/core/finalizer.go @@ -17,7 +17,7 @@ limitations under the License. package core // removeFinalizer removes the wave finalizer from the given podController -func removeFinalizer(obj podController) { +func removeFinalizer[I InstanceType](obj I) { finalizers := obj.GetFinalizers() // Filter existing finalizers removing any that match the finalizerString @@ -33,7 +33,7 @@ func removeFinalizer(obj podController) { } // hasFinalizer checks for the presence of the Wave finalizer -func hasFinalizer(obj podController) bool { +func hasFinalizer[I InstanceType](obj I) bool { finalizers := obj.GetFinalizers() for _, finalizer := range finalizers { if finalizer == FinalizerString { diff --git a/pkg/core/finalizer_test.go b/pkg/core/finalizer_test.go index 3d09d653..9f5038f4 100644 --- a/pkg/core/finalizer_test.go +++ b/pkg/core/finalizer_test.go @@ -25,11 +25,11 @@ import ( var _ = Describe("Wave finalizer Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject // TODO: remove }) Context("removeFinalizer", func() { diff --git a/pkg/core/handler.go b/pkg/core/handler.go index 20e13c4e..a91344f2 100644 --- a/pkg/core/handler.go +++ b/pkg/core/handler.go @@ -19,10 +19,8 @@ package core import ( "context" "fmt" - "reflect" "sync" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -32,7 +30,7 @@ import ( ) // Handler performs the main business logic of the Wave controller -type Handler struct { +type Handler[I InstanceType] struct { client.Client recorder record.EventRecorder watchedConfigmaps WatcherList @@ -40,8 +38,8 @@ type Handler struct { } // NewHandler constructs a new instance of Handler -func NewHandler(c client.Client, r record.EventRecorder) *Handler { - return &Handler{Client: c, recorder: r, +func NewHandler[I InstanceType](c client.Client, r record.EventRecorder) *Handler[I] { + return &Handler[I]{Client: c, recorder: r, watchedConfigmaps: WatcherList{ watchers: make(map[types.NamespacedName]map[types.NamespacedName]bool), watchersMutex: &sync.RWMutex{}, @@ -52,38 +50,18 @@ func NewHandler(c client.Client, r record.EventRecorder) *Handler { }} } -// HandleDeployment is called by the deployment controller to reconcile deployments -func (h *Handler) HandleDeployment(instance *appsv1.Deployment) (reconcile.Result, error) { - return h.handlePodController(&deployment{Deployment: instance}) +// HandleWebhook is called by the webhook +func (h *Handler[I]) HandleWebhook(instance I, dryRun *bool, isCreate bool) error { + return h.updatePodController(instance, (dryRun != nil && *dryRun), isCreate) } -// HandleDeploymentWebhook is called by the deployment webhook -func (h *Handler) HandleDeploymentWebhook(instance *appsv1.Deployment, dryRun *bool, isCreate bool) error { - return h.updatePodController(&deployment{Deployment: instance}, (dryRun != nil && *dryRun), isCreate) -} - -// HandleStatefulSetWebhook is called by the statefulset webhook -func (h *Handler) HandleStatefulSetWebhook(instance *appsv1.StatefulSet, dryRun *bool, isCreate bool) error { - return h.updatePodController(&statefulset{StatefulSet: instance}, (dryRun != nil && *dryRun), isCreate) -} - -// HandleDaemonSetWebhook is called by the daemonset webhook -func (h *Handler) HandleDaemonSetWebhook(instance *appsv1.DaemonSet, dryRun *bool, isCreate bool) error { - return h.updatePodController(&daemonset{DaemonSet: instance}, (dryRun != nil && *dryRun), isCreate) -} - -// HandleStatefulSet is called by the StatefulSet controller to reconcile StatefulSets -func (h *Handler) HandleStatefulSet(instance *appsv1.StatefulSet) (reconcile.Result, error) { - return h.handlePodController(&statefulset{StatefulSet: instance}) -} - -// HandleDaemonSet is called by the DaemonSet controller to reconcile DaemonSets -func (h *Handler) HandleDaemonSet(instance *appsv1.DaemonSet) (reconcile.Result, error) { - return h.handlePodController(&daemonset{DaemonSet: instance}) +// Handle is called by the controller to reconcile its object +func (h *Handler[I]) Handle(instance I) (reconcile.Result, error) { + return h.handlePodController(instance) } // handlePodController reconciles the state of a podController -func (h *Handler) handlePodController(instance podController) (reconcile.Result, error) { +func (h *Handler[I]) handlePodController(instance I) (reconcile.Result, error) { log := logf.Log.WithName("wave").WithValues("namespace", instance.GetNamespace(), "name", instance.GetName()) // To cleanup legacy ownerReferences and finalizer @@ -121,19 +99,21 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, } // Update the desired state of the Deployment in a DeepCopy - copy := instance.DeepCopyPodController() - setConfigHash(copy, hash) + oldHash := getConfigHash(instance) + setConfigHash(instance, hash) - if isSchedulingDisabled(copy) { - restoreScheduling(copy) + schedulingChange := false + if isSchedulingDisabled(instance) { + restoreScheduling(instance) + schedulingChange = true } // If the desired state doesn't match the existing state, update it - if !reflect.DeepEqual(instance, copy) { + if hash != oldHash || schedulingChange { log.V(0).Info("Updating instance hash", "hash", hash) - h.recorder.Eventf(copy.GetApiObject(), corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) + h.recorder.Eventf(instance, corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) - err := h.Update(context.TODO(), copy.GetApiObject()) + err := h.Update(context.TODO(), instance) if err != nil { return reconcile.Result{}, fmt.Errorf("error updating instance %s/%s: %v", instance.GetNamespace(), instance.GetName(), err) } @@ -142,7 +122,7 @@ func (h *Handler) handlePodController(instance podController) (reconcile.Result, } // handlePodController will only update the hash. Everything else is left to the reconciler. -func (h *Handler) updatePodController(instance podController, dryRun bool, isCreate bool) error { +func (h *Handler[I]) updatePodController(instance I, dryRun bool, isCreate bool) error { log := logf.Log.WithName("wave").WithValues("namespace", instance.GetNamespace(), "name", instance.GetName(), "dryRun", dryRun, "isCreate", isCreate) log.V(5).Info("Running webhook") @@ -179,7 +159,7 @@ func (h *Handler) updatePodController(instance podController, dryRun bool, isCre if !dryRun && oldHash != hash { log.V(0).Info("Updating instance hash", "hash", hash) - h.recorder.Eventf(instance.GetApiObject(), corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) + h.recorder.Eventf(instance, corev1.EventTypeNormal, "ConfigChanged", "Configuration hash updated to %s", hash) } return nil diff --git a/pkg/core/handler_test.go b/pkg/core/handler_test.go index 89c18a5f..e4ebdec0 100644 --- a/pkg/core/handler_test.go +++ b/pkg/core/handler_test.go @@ -36,7 +36,7 @@ import ( var _ = Describe("Wave controller Suite", func() { var c client.Client - var h *Handler + var h *Handler[*appsv1.Deployment] var m utils.Matcher var deployment *appsv1.Deployment @@ -76,7 +76,7 @@ var _ = Describe("Wave controller Suite", func() { c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(cerr).NotTo(HaveOccurred()) - h = NewHandler(c, mgr.GetEventRecorderFor("wave")) + h = NewHandler[*appsv1.Deployment](c, mgr.GetEventRecorderFor("wave")) m = utils.Matcher{Client: c} stopMgr, mgrStopped = StartTestManager(mgr) @@ -141,7 +141,7 @@ var _ = Describe("Wave controller Suite", func() { // Create a deployment and wait for it to be reconciled m.Create(deployment).Should(Succeed()) - _, err = h.HandleDeployment(deployment) + _, err = h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) m.Get(deployment).Should(Succeed()) @@ -172,12 +172,12 @@ var _ = Describe("Wave controller Suite", func() { obj.SetAnnotations(annotations) return obj }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment m.Get(deployment, timeout).Should(Succeed()) - _, err = h.HandleDeployment(deployment) + _, err = h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -226,7 +226,7 @@ var _ = Describe("Wave controller Suite", func() { dpl.Spec.Template.Spec.Containers = []corev1.Container{containers[0]} return dpl }).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -265,7 +265,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -287,7 +287,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -310,7 +310,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -333,7 +333,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -357,7 +357,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -383,7 +383,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -409,7 +409,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -435,7 +435,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -457,7 +457,7 @@ var _ = Describe("Wave controller Suite", func() { return obj }, timeout).Should(Succeed()) - _, err := h.HandleDeployment(deployment) + _, err := h.Handle(deployment) Expect(err).NotTo(HaveOccurred()) Eventually(deployment, timeout).ShouldNot(utils.WithAnnotations(HaveKey(RequiredAnnotation))) diff --git a/pkg/core/hash.go b/pkg/core/hash.go index c179f73b..3dbec398 100644 --- a/pkg/core/hash.go +++ b/pkg/core/hash.go @@ -107,9 +107,9 @@ func getSecretData(child configObject) map[string][]byte { // setConfigHash updates the configuration hash of the given Deployment to the // given string -func setConfigHash(obj podController, hash string) { +func setConfigHash[I InstanceType](obj I, hash string) { // Get the existing annotations - podTemplate := obj.GetPodTemplate() + podTemplate := GetPodTemplate(obj) annotations := podTemplate.GetAnnotations() if annotations == nil { annotations = make(map[string]string) @@ -118,11 +118,11 @@ func setConfigHash(obj podController, hash string) { // Update the annotations annotations[ConfigHashAnnotation] = hash podTemplate.SetAnnotations(annotations) - obj.SetPodTemplate(podTemplate) + SetPodTemplate(obj, podTemplate) } // getConfigHash return the config hash string -func getConfigHash(obj podController) string { - podTemplate := obj.GetPodTemplate() +func getConfigHash[I InstanceType](obj I) string { + podTemplate := GetPodTemplate(obj) return podTemplate.GetAnnotations()[ConfigHashAnnotation] } diff --git a/pkg/core/hash_test.go b/pkg/core/hash_test.go index 75b35d10..4a9bf5d5 100644 --- a/pkg/core/hash_test.go +++ b/pkg/core/hash_test.go @@ -326,11 +326,11 @@ var _ = Describe("Wave hash Suite", func() { Context("setConfigHash", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment // TODO: remove BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject }) It("sets the hash annotation to the provided value", func() { diff --git a/pkg/core/owner_references.go b/pkg/core/owner_references.go index 40ae9139..0c786b26 100644 --- a/pkg/core/owner_references.go +++ b/pkg/core/owner_references.go @@ -21,13 +21,14 @@ import ( "fmt" "reflect" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // removeOwnerReferences iterates over a list of children and removes the owner // reference from the child before updating it -func (h *Handler) removeOwnerReferences(obj podController, children []Object) error { +func (h *Handler[I]) removeOwnerReferences(obj I, children []Object) error { for _, child := range children { // Filter the existing ownerReferences ownerRefs := []metav1.OwnerReference{} @@ -57,11 +58,11 @@ func kindOf(obj Object) string { return "ConfigMap" case *corev1.Secret: return "Secret" - case *deployment: + case *appsv1.Deployment: return "Deployment" - case *statefulset: + case *appsv1.StatefulSet: return "StatefulSet" - case *daemonset: + case *appsv1.DaemonSet: return "DaemonSet" default: return "Unknown" diff --git a/pkg/core/owner_references_test.go b/pkg/core/owner_references_test.go index 4055ed42..117dde70 100644 --- a/pkg/core/owner_references_test.go +++ b/pkg/core/owner_references_test.go @@ -36,10 +36,10 @@ import ( var _ = Describe("Wave owner references Suite", func() { var c client.Client - var h *Handler + var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -64,7 +64,7 @@ var _ = Describe("Wave owner references Suite", func() { var cerr error c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(cerr).NotTo(HaveOccurred()) - h = NewHandler(c, mgr.GetEventRecorderFor("wave")) + h = NewHandler[*appsv1.Deployment](c, mgr.GetEventRecorderFor("wave")) m = utils.Matcher{Client: c} // Create some configmaps and secrets @@ -83,7 +83,7 @@ var _ = Describe("Wave owner references Suite", func() { m.Create(s3).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) diff --git a/pkg/core/required_annotation.go b/pkg/core/required_annotation.go index 286ef6e6..78fdb7d1 100644 --- a/pkg/core/required_annotation.go +++ b/pkg/core/required_annotation.go @@ -18,7 +18,7 @@ package core // hasRequiredAnnotation returns true if the given PodController has the wave // annotation present -func hasRequiredAnnotation(obj podController) bool { +func hasRequiredAnnotation[I InstanceType](obj I) bool { annotations := obj.GetAnnotations() if value, ok := annotations[RequiredAnnotation]; ok { if value == requiredAnnotationValue { diff --git a/pkg/core/required_annotation_test.go b/pkg/core/required_annotation_test.go index f74d9157..6516bbc3 100644 --- a/pkg/core/required_annotation_test.go +++ b/pkg/core/required_annotation_test.go @@ -25,11 +25,11 @@ import ( var _ = Describe("Wave required annotation Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject // TODO: remove }) Context("hasRequiredAnnotation", func() { diff --git a/pkg/core/scheduler.go b/pkg/core/scheduler.go index 1bfdf163..04e49f52 100644 --- a/pkg/core/scheduler.go +++ b/pkg/core/scheduler.go @@ -1,7 +1,7 @@ package core // disableScheduling sets an invalid scheduler and adds an annotation with the original scheduler -func disableScheduling(obj podController) { +func disableScheduling[I InstanceType](obj I) { if isSchedulingDisabled(obj) { return } @@ -13,18 +13,18 @@ func disableScheduling(obj podController) { } // Store previous scheduler in annotation - schedulerName := obj.GetPodTemplate().Spec.SchedulerName + schedulerName := GetPodTemplate(obj).Spec.SchedulerName annotations[SchedulingDisabledAnnotation] = schedulerName obj.SetAnnotations(annotations) // Set invalid scheduler - podTemplate := obj.GetPodTemplate() + podTemplate := GetPodTemplate(obj) podTemplate.Spec.SchedulerName = SchedulingDisabledSchedulerName - obj.SetPodTemplate(podTemplate) + SetPodTemplate(obj, podTemplate) } // isSchedulingDisabled returns true if scheduling has been disabled by wave -func isSchedulingDisabled(obj podController) bool { +func isSchedulingDisabled[I InstanceType](obj I) bool { // Get the existing annotations annotations := obj.GetAnnotations() if annotations == nil { @@ -35,7 +35,7 @@ func isSchedulingDisabled(obj podController) bool { } // enableScheduling restore scheduling if it has been disabled by wave -func restoreScheduling(obj podController) { +func restoreScheduling[I InstanceType](obj I) { // Get the existing annotations annotations := obj.GetAnnotations() if annotations == nil { @@ -50,7 +50,7 @@ func restoreScheduling(obj podController) { obj.SetAnnotations(annotations) // Restore scheduler - podTemplate := obj.GetPodTemplate() + podTemplate := GetPodTemplate(obj) podTemplate.Spec.SchedulerName = schedulerName - obj.SetPodTemplate(podTemplate) + SetPodTemplate(obj, podTemplate) } diff --git a/pkg/core/scheduler_test.go b/pkg/core/scheduler_test.go index 2a4a2db5..aba8c94d 100644 --- a/pkg/core/scheduler_test.go +++ b/pkg/core/scheduler_test.go @@ -25,11 +25,11 @@ import ( var _ = Describe("Wave scheduler Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment podController + var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = &deployment{deploymentObject} + podControllerDeployment = deploymentObject }) Context("When scheduler is disabled", func() { @@ -43,7 +43,7 @@ var _ = Describe("Wave scheduler Suite", func() { }) It("Disables scheduling", func() { - podTemplate := podControllerDeployment.GetPodTemplate() + podTemplate := GetPodTemplate(podControllerDeployment) Expect(podTemplate.Spec.SchedulerName).To(Equal(SchedulingDisabledSchedulerName)) }) @@ -62,7 +62,7 @@ var _ = Describe("Wave scheduler Suite", func() { }) It("Restores the scheduler", func() { - podTemplate := podControllerDeployment.GetPodTemplate() + podTemplate := GetPodTemplate(podControllerDeployment) Expect(podTemplate.Spec.SchedulerName).To(Equal("default-scheduler")) }) diff --git a/pkg/core/types.go b/pkg/core/types.go index 93b1373a..bea5badd 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -1,9 +1,13 @@ package core import ( + "fmt" + "reflect" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -57,88 +61,40 @@ type configObject struct { keys map[string]struct{} } -// podController interface adjusted to include client.Object explicitly -type podController interface { +type InstanceType interface { + *appsv1.Deployment | *appsv1.StatefulSet | *appsv1.DaemonSet client.Object + runtime.Object metav1.Object - GetPodTemplate() *corev1.PodTemplateSpec - SetPodTemplate(*corev1.PodTemplateSpec) - DeepCopyPodController() podController - GetApiObject() client.Object -} - -// Deployment struct implementing the podController interface -type deployment struct { - *appsv1.Deployment -} - -func (d *deployment) GetPodTemplate() *corev1.PodTemplateSpec { - return &d.Spec.Template -} - -func (d *deployment) SetPodTemplate(template *corev1.PodTemplateSpec) { - d.Spec.Template = *template } -func (d *deployment) DeepCopyPodController() podController { - return &deployment{d.Deployment.DeepCopy()} +type DeplyomentInterface interface { + metav1.TypeMeta + metav1.ObjectMeta } -func (d *deployment) GetApiObject() client.Object { - return &appsv1.Deployment{ - Status: d.Status, - Spec: d.Spec, - ObjectMeta: d.ObjectMeta, +func GetPodTemplate[I InstanceType](instance I) *corev1.PodTemplateSpec { + if deployment, ok := any(instance).(*appsv1.Deployment); ok { + return &deployment.Spec.Template } -} - -// StatefulSet struct implementing the podController interface -type statefulset struct { - *appsv1.StatefulSet -} - -func (s *statefulset) GetPodTemplate() *corev1.PodTemplateSpec { - return &s.Spec.Template -} - -func (s *statefulset) SetPodTemplate(template *corev1.PodTemplateSpec) { - s.Spec.Template = *template -} - -func (s *statefulset) DeepCopyPodController() podController { - return &statefulset{s.StatefulSet.DeepCopy()} -} - -func (d *statefulset) GetApiObject() client.Object { - return &appsv1.StatefulSet{ - Status: d.Status, - Spec: d.Spec, - ObjectMeta: d.ObjectMeta, + if statefulset, ok := any(instance).(*appsv1.StatefulSet); ok { + return &statefulset.Spec.Template } -} - -// DaemonSet struct implementing the podController interface -type daemonset struct { - *appsv1.DaemonSet -} - -func (d *daemonset) GetPodTemplate() *corev1.PodTemplateSpec { - return &d.Spec.Template -} - -func (d *daemonset) SetPodTemplate(template *corev1.PodTemplateSpec) { - d.Spec.Template = *template -} - -func (d *daemonset) DeepCopyPodController() podController { - return &daemonset{d.DaemonSet.DeepCopy()} -} - -func (d *daemonset) GetApiObject() client.Object { - return &appsv1.DaemonSet{ - Status: d.Status, - Spec: d.Spec, - ObjectMeta: d.ObjectMeta, + if daemonset, ok := any(instance).(*appsv1.DaemonSet); ok { + return &daemonset.Spec.Template + } + panic(fmt.Sprintf("Invalid type %s", reflect.TypeOf(instance))) +} + +func SetPodTemplate[I InstanceType](instance I, template *corev1.PodTemplateSpec) { + if deployment, ok := any(instance).(*appsv1.Deployment); ok { + deployment.Spec.Template = *template + } else if statefulset, ok := any(instance).(*appsv1.StatefulSet); ok { + statefulset.Spec.Template = *template + } else if daemonset, ok := any(instance).(*appsv1.DaemonSet); ok { + daemonset.Spec.Template = *template + } else { + panic(fmt.Sprintf("Invalid type %s", reflect.TypeOf(instance))) } } diff --git a/pkg/core/watcher.go b/pkg/core/watcher.go index c435ee2a..ceb36e82 100644 --- a/pkg/core/watcher.go +++ b/pkg/core/watcher.go @@ -65,15 +65,15 @@ func (e *enqueueRequestForWatcher) queueOwnerReconcileRequest(object metav1.Obje e.watchersMutex.Unlock() } -func (h *Handler) GetWatchedConfigmaps() WatcherList { +func (h *Handler[I]) GetWatchedConfigmaps() WatcherList { return h.watchedConfigmaps } -func (h *Handler) GetWatchedSecrets() WatcherList { +func (h *Handler[I]) GetWatchedSecrets() WatcherList { return h.watchedSecrets } -func (h *Handler) watchChildrenForInstance(instance podController, configMaps configMetadataMap, secrets configMetadataMap) { +func (h *Handler[I]) watchChildrenForInstance(instance I, configMaps configMetadataMap, secrets configMetadataMap) { instanceName := GetNamespacedNameFromObject(instance) h.watchedConfigmaps.watchersMutex.Lock() for childName := range configMaps { @@ -93,11 +93,11 @@ func (h *Handler) watchChildrenForInstance(instance podController, configMaps co h.watchedSecrets.watchersMutex.Unlock() } -func (h *Handler) removeWatchesForInstance(instance podController) { +func (h *Handler[I]) removeWatchesForInstance(instance I) { h.RemoveWatches(GetNamespacedNameFromObject(instance)) } -func (h *Handler) RemoveWatches(instanceName types.NamespacedName) { +func (h *Handler[I]) RemoveWatches(instanceName types.NamespacedName) { h.watchedConfigmaps.watchersMutex.Lock() for child, watchers := range h.watchedConfigmaps.watchers { delete(watchers, instanceName) From 54fdb98d72e9afa308fcff63648d2f063ccd2f8b Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Thu, 9 May 2024 23:20:28 +0200 Subject: [PATCH 2/8] refactor controllers with generics --- .../daemonset/daemonset_controller.go | 46 +--------------- .../deployment/deployment_controller.go | 47 +--------------- .../statefulset/statefulset_controller.go | 47 +--------------- pkg/core/controller.go | 55 +++++++++++++++++++ pkg/core/handler.go | 14 ++++- pkg/core/handler_test.go | 26 ++++----- 6 files changed, 87 insertions(+), 148 deletions(-) create mode 100644 pkg/core/controller.go diff --git a/pkg/controller/daemonset/daemonset_controller.go b/pkg/controller/daemonset/daemonset_controller.go index c5197049..d462b9e5 100644 --- a/pkg/controller/daemonset/daemonset_controller.go +++ b/pkg/controller/daemonset/daemonset_controller.go @@ -21,15 +21,9 @@ import ( "github.com/wave-k8s/wave/pkg/core" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;update;patch @@ -54,30 +48,7 @@ func newReconciler(mgr manager.Manager) *ReconcileDaemonSet { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.DaemonSet]) error { - // Create a new controller - c, err := controller.New("daemonset-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - err = c.Watch(source.Kind(mgr.GetCache(), &appsv1.DaemonSet{}), &handler.EnqueueRequestForObject{}, predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) - if err != nil { - return err - } - - // Watch ConfigMaps owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), core.EnqueueRequestForWatcher(h.GetWatchedConfigmaps())) - if err != nil { - return err - } - - // Watch Secrets owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}), core.EnqueueRequestForWatcher(h.GetWatchedSecrets())) - if err != nil { - return err - } - - return nil + return core.AddController("daemonset-controller", &appsv1.DaemonSet{}, mgr, r, h) } var _ reconcile.Reconciler = &ReconcileDaemonSet{} @@ -91,18 +62,5 @@ type ReconcileDaemonSet struct { // Reconcile reads that state of the cluster for a DaemonSet object and // updates its PodSpec based on mounted configuration func (r *ReconcileDaemonSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // Fetch the DaemonSet instance - instance := &appsv1.DaemonSet{} - err := r.handler.Get(ctx, request.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - r.handler.RemoveWatches(request.NamespacedName) - // Object not found, return. Created objects are automatically garbage collected. - return reconcile.Result{}, nil - } - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - return r.handler.Handle(instance) + return r.handler.Handle(ctx, request.NamespacedName, &appsv1.DaemonSet{}) } diff --git a/pkg/controller/deployment/deployment_controller.go b/pkg/controller/deployment/deployment_controller.go index 32b9f7f0..3366d5ac 100644 --- a/pkg/controller/deployment/deployment_controller.go +++ b/pkg/controller/deployment/deployment_controller.go @@ -21,15 +21,9 @@ import ( "github.com/wave-k8s/wave/pkg/core" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;update;patch @@ -54,31 +48,7 @@ func newReconciler(mgr manager.Manager) *ReconcileDeployment { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.Deployment]) error { - // Create a new controller - c, err := controller.New("deployment-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to Deployment - err = c.Watch(source.Kind(mgr.GetCache(), &appsv1.Deployment{}), &handler.EnqueueRequestForObject{}, predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) - if err != nil { - return err - } - - // Watch ConfigMaps owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), core.EnqueueRequestForWatcher(h.GetWatchedConfigmaps())) - if err != nil { - return err - } - - // Watch Secrets owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}), core.EnqueueRequestForWatcher(h.GetWatchedSecrets())) - if err != nil { - return err - } - - return nil + return core.AddController("deployment-controller", &appsv1.Deployment{}, mgr, r, h) } var _ reconcile.Reconciler = &ReconcileDeployment{} @@ -92,18 +62,5 @@ type ReconcileDeployment struct { // Reconcile reads that state of the cluster for a Deployment object and // updates its PodSpec based on mounted configuration func (r *ReconcileDeployment) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // Fetch the Deployment instance - instance := &appsv1.Deployment{} - err := r.handler.Get(ctx, request.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - r.handler.RemoveWatches(request.NamespacedName) - // Object not found, return. Created objects are automatically garbage collected. - return reconcile.Result{}, nil - } - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - return r.handler.Handle(instance) + return r.handler.Handle(ctx, request.NamespacedName, &appsv1.Deployment{}) } diff --git a/pkg/controller/statefulset/statefulset_controller.go b/pkg/controller/statefulset/statefulset_controller.go index 6335e1b7..09fecba9 100644 --- a/pkg/controller/statefulset/statefulset_controller.go +++ b/pkg/controller/statefulset/statefulset_controller.go @@ -21,15 +21,9 @@ import ( "github.com/wave-k8s/wave/pkg/core" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;update;patch @@ -54,31 +48,7 @@ func newReconciler(mgr manager.Manager) *ReconcileStatefulSet { // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler, h *core.Handler[*appsv1.StatefulSet]) error { - // Create a new controller - c, err := controller.New("statefulset-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to StatefulSet - err = c.Watch(source.Kind(mgr.GetCache(), &appsv1.StatefulSet{}), &handler.EnqueueRequestForObject{}, predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) - if err != nil { - return err - } - - // Watch ConfigMaps owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), core.EnqueueRequestForWatcher(h.GetWatchedConfigmaps())) - if err != nil { - return err - } - - // Watch Secrets owned by a DaemonSet - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}), core.EnqueueRequestForWatcher(h.GetWatchedSecrets())) - if err != nil { - return err - } - - return nil + return core.AddController("statefulset-controller", &appsv1.StatefulSet{}, mgr, r, h) } var _ reconcile.Reconciler = &ReconcileStatefulSet{} @@ -92,18 +62,5 @@ type ReconcileStatefulSet struct { // Reconcile reads that state of the cluster for a StatefulSet object and // updates its PodSpec based on mounted configuration func (r *ReconcileStatefulSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // Fetch the StatefulSet instance - instance := &appsv1.StatefulSet{} - err := r.handler.Get(ctx, request.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - r.handler.RemoveWatches(request.NamespacedName) - // Object not found, return. Created objects are automatically garbage collected. - return reconcile.Result{}, nil - } - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - return r.handler.Handle(instance) + return r.handler.Handle(ctx, request.NamespacedName, &appsv1.StatefulSet{}) } diff --git a/pkg/core/controller.go b/pkg/core/controller.go new file mode 100644 index 00000000..8548cb42 --- /dev/null +++ b/pkg/core/controller.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 Pusher Ltd. and Wave Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func AddController[I InstanceType](name string, typeInstance I, mgr manager.Manager, r reconcile.Reconciler, h *Handler[I]) error { + // Create a new controller + c, err := controller.New(name, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + err = c.Watch(source.Kind(mgr.GetCache(), typeInstance), &handler.EnqueueRequestForObject{}, predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) + if err != nil { + return err + } + + // Watch ConfigMaps owned by a DaemonSet + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), EnqueueRequestForWatcher(h.GetWatchedConfigmaps())) + if err != nil { + return err + } + + // Watch Secrets owned by a DaemonSet + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}), EnqueueRequestForWatcher(h.GetWatchedSecrets())) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/core/handler.go b/pkg/core/handler.go index a91344f2..1445da48 100644 --- a/pkg/core/handler.go +++ b/pkg/core/handler.go @@ -22,6 +22,7 @@ import ( "sync" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,7 +57,18 @@ func (h *Handler[I]) HandleWebhook(instance I, dryRun *bool, isCreate bool) erro } // Handle is called by the controller to reconcile its object -func (h *Handler[I]) Handle(instance I) (reconcile.Result, error) { +func (h *Handler[I]) Handle(ctx context.Context, namespacesName types.NamespacedName, instance I) (reconcile.Result, error) { + err := h.Get(ctx, namespacesName, instance) + if err != nil { + if errors.IsNotFound(err) { + h.RemoveWatches(namespacesName) + // Object not found, return. Created objects are automatically garbage collected. + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + return h.handlePodController(instance) } diff --git a/pkg/core/handler_test.go b/pkg/core/handler_test.go index e4ebdec0..55f05fb7 100644 --- a/pkg/core/handler_test.go +++ b/pkg/core/handler_test.go @@ -141,7 +141,7 @@ var _ = Describe("Wave controller Suite", func() { // Create a deployment and wait for it to be reconciled m.Create(deployment).Should(Succeed()) - _, err = h.Handle(deployment) + _, err = h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) m.Get(deployment).Should(Succeed()) @@ -172,12 +172,12 @@ var _ = Describe("Wave controller Suite", func() { obj.SetAnnotations(annotations) return obj }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment m.Get(deployment, timeout).Should(Succeed()) - _, err = h.Handle(deployment) + _, err = h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -226,7 +226,7 @@ var _ = Describe("Wave controller Suite", func() { dpl.Spec.Template.Spec.Containers = []corev1.Container{containers[0]} return dpl }).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -265,7 +265,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -287,7 +287,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -310,7 +310,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -333,7 +333,7 @@ var _ = Describe("Wave controller Suite", func() { return cm }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -357,7 +357,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -383,7 +383,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -409,7 +409,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -435,7 +435,7 @@ var _ = Describe("Wave controller Suite", func() { return s }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) // Get the updated Deployment @@ -457,7 +457,7 @@ var _ = Describe("Wave controller Suite", func() { return obj }, timeout).Should(Succeed()) - _, err := h.Handle(deployment) + _, err := h.Handle(context.TODO(), instanceName, &appsv1.Deployment{}) Expect(err).NotTo(HaveOccurred()) Eventually(deployment, timeout).ShouldNot(utils.WithAnnotations(HaveKey(RequiredAnnotation))) From 6374bf0c2d3cdf4a9102d24eadd9aff14f68ddcf Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Fri, 10 May 2024 01:04:39 +0200 Subject: [PATCH 3/8] deduplicate tests and test the same thing for all objects --- .../daemonset/daemonset_controller_test.go | 467 +-------- .../deployment/deployment_controller_test.go | 55 + .../statefulset_controller_test.go | 471 +-------- pkg/core/controller_suite.go | 481 +++++++++ test/utils/test_objects.go | 956 ++++++------------ 5 files changed, 859 insertions(+), 1571 deletions(-) create mode 100644 pkg/core/controller_suite.go diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go index 54116668..621572a5 100644 --- a/pkg/controller/daemonset/daemonset_controller_test.go +++ b/pkg/controller/daemonset/daemonset_controller_test.go @@ -18,462 +18,35 @@ package daemonset import ( "context" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" "github.com/wave-k8s/wave/pkg/core" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" - webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("DaemonSet controller Suite", func() { - var c client.Client - var m utils.Matcher - - var daemonset *appsv1.DaemonSet - var requestsStart <-chan reconcile.Request - var requests <-chan reconcile.Request - - const timeout = time.Second * 5 - const consistentlyTimeout = time.Second - - var cm1 *corev1.ConfigMap - var cm2 *corev1.ConfigMap - var cm3 *corev1.ConfigMap - var s1 *corev1.Secret - var s2 *corev1.Secret - var s3 *corev1.Secret - - const modified = "modified" - - var waitForDaemonSetReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the DaemonSet - Eventually(requestsStart, timeout).Should(Receive(Equal(request))) - Eventually(requests, timeout).Should(Receive(Equal(request))) - } - - var consistentlyDaemonSetNotReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the DaemonSet - Consistently(requestsStart, consistentlyTimeout).ShouldNot(Receive(Equal(request))) - } - - var clearReconciled = func() { - for len(requestsStart) > 0 { - <-requestsStart - <-requests - } - } - - BeforeEach(func() { - // Reset the Prometheus Registry before each test to avoid errors - metrics.Registry = prometheus.NewRegistry() - - mgr, err := manager.New(cfg, manager.Options{ - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: t.WebhookInstallOptions.LocalServingHost, - Port: t.WebhookInstallOptions.LocalServingPort, - CertDir: t.WebhookInstallOptions.LocalServingCertDir, - }), - Cache: cache.Options{ - DefaultNamespaces: core.BuildCacheDefaultNamespaces(""), - }, - }) - Expect(err).NotTo(HaveOccurred()) - var cerr error - c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(cerr).NotTo(HaveOccurred()) - m = utils.Matcher{Client: c} - - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests = SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err = AddDaemonSetWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - - // Create some configmaps and secrets - cm1 = utils.ExampleConfigMap1.DeepCopy() - cm2 = utils.ExampleConfigMap2.DeepCopy() - cm3 = utils.ExampleConfigMap3.DeepCopy() - s1 = utils.ExampleSecret1.DeepCopy() - s2 = utils.ExampleSecret2.DeepCopy() - s3 = utils.ExampleSecret3.DeepCopy() - - m.Create(cm1).Should(Succeed()) - m.Create(cm2).Should(Succeed()) - m.Create(cm3).Should(Succeed()) - m.Create(s1).Should(Succeed()) - m.Create(s2).Should(Succeed()) - m.Create(s3).Should(Succeed()) - m.Get(cm1, timeout).Should(Succeed()) - m.Get(cm2, timeout).Should(Succeed()) - m.Get(cm3, timeout).Should(Succeed()) - m.Get(s1, timeout).Should(Succeed()) - m.Get(s2, timeout).Should(Succeed()) - m.Get(s3, timeout).Should(Succeed()) - - daemonset = utils.ExampleDaemonSet.DeepCopy() - }) - - AfterEach(func() { - testCancel() - - utils.DeleteAll(cfg, timeout, - &appsv1.DaemonSetList{}, - &corev1.ConfigMapList{}, - &corev1.SecretList{}, - &corev1.EventList{}, - ) - }) - - Context("When a DaemonSet with all children existing is reconciled", func() { - BeforeEach(func() { - // Create a daemonset and wait for it to be reconciled - clearReconciled() - m.Create(daemonset).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - }) - - Context("And it has the required annotation", func() { - BeforeEach(func() { - addAnnotation := func(obj client.Object) client.Object { - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - obj.SetAnnotations(annotations) - return obj - } - clearReconciled() - m.Update(daemonset, addAnnotation).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Has scheduling enabled", func() { - m.Get(daemonset, timeout).Should(Succeed()) - Expect(daemonset.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(daemonset.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - - It("Adds a config hash to the Pod Template", func() { - Eventually(daemonset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - }) - - It("Sends an event when updating the hash", func() { - eventMessage := func(event *corev1.Event) string { - return event.Message - } - hashMessage := "Configuration hash updated to bd786f47ef9b79841ddba1059752f95c4fe21906df5e2964786b4658e02758d5" - Eventually(func() *corev1.EventList { - events := &corev1.EventList{} - m.Client.List(context.TODO(), events) - return events - }, timeout).Should(utils.WithItems(ContainElement(WithTransform(eventMessage, Equal(hashMessage))))) - }) - - Context("And a child is removed", func() { - var originalHash string - BeforeEach(func() { - Eventually(daemonset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - - // Remove "container2" which references Secret example2 and ConfigMap - // example2 - removeContainer2 := func(obj client.Object) client.Object { - ss, _ := obj.(*appsv1.DaemonSet) - containers := ss.Spec.Template.Spec.Containers - Expect(containers[0].Name).To(Equal("container1")) - ss.Spec.Template.Spec.Containers = []corev1.Container{containers[0]} - return ss - } - clearReconciled() - m.Update(daemonset, removeContainer2).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - - It("Changes to the removed children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm2, modifyCM).Should(Succeed()) - consistentlyDaemonSetNotReconciled(daemonset) - }) - }) - - Context("And a child is updated", func() { - var originalHash string - - BeforeEach(func() { - m.Get(daemonset, timeout).Should(Succeed()) - Eventually(daemonset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }) - - Context("A ConfigMap volume is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm1, modifyCM).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A ConfigMap EnvSource is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm2, modifyCM).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret volume is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s1, modifyS).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret EnvSource is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s2, modifyS).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - waitForDaemonSetReconciled(daemonset) - - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return daemonset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - }) - - Context("And the annotation is removed", func() { - BeforeEach(func() { - removeAnnotations := func(obj client.Object) client.Object { - obj.SetAnnotations(make(map[string]string)) - return obj - } - clearReconciled() - m.Update(daemonset, removeAnnotations).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - m.Get(daemonset).Should(Succeed()) - Eventually(daemonset, timeout).ShouldNot(utils.WithAnnotations(HaveKey(core.RequiredAnnotation))) - }) - - It("Removes the config hash annotation", func() { - m.Consistently(daemonset, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDaemonSetNotReconciled(daemonset) - }) - }) - - Context("And is deleted", func() { - BeforeEach(func() { - // Make sure the cache has synced before we run the test - Eventually(daemonset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - clearReconciled() - m.Delete(daemonset).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - }) - - It("Not longer exists", func() { - m.Get(daemonset).Should(MatchError(MatchRegexp(`not found`))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDaemonSetNotReconciled(daemonset) - }) - }) - }) - - Context("And it does not have the required annotation", func() { - BeforeEach(func() { - // Get the updated DaemonSet - m.Get(daemonset, timeout).Should(Succeed()) - }) - - It("Doesn't add a config hash to the Pod Template", func() { - m.Consistently(daemonset, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no do not trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDaemonSetNotReconciled(daemonset) - }) - }) - }) - - Context("When a DaemonSet with missing children is reconciled", func() { - BeforeEach(func() { - m.Delete(cm1).Should(Succeed()) - - annotations := daemonset.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - daemonset.SetAnnotations(annotations) - - // Create a daemonset and wait for it to be reconciled - clearReconciled() - m.Create(daemonset).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - }) - - It("Has scheduling disabled", func() { - m.Get(daemonset, timeout).Should(Succeed()) - Expect(daemonset.Spec.Template.Spec.SchedulerName).To(Equal(core.SchedulingDisabledSchedulerName)) - Expect(daemonset.ObjectMeta.Annotations[core.SchedulingDisabledAnnotation]).To(Equal("default-scheduler")) - }) - - Context("And the missing child is created", func() { - BeforeEach(func() { - clearReconciled() - cm1 = utils.ExampleConfigMap1.DeepCopy() - m.Create(cm1).Should(Succeed()) - waitForDaemonSetReconciled(daemonset) - }) - - It("Has scheduling renabled", func() { - m.Get(daemonset, timeout).Should(Succeed()) - Expect(daemonset.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(daemonset.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - }) - }) + core.ControllerTestSuite( + &t, &cfg, + func() *appsv1.DaemonSet { + return utils.ExampleDaemonSet.DeepCopy() + }, + func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests := SetupTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err := AddDaemonSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go Run(testCtx, mgr) + return testCancel, requestsStart, requests + }, + ) }) diff --git a/pkg/controller/deployment/deployment_controller_test.go b/pkg/controller/deployment/deployment_controller_test.go index 32874ab4..5308453e 100644 --- a/pkg/controller/deployment/deployment_controller_test.go +++ b/pkg/controller/deployment/deployment_controller_test.go @@ -16,6 +16,60 @@ limitations under the License. package deployment +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/wave-k8s/wave/pkg/core" + "github.com/wave-k8s/wave/test/utils" + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("Deployment controller Suite", func() { + core.ControllerTestSuite( + &t, &cfg, + func() *appsv1.Deployment { + return utils.ExampleDeployment.DeepCopy() + }, + func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests := SetupTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err := AddDeploymentWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go Run(testCtx, mgr) + return testCancel, requestsStart, requests + }, + ) +}) + +/* +Copyright 2018 Pusher Ltd. and Wave Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +package deployment + import ( "context" "time" @@ -504,3 +558,4 @@ var _ = Describe("Deployment controller Suite", func() { }) }) +*/ diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index 2785b42a..bfa6ef30 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -18,466 +18,35 @@ package statefulset import ( "context" - "time" - - "sigs.k8s.io/controller-runtime/pkg/cache" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" "github.com/wave-k8s/wave/pkg/core" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" - webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("StatefulSet controller Suite", func() { - var c client.Client - var m utils.Matcher - - var statefulset *appsv1.StatefulSet - var requestsStart <-chan reconcile.Request - var requests <-chan reconcile.Request - - const timeout = time.Second * 5 - const consistentlyTimeout = time.Second - - var cm1 *corev1.ConfigMap - var cm2 *corev1.ConfigMap - var cm3 *corev1.ConfigMap - var s1 *corev1.Secret - var s2 *corev1.Secret - var s3 *corev1.Secret - - const modified = "modified" - - var waitForStatefulSetReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the StatefulSet - Eventually(requestsStart, timeout).Should(Receive(Equal(request))) - Eventually(requests, timeout).Should(Receive(Equal(request))) - } - - var consistentlyStatefulSetNotReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the DaemonSet - Consistently(requestsStart, consistentlyTimeout).ShouldNot(Receive(Equal(request))) - } - - var clearReconciled = func() { - for len(requestsStart) > 0 { - <-requestsStart - <-requests - } - } - - BeforeEach(func() { - // Reset the Prometheus Registry before each test to avoid errors - metrics.Registry = prometheus.NewRegistry() - - mgr, err := manager.New(cfg, manager.Options{ - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: t.WebhookInstallOptions.LocalServingHost, - Port: t.WebhookInstallOptions.LocalServingPort, - CertDir: t.WebhookInstallOptions.LocalServingCertDir, - }), - Cache: cache.Options{ - DefaultNamespaces: core.BuildCacheDefaultNamespaces(""), - }, - }) - Expect(err).NotTo(HaveOccurred()) - - var cerr error - c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(cerr).NotTo(HaveOccurred()) - - m = utils.Matcher{Client: c} - - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests = SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err = AddStatefulSetWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - - // Create some configmaps and secrets - cm1 = utils.ExampleConfigMap1.DeepCopy() - cm2 = utils.ExampleConfigMap2.DeepCopy() - cm3 = utils.ExampleConfigMap3.DeepCopy() - s1 = utils.ExampleSecret1.DeepCopy() - s2 = utils.ExampleSecret2.DeepCopy() - s3 = utils.ExampleSecret3.DeepCopy() - - m.Create(cm1).Should(Succeed()) - m.Create(cm2).Should(Succeed()) - m.Create(cm3).Should(Succeed()) - m.Create(s1).Should(Succeed()) - m.Create(s2).Should(Succeed()) - m.Create(s3).Should(Succeed()) - m.Get(cm1, timeout).Should(Succeed()) - m.Get(cm2, timeout).Should(Succeed()) - m.Get(cm3, timeout).Should(Succeed()) - m.Get(s1, timeout).Should(Succeed()) - m.Get(s2, timeout).Should(Succeed()) - m.Get(s3, timeout).Should(Succeed()) - - statefulset = utils.ExampleStatefulSet.DeepCopy() - }) - - AfterEach(func() { - testCancel() - - utils.DeleteAll(cfg, timeout, - &appsv1.StatefulSetList{}, - &corev1.ConfigMapList{}, - &corev1.SecretList{}, - &corev1.EventList{}, - ) - }) - - Context("When a StatefulSet with all children existing is reconciled", func() { - BeforeEach(func() { - // Create a statefulset and wait for it to be reconciled - clearReconciled() - m.Create(statefulset).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - }) - - Context("And it has the required annotation", func() { - BeforeEach(func() { - addAnnotation := func(obj client.Object) client.Object { - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - obj.SetAnnotations(annotations) - return obj - } - clearReconciled() - m.Update(statefulset, addAnnotation).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Has scheduling enabled", func() { - m.Get(statefulset, timeout).Should(Succeed()) - Expect(statefulset.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(statefulset.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - - It("Adds a config hash to the Pod Template", func() { - Eventually(statefulset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - }) - - It("Sends an event when updating the hash", func() { - Eventually(statefulset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - - eventMessage := func(event *corev1.Event) string { - return event.Message - } - hashMessage := "Configuration hash updated to bd786f47ef9b79841ddba1059752f95c4fe21906df5e2964786b4658e02758d5" - Eventually(func() *corev1.EventList { - events := &corev1.EventList{} - m.Client.List(context.TODO(), events) - return events - }, timeout).Should(utils.WithItems(ContainElement(WithTransform(eventMessage, Equal(hashMessage))))) - }) - - Context("And a child is removed", func() { - var originalHash string - BeforeEach(func() { - Eventually(statefulset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - - // Remove "container2" which references Secret example2 and ConfigMap - // example2 - removeContainer2 := func(obj client.Object) client.Object { - ss, _ := obj.(*appsv1.StatefulSet) - containers := ss.Spec.Template.Spec.Containers - Expect(containers[0].Name).To(Equal("container1")) - ss.Spec.Template.Spec.Containers = []corev1.Container{containers[0]} - return ss - } - clearReconciled() - m.Update(statefulset, removeContainer2).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - - It("Changes to the removed children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm2, modifyCM).Should(Succeed()) - consistentlyStatefulSetNotReconciled(statefulset) - }) - }) - - Context("And a child is updated", func() { - var originalHash string - - BeforeEach(func() { - m.Get(statefulset, timeout).Should(Succeed()) - Eventually(statefulset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }) - - Context("A ConfigMap volume is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm1, modifyCM).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A ConfigMap EnvSource is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm2, modifyCM).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret volume is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s1, modifyS).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret EnvSource is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s2, modifyS).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - waitForStatefulSetReconciled(statefulset) - - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return statefulset.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - }) - - Context("And the annotation is removed", func() { - BeforeEach(func() { - removeAnnotations := func(obj client.Object) client.Object { - obj.SetAnnotations(make(map[string]string)) - return obj - } - clearReconciled() - m.Update(statefulset, removeAnnotations).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - m.Get(statefulset, timeout).Should(Succeed()) - Eventually(statefulset, timeout).ShouldNot(utils.WithAnnotations(HaveKey(core.RequiredAnnotation))) - }) - - It("Removes the config hash annotation", func() { - m.Consistently(statefulset, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyStatefulSetNotReconciled(statefulset) - }) - }) - - Context("And is deleted", func() { - BeforeEach(func() { - // Make sure the cache has synced before we run the test - Eventually(statefulset, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - clearReconciled() - m.Delete(statefulset).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - }) - It("Not longer exists", func() { - m.Get(statefulset).Should(MatchError(MatchRegexp(`not found`))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyStatefulSetNotReconciled(statefulset) - }) - }) - }) - - Context("And it does not have the required annotation", func() { - BeforeEach(func() { - // Get the updated StatefulSet - m.Get(statefulset, timeout).Should(Succeed()) - }) - - It("Doesn't add a config hash to the Pod Template", func() { - m.Consistently(statefulset, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no do not trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyStatefulSetNotReconciled(statefulset) - }) - }) - }) - - Context("When a Deployment with missing children is reconciled", func() { - BeforeEach(func() { - m.Delete(cm1).Should(Succeed()) - - annotations := statefulset.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - statefulset.SetAnnotations(annotations) - - // Create a statefulset and wait for it to be reconciled - clearReconciled() - m.Create(statefulset).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - }) - - It("Has scheduling disabled", func() { - m.Get(statefulset, timeout).Should(Succeed()) - Expect(statefulset.Spec.Template.Spec.SchedulerName).To(Equal(core.SchedulingDisabledSchedulerName)) - Expect(statefulset.ObjectMeta.Annotations[core.SchedulingDisabledAnnotation]).To(Equal("default-scheduler")) - }) - - Context("And the missing child is created", func() { - BeforeEach(func() { - clearReconciled() - cm1 = utils.ExampleConfigMap1.DeepCopy() - m.Create(cm1).Should(Succeed()) - waitForStatefulSetReconciled(statefulset) - }) - - It("Has scheduling renabled", func() { - m.Get(statefulset, timeout).Should(Succeed()) - Expect(statefulset.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(statefulset.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - }) - }) + core.ControllerTestSuite( + &t, &cfg, + func() *appsv1.StatefulSet { + return utils.ExampleStatefulSet.DeepCopy() + }, + func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests := SetupTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err := AddStatefulSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go Run(testCtx, mgr) + return testCancel, requestsStart, requests + }, + ) }) diff --git a/pkg/core/controller_suite.go b/pkg/core/controller_suite.go new file mode 100644 index 00000000..0cdc2697 --- /dev/null +++ b/pkg/core/controller_suite.go @@ -0,0 +1,481 @@ +package core + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.com/wave-k8s/wave/test/utils" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + webhook "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +func ControllerTestSuite[I InstanceType]( + t **envtest.Environment, cfg **rest.Config, + makeObject func() I, + startController func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request)) { + var c client.Client + var m utils.Matcher + + var testCancel context.CancelFunc + + var instance I + var requestsStart <-chan reconcile.Request + var requests <-chan reconcile.Request + + const timeout = time.Second * 5 + const consistentlyTimeout = time.Second + + var cm1 *corev1.ConfigMap + var cm2 *corev1.ConfigMap + var cm3 *corev1.ConfigMap + var cm4 *corev1.ConfigMap + var cm5 *corev1.ConfigMap + var cm6 *corev1.ConfigMap + var s1 *corev1.Secret + var s2 *corev1.Secret + var s3 *corev1.Secret + var s4 *corev1.Secret + var s5 *corev1.Secret + var s6 *corev1.Secret + + const modified = "modified" + + var waitForInstanceReconciled = func(obj Object) { + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }, + } + // wait for reconcile for creating the DaemonSet + Eventually(requestsStart, timeout).Should(Receive(Equal(request))) + Eventually(requests, timeout).Should(Receive(Equal(request))) + } + + var consistentlyInstanceNotReconciled = func(obj Object) { + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }, + } + // wait for reconcile for creating the DaemonSet + Consistently(requestsStart, consistentlyTimeout).ShouldNot(Receive(Equal(request))) + } + + var clearReconciled = func() { + for len(requestsStart) > 0 { + <-requestsStart + <-requests + } + } + + BeforeEach(func() { + // Reset the Prometheus Registry before each test to avoid errors + metrics.Registry = prometheus.NewRegistry() + + mgr, err := manager.New(*cfg, manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: (*t).WebhookInstallOptions.LocalServingHost, + Port: (*t).WebhookInstallOptions.LocalServingPort, + CertDir: (*t).WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + var cerr error + c, cerr = client.New(*cfg, client.Options{Scheme: scheme.Scheme}) + Expect(cerr).NotTo(HaveOccurred()) + m = utils.Matcher{Client: c} + + testCancel, requestsStart, requests = startController(mgr) + + // Create some configmaps and secrets + cm1 = utils.ExampleConfigMap1.DeepCopy() + cm2 = utils.ExampleConfigMap2.DeepCopy() + cm3 = utils.ExampleConfigMap3.DeepCopy() + cm4 = utils.ExampleConfigMap4.DeepCopy() + cm5 = utils.ExampleConfigMap5.DeepCopy() + cm6 = utils.ExampleConfigMap6.DeepCopy() + s1 = utils.ExampleSecret1.DeepCopy() + s2 = utils.ExampleSecret2.DeepCopy() + s3 = utils.ExampleSecret3.DeepCopy() + s4 = utils.ExampleSecret4.DeepCopy() + s5 = utils.ExampleSecret5.DeepCopy() + s6 = utils.ExampleSecret6.DeepCopy() + + m.Create(cm1).Should(Succeed()) + m.Create(cm2).Should(Succeed()) + m.Create(cm3).Should(Succeed()) + m.Create(cm4).Should(Succeed()) + m.Create(cm5).Should(Succeed()) + m.Create(cm6).Should(Succeed()) + m.Create(s1).Should(Succeed()) + m.Create(s2).Should(Succeed()) + m.Create(s3).Should(Succeed()) + m.Create(s4).Should(Succeed()) + m.Create(s5).Should(Succeed()) + m.Create(s6).Should(Succeed()) + m.Get(cm1, timeout).Should(Succeed()) + m.Get(cm2, timeout).Should(Succeed()) + m.Get(cm3, timeout).Should(Succeed()) + m.Get(cm4, timeout).Should(Succeed()) + m.Get(cm5, timeout).Should(Succeed()) + m.Get(cm6, timeout).Should(Succeed()) + m.Get(s1, timeout).Should(Succeed()) + m.Get(s2, timeout).Should(Succeed()) + m.Get(s3, timeout).Should(Succeed()) + m.Get(s4, timeout).Should(Succeed()) + m.Get(s5, timeout).Should(Succeed()) + m.Get(s6, timeout).Should(Succeed()) + + instance = makeObject() + }) + + AfterEach(func() { + testCancel() + + utils.DeleteAll(*cfg, timeout, + &appsv1.DaemonSetList{}, + &appsv1.DeploymentList{}, + &appsv1.StatefulSetList{}, + &corev1.ConfigMapList{}, + &corev1.SecretList{}, + &corev1.EventList{}, + ) + }) + + Context("When a instance with all children existing is reconciled", func() { + BeforeEach(func() { + // Create a instance and wait for it to be reconciled + clearReconciled() + m.Create(instance).Should(Succeed()) + waitForInstanceReconciled(instance) + }) + + Context("And it has the required annotation", func() { + BeforeEach(func() { + addAnnotation := func(obj client.Object) client.Object { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[RequiredAnnotation] = "true" + obj.SetAnnotations(annotations) + return obj + } + clearReconciled() + m.Update(instance, addAnnotation).Should(Succeed()) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Has scheduling enabled", func() { + m.Get(instance, timeout).Should(Succeed()) + Expect(GetPodTemplate(instance).Spec.SchedulerName).To(Equal("default-scheduler")) + Expect(instance.GetAnnotations()).NotTo(HaveKey(SchedulingDisabledAnnotation)) + }) + + It("Adds a config hash to the Pod Template", func() { + Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) + }) + + It("Sends an event when updating the hash", func() { + Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) + + eventMessage := func(event *corev1.Event) string { + return event.Message + } + hashMessage := "Configuration hash updated to 421778c325761f51dbf7a23a20eb9c1bc516ffd4aa7362ebec03175d427d7557" + Eventually(func() *corev1.EventList { + events := &corev1.EventList{} + m.Client.List(context.TODO(), events) + return events + }, timeout).Should(utils.WithItems(ContainElement(WithTransform(eventMessage, Equal(hashMessage))))) + }) + + Context("And a child is removed", func() { + var originalHash string + BeforeEach(func() { + Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) + originalHash = GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + + // Remove "container2" which references Secret example2 and ConfigMap + // example2 + clearReconciled() + m.Get(instance, timeout).Should(Succeed()) + podTemplate := GetPodTemplate(instance) + Expect(podTemplate.Spec.Containers[0].Name).To(Equal("container1")) + podTemplate.Spec.Containers = []corev1.Container{podTemplate.Spec.Containers[0]} + SetPodTemplate(instance, podTemplate) + Expect(m.Client.Update(context.TODO(), instance)).Should(Succeed()) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Updates the config hash in the Pod Template", func() { + Eventually(func() string { + return GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }, timeout).ShouldNot(Equal(originalHash)) + }) + + It("Changes to the removed children no longer trigger a reconcile", func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = "modified" + return cm + } + clearReconciled() + + m.Update(cm2, modifyCM).Should(Succeed()) + consistentlyInstanceNotReconciled(instance) + }) + }) + + Context("And a child is updated", func() { + var originalHash string + + BeforeEach(func() { + m.Get(instance, timeout).Should(Succeed()) + Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) + originalHash = GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }) + + Context("A ConfigMap volume is updated", func() { + BeforeEach(func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = modified + return cm + } + clearReconciled() + m.Update(cm1, modifyCM).Should(Succeed()) + waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Updates the config hash in the Pod Template", func() { + Eventually(func() string { + return GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }, timeout).ShouldNot(Equal(originalHash)) + }) + }) + + Context("A ConfigMap EnvSource is updated", func() { + BeforeEach(func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = modified + return cm + } + clearReconciled() + m.Update(cm2, modifyCM).Should(Succeed()) + waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Updates the config hash in the Pod Template", func() { + Eventually(func() string { + return GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }, timeout).ShouldNot(Equal(originalHash)) + }) + }) + + Context("A Secret volume is updated", func() { + BeforeEach(func() { + modifyS := func(obj client.Object) client.Object { + s, _ := obj.(*corev1.Secret) + if s.StringData == nil { + s.StringData = make(map[string]string) + } + s.StringData["key1"] = modified + return s + } + clearReconciled() + m.Update(s1, modifyS).Should(Succeed()) + waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Updates the config hash in the Pod Template", func() { + Eventually(func() string { + return GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }, timeout).ShouldNot(Equal(originalHash)) + }) + }) + + Context("A Secret EnvSource is updated", func() { + BeforeEach(func() { + modifyS := func(obj client.Object) client.Object { + s, _ := obj.(*corev1.Secret) + if s.StringData == nil { + s.StringData = make(map[string]string) + } + s.StringData["key1"] = modified + return s + } + clearReconciled() + m.Update(s2, modifyS).Should(Succeed()) + waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance) + + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Updates the config hash in the Pod Template", func() { + Eventually(func() string { + return GetPodTemplate(instance).GetAnnotations()[ConfigHashAnnotation] + }, timeout).ShouldNot(Equal(originalHash)) + }) + }) + }) + + Context("And the annotation is removed", func() { + BeforeEach(func() { + removeAnnotations := func(obj client.Object) client.Object { + obj.SetAnnotations(make(map[string]string)) + return obj + } + clearReconciled() + m.Update(instance, removeAnnotations).Should(Succeed()) + waitForInstanceReconciled(instance) + m.Get(instance).Should(Succeed()) + Eventually(instance, timeout).ShouldNot(utils.WithAnnotations(HaveKey(RequiredAnnotation))) + }) + + It("Removes the config hash annotation", func() { + m.Consistently(instance, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(ConfigHashAnnotation))) + }) + + It("Changes to children no longer trigger a reconcile", func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = "modified" + return cm + } + clearReconciled() + + m.Update(cm1, modifyCM).Should(Succeed()) + consistentlyInstanceNotReconciled(instance) + }) + }) + + Context("And is deleted", func() { + BeforeEach(func() { + // Make sure the cache has synced before we run the test + Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) + clearReconciled() + m.Delete(instance).Should(Succeed()) + waitForInstanceReconciled(instance) + }) + It("Not longer exists", func() { + m.Get(instance).Should(MatchError(MatchRegexp(`not found`))) + }) + + It("Changes to children no longer trigger a reconcile", func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = "modified" + return cm + } + clearReconciled() + + m.Update(cm1, modifyCM).Should(Succeed()) + consistentlyInstanceNotReconciled(instance) + }) + }) + }) + + Context("And it does not have the required annotation", func() { + BeforeEach(func() { + // Get the updated instance + m.Get(instance, timeout).Should(Succeed()) + }) + + It("Doesn't add a config hash to the Pod Template", func() { + m.Consistently(instance, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(ConfigHashAnnotation))) + }) + + It("Changes to children no do not trigger a reconcile", func() { + modifyCM := func(obj client.Object) client.Object { + cm, _ := obj.(*corev1.ConfigMap) + cm.Data["key1"] = "modified" + return cm + } + clearReconciled() + + m.Update(cm1, modifyCM).Should(Succeed()) + consistentlyInstanceNotReconciled(instance) + }) + }) + }) + + Context("When a instance with missing children is reconciled", func() { + BeforeEach(func() { + m.Delete(cm1).Should(Succeed()) + + annotations := instance.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[RequiredAnnotation] = "true" + instance.SetAnnotations(annotations) + + // Create a instance and wait for it to be reconciled + clearReconciled() + m.Create(instance).Should(Succeed()) + waitForInstanceReconciled(instance) + }) + + It("Has scheduling disabled", func() { + m.Get(instance, timeout).Should(Succeed()) + Expect(GetPodTemplate(instance).Spec.SchedulerName).To(Equal(SchedulingDisabledSchedulerName)) + Expect(instance.GetAnnotations()[SchedulingDisabledAnnotation]).To(Equal("default-scheduler")) + }) + + Context("And the missing child is created", func() { + BeforeEach(func() { + clearReconciled() + cm1 = utils.ExampleConfigMap1.DeepCopy() + m.Create(cm1).Should(Succeed()) + waitForInstanceReconciled(instance) + }) + + It("Has scheduling renabled", func() { + m.Get(instance, timeout).Should(Succeed()) + Expect(GetPodTemplate(instance).Spec.SchedulerName).To(Equal("default-scheduler")) + Expect(instance.GetAnnotations()).NotTo(HaveKey(SchedulingDisabledAnnotation)) + }) + }) + }) + +} diff --git a/test/utils/test_objects.go b/test/utils/test_objects.go index 19dbe6e0..0e7deba8 100644 --- a/test/utils/test_objects.go +++ b/test/utils/test_objects.go @@ -33,773 +33,335 @@ var annotations = map[string]string{ var trueValue = true -// ExampleDeployment is an example Deployment object for use within test suites -var ExampleDeployment = &appsv1.Deployment{ +var podTemplate = &corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - Labels: labels, - Annotations: annotations, + Labels: labels, }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Spec: corev1.PodSpec{ + SchedulerName: "default-scheduler", + Volumes: []corev1.Volume{ + { + Name: "secret1", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "example1", + }, + }, }, - Spec: corev1.PodSpec{ - SchedulerName: "default-scheduler", - Volumes: []corev1.Volume{ - { - Name: "secret1", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "example1", - }, + { + Name: "secret-optional", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "volume-optional", + Optional: &trueValue, + }, + }, + }, + { + Name: "configmap1", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example1", }, }, - { - Name: "secret-optional", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "volume-optional", - Optional: &trueValue, - }, + }, + }, + { + Name: "configmap-optional", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "volume-optional", }, + Optional: &trueValue, }, - { - Name: "configmap1", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ + }, + }, + { + Name: "projection1", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", + Name: "example5", }, + Optional: &trueValue, }, - }, + }}, }, - { - Name: "configmap-optional", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ + }, + }, + { + Name: "projection2", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "volume-optional", + Name: "example5", }, Optional: &trueValue, }, - }, - }, - { - Name: "projection1", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{{ - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example5", - }, - Optional: &trueValue, - }, - }}, - }, - }, - }, - { - Name: "projection2", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{{ - ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example5", - }, - Optional: &trueValue, - }, - }}, - }, - }, - }, - { - Name: "projection3", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{{ - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example6", - }, - Items: []corev1.KeyToPath{ - { - Key: "example6_key1", - Path: "example6_key1.txt", - }, - { - Key: "example6_key3", - Path: "example6_key3.txt", - }, - }, - }, - }}, - }, - }, - }, - { - Name: "projection4", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{{ - ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example6", - }, - Items: []corev1.KeyToPath{ - { - Key: "example6_key1", - Path: "example6_key1.txt", - }, - { - Key: "example6_key3", - Path: "example6_key3.txt", - }, - }, - }, - }}, - }, - }, + }}, }, }, - Containers: []corev1.Container{ - { - Name: "container1", - Image: "container1", - Env: []corev1.EnvVar{ - { - Name: "example1_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example1_key1_new_name", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", - }, + }, + { + Name: "projection3", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example6", }, - }, - { - Name: "example3_key4", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, + Items: []corev1.KeyToPath{ + { + Key: "example6_key1", + Path: "example6_key1.txt", }, - }, - }, - { - Name: "example4_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, + { + Key: "example6_key3", + Path: "example6_key3.txt", }, }, }, - { - Name: "example1_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, + }}, + }, + }, + }, + { + Name: "projection4", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example6", }, - }, - { - Name: "example3_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", + Items: []corev1.KeyToPath{ + { + Key: "example6_key1", + Path: "example6_key1.txt", }, - }, - }, - { - Name: "example3_secret_key4", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, + { + Key: "example6_key3", + Path: "example6_key3.txt", }, }, }, - { - Name: "example4_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, - }, + }}, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "container1", + Image: "container1", + Env: []corev1.EnvVar{ + { + Name: "example1_key1", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example1", }, + Key: "key1", }, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - }, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "envfrom-optional", - }, - Optional: &trueValue, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "envfrom-optional", - }, - Optional: &trueValue, + }, + { + Name: "example1_key1_new_name", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example1", }, + Key: "key1", }, }, }, { - Name: "container2", - Image: "container2", - Env: []corev1.EnvVar{ - { - Name: "env_optional_key2", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "env-optional", - }, - Key: "key2", - Optional: &trueValue, - }, - }, - }, - { - Name: "example3_key2", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, - }, - { - Name: "example3_secret_key2", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, - }, - { - Name: "env_optional_secret_key2", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "env-optional", - }, - Key: "key2", - Optional: &trueValue, - }, + Name: "example3_key1", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example3", }, + Key: "key1", }, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, + }, + { + Name: "example3_key4", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example3", }, + Key: "key4", + Optional: &trueValue, }, }, }, - }, - }, - }, - }, -} - -// ExampleStatefulSet is an example StatefulSet object for use within test suites -var ExampleStatefulSet = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - Labels: labels, - }, - Spec: appsv1.StatefulSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ { - Name: "secret1", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "example1", + Name: "example4_key1", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example4", + }, + Key: "key1", + Optional: &trueValue, }, }, }, { - Name: "configmap1", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ + Name: "example1_secret_key1", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ Name: "example1", }, + Key: "key1", }, }, }, - }, - Containers: []corev1.Container{ { - Name: "container1", - Image: "container1", - Env: []corev1.EnvVar{ - { - Name: "example1_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example1_key1_new_name", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_key4", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, - }, - }, - }, - { - Name: "example4_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, - }, - }, - }, - { - Name: "example1_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_secret_key4", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, - }, - }, - }, - { - Name: "example4_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, - }, + Name: "example3_secret_key1", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example3", }, + Key: "key1", }, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, + }, + { + Name: "example3_secret_key4", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example3", }, + Key: "key4", + Optional: &trueValue, }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, + }, + }, + { + Name: "example4_secret_key1", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example4", }, + Key: "key1", + Optional: &trueValue, }, }, }, + }, + EnvFrom: []corev1.EnvFromSource{ { - Name: "container2", - Image: "container2", - Env: []corev1.EnvVar{ - { - Name: "example3_key2", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example1", }, - { - Name: "example3_secret_key2", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, + }, + }, + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "envfrom-optional", }, + Optional: &trueValue, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, - }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example1", }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, - }, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "envfrom-optional", }, + Optional: &trueValue, }, }, }, }, - }, - }, -} - -// ExampleDaemonSet is an example DaemonSet object for use within test suites -var ExampleDaemonSet = &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - Labels: labels, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ + { + Name: "container2", + Image: "container2", + Env: []corev1.EnvVar{ { - Name: "secret1", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "example1", + Name: "env_optional_key2", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "env-optional", + }, + Key: "key2", + Optional: &trueValue, }, }, }, { - Name: "configmap1", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ + Name: "example3_key2", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", + Name: "example3", }, + Key: "key2", }, }, }, - }, - Containers: []corev1.Container{ { - Name: "container1", - Image: "container1", - Env: []corev1.EnvVar{ - { - Name: "example1_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example1_key1_new_name", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_key4", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, - }, - }, - }, - { - Name: "example4_key1", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, - }, - }, - }, - { - Name: "example1_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key1", - }, - }, - }, - { - Name: "example3_secret_key4", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key4", - Optional: &trueValue, - }, - }, - }, - { - Name: "example4_secret_key1", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example4", - }, - Key: "key1", - Optional: &trueValue, - }, + Name: "example3_secret_key2", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example3", }, + Key: "key2", }, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example1", - }, + }, + { + Name: "env_optional_secret_key2", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "env-optional", }, + Key: "key2", + Optional: &trueValue, }, }, }, + }, + EnvFrom: []corev1.EnvFromSource{ { - Name: "container2", - Image: "container2", - Env: []corev1.EnvVar{ - { - Name: "example3_key2", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, - }, - { - Name: "example3_secret_key2", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example3", - }, - Key: "key2", - }, - }, + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example2", }, }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "example2", - }, - }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "example2", }, }, }, @@ -809,6 +371,54 @@ var ExampleDaemonSet = &appsv1.DaemonSet{ }, } +// ExampleDeployment is an example Deployment object for use within test suites +var ExampleDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "default", + Labels: labels, + Annotations: annotations, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: *podTemplate, + }, +} + +// ExampleStatefulSet is an example StatefulSet object for use within test suites +var ExampleStatefulSet = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "default", + Labels: labels, + Annotations: annotations, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: *podTemplate, + }, +} + +// ExampleDaemonSet is an example DaemonSet object for use within test suites +var ExampleDaemonSet = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "default", + Labels: labels, + Annotations: annotations, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: *podTemplate, + }, +} + // ExampleConfigMap1 is an example ConfigMap object for use within test suites var ExampleConfigMap1 = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ From ad8fabdecb61e8d814e672a426e91aa20bed0ec4 Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Fri, 10 May 2024 01:11:02 +0200 Subject: [PATCH 4/8] cleanup tests --- pkg/core/children_test.go | 40 +++++++++++++--------------- pkg/core/delete_test.go | 4 +-- pkg/core/finalizer_test.go | 12 ++++----- pkg/core/hash_test.go | 6 ++--- pkg/core/owner_references_test.go | 4 +-- pkg/core/required_annotation_test.go | 8 +++--- pkg/core/scheduler_test.go | 18 ++++++------- 7 files changed, 39 insertions(+), 53 deletions(-) diff --git a/pkg/core/children_test.go b/pkg/core/children_test.go index 6e4babdd..ea5b6188 100644 --- a/pkg/core/children_test.go +++ b/pkg/core/children_test.go @@ -38,7 +38,6 @@ var _ = Describe("Wave children Suite", func() { var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment var currentChildren []configObject var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -102,7 +101,6 @@ var _ = Describe("Wave children Suite", func() { m.Create(s6).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) @@ -138,7 +136,7 @@ var _ = Describe("Wave children Suite", func() { Context("getCurrentChildren", func() { BeforeEach(func() { var err error - configMaps, secrets := getChildNamesByType(podControllerDeployment) + configMaps, secrets := getChildNamesByType(deploymentObject) currentChildren, err = h.getCurrentChildren(configMaps, secrets) Expect(err).NotTo(HaveOccurred()) }) @@ -260,7 +258,7 @@ var _ = Describe("Wave children Suite", func() { m.Delete(s2).Should(Succeed()) m.Get(s2, timeout).ShouldNot(Succeed()) - configMaps, secrets := getChildNamesByType(podControllerDeployment) + configMaps, secrets := getChildNamesByType(deploymentObject) current, err := h.getCurrentChildren(configMaps, secrets) Expect(err).To(HaveOccurred()) Expect(current).To(BeEmpty()) @@ -272,7 +270,7 @@ var _ = Describe("Wave children Suite", func() { var secrets configMetadataMap BeforeEach(func() { - configMaps, secrets = getChildNamesByType(podControllerDeployment) + configMaps, secrets = getChildNamesByType(deploymentObject) }) It("returns ConfigMaps referenced in extra-configmaps annotations", func() { @@ -280,7 +278,7 @@ var _ = Describe("Wave children Suite", func() { configMetadata{required: false, allKeys: true})) Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("test-cm2", "ns2"), configMetadata{required: false, allKeys: true})) - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("local-cm1", podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("local-cm1", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) @@ -289,37 +287,37 @@ var _ = Describe("Wave children Suite", func() { configMetadata{required: false, allKeys: true})) Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("test-secret2", "ns2"), configMetadata{required: false, allKeys: true})) - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("local-secret1", podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("local-secret1", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) It("returns ConfigMaps referenced in Volumes", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm1.GetName(), podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm1.GetName(), deploymentObject.GetNamespace()), configMetadata{required: true, allKeys: true})) }) It("optional ConfigMaps referenced in Volumes are returned as optional", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("volume-optional", podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("volume-optional", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) It("optional Secrets referenced in Volumes are returned as optional", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("volume-optional", podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("volume-optional", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) It("returns ConfigMaps referenced in EnvFrom", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm2.GetName(), podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm2.GetName(), deploymentObject.GetNamespace()), configMetadata{required: true, allKeys: true})) }) It("optional ConfigMaps referenced in EnvFrom are returned as optional", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("envfrom-optional", podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("envfrom-optional", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) It("returns ConfigMaps referenced in Env", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm3.GetName(), podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName(cm3.GetName(), deploymentObject.GetNamespace()), configMetadata{ required: true, allKeys: false, @@ -332,7 +330,7 @@ var _ = Describe("Wave children Suite", func() { }) It("returns ConfigMaps referenced in Env as optional correctly", func() { - Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("env-optional", podControllerDeployment.GetNamespace()), + Expect(configMaps).To(HaveKeyWithValue(GetNamespacedName("env-optional", deploymentObject.GetNamespace()), configMetadata{ required: false, allKeys: false, @@ -343,22 +341,22 @@ var _ = Describe("Wave children Suite", func() { }) It("returns Secrets referenced in Volumes", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s1.GetName(), podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s1.GetName(), deploymentObject.GetNamespace()), configMetadata{required: true, allKeys: true})) }) It("returns Secrets referenced in EnvFrom", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s2.GetName(), podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s2.GetName(), deploymentObject.GetNamespace()), configMetadata{required: true, allKeys: true})) }) It("optional Secrets referenced in EnvFrom are returned as optional", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("envfrom-optional", podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("envfrom-optional", deploymentObject.GetNamespace()), configMetadata{required: false, allKeys: true})) }) It("returns Secrets referenced in Env", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s3.GetName(), podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName(s3.GetName(), deploymentObject.GetNamespace()), configMetadata{ required: true, allKeys: false, @@ -371,7 +369,7 @@ var _ = Describe("Wave children Suite", func() { }) It("returns secrets referenced in Env as optional correctly", func() { - Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("env-optional", podControllerDeployment.GetNamespace()), + Expect(secrets).To(HaveKeyWithValue(GetNamespacedName("env-optional", deploymentObject.GetNamespace()), configMetadata{ required: false, allKeys: false, @@ -401,12 +399,12 @@ var _ = Describe("Wave children Suite", func() { } var err error - _, err = h.getExistingChildren(podControllerDeployment) + _, err = h.getExistingChildren(deploymentObject) Expect(err).NotTo(HaveOccurred()) }) existingChildrenFn := func() ([]Object, error) { - return h.getExistingChildren(podControllerDeployment) + return h.getExistingChildren(deploymentObject) } It("returns ConfigMaps with the correct OwnerReference", func() { diff --git a/pkg/core/delete_test.go b/pkg/core/delete_test.go index 96cb9ec0..a4cea580 100644 --- a/pkg/core/delete_test.go +++ b/pkg/core/delete_test.go @@ -38,7 +38,6 @@ var _ = Describe("Wave migration Suite", func() { var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -66,7 +65,6 @@ var _ = Describe("Wave migration Suite", func() { m.Create(utils.ExampleSecret2.DeepCopy()).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) @@ -113,7 +111,7 @@ var _ = Describe("Wave migration Suite", func() { return obj }, timeout).Should(Succeed()) - _, err := h.handlePodController(podControllerDeployment) + _, err := h.handlePodController(deploymentObject) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/core/finalizer_test.go b/pkg/core/finalizer_test.go index 9f5038f4..f3a16c02 100644 --- a/pkg/core/finalizer_test.go +++ b/pkg/core/finalizer_test.go @@ -25,11 +25,9 @@ import ( var _ = Describe("Wave finalizer Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject // TODO: remove }) Context("removeFinalizer", func() { @@ -37,7 +35,7 @@ var _ = Describe("Wave finalizer Suite", func() { f := deploymentObject.GetFinalizers() f = append(f, FinalizerString) deploymentObject.SetFinalizers(f) - removeFinalizer(podControllerDeployment) + removeFinalizer(deploymentObject) Expect(deploymentObject.GetFinalizers()).NotTo(ContainElement(FinalizerString)) }) @@ -46,7 +44,7 @@ var _ = Describe("Wave finalizer Suite", func() { f := deploymentObject.GetFinalizers() f = append(f, "kubernetes") deploymentObject.SetFinalizers(f) - removeFinalizer(podControllerDeployment) + removeFinalizer(deploymentObject) Expect(deploymentObject.GetFinalizers()).To(ContainElement("kubernetes")) }) @@ -58,18 +56,18 @@ var _ = Describe("Wave finalizer Suite", func() { f = append(f, FinalizerString) deploymentObject.SetFinalizers(f) - Expect(hasFinalizer(podControllerDeployment)).To(BeTrue()) + Expect(hasFinalizer(deploymentObject)).To(BeTrue()) }) It("returns false if the deployment doesn't have the finalizer", func() { // Test without any finalizers - Expect(hasFinalizer(podControllerDeployment)).To(BeFalse()) + Expect(hasFinalizer(deploymentObject)).To(BeFalse()) // Test with a different finalizer f := deploymentObject.GetFinalizers() f = append(f, "kubernetes") deploymentObject.SetFinalizers(f) - Expect(hasFinalizer(podControllerDeployment)).To(BeFalse()) + Expect(hasFinalizer(deploymentObject)).To(BeFalse()) }) }) }) diff --git a/pkg/core/hash_test.go b/pkg/core/hash_test.go index 4a9bf5d5..90870be8 100644 --- a/pkg/core/hash_test.go +++ b/pkg/core/hash_test.go @@ -326,15 +326,13 @@ var _ = Describe("Wave hash Suite", func() { Context("setConfigHash", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment // TODO: remove BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject }) It("sets the hash annotation to the provided value", func() { - setConfigHash(podControllerDeployment, "1234") + setConfigHash(deploymentObject, "1234") podAnnotations := deploymentObject.Spec.Template.GetAnnotations() Expect(podAnnotations).NotTo(BeNil()) @@ -354,7 +352,7 @@ var _ = Describe("Wave hash Suite", func() { deploymentObject.Spec.Template.SetAnnotations(podAnnotations) // Set the config hash - setConfigHash(podControllerDeployment, "1234") + setConfigHash(deploymentObject, "1234") // Check the existing annotation is still in place podAnnotations = deploymentObject.Spec.Template.GetAnnotations() diff --git a/pkg/core/owner_references_test.go b/pkg/core/owner_references_test.go index 117dde70..d71a8e3f 100644 --- a/pkg/core/owner_references_test.go +++ b/pkg/core/owner_references_test.go @@ -39,7 +39,6 @@ var _ = Describe("Wave owner references Suite", func() { var h *Handler[*appsv1.Deployment] var m utils.Matcher var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment var mgrStopped *sync.WaitGroup var stopMgr chan struct{} @@ -83,7 +82,6 @@ var _ = Describe("Wave owner references Suite", func() { m.Create(s3).Should(Succeed()) deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject // TODO: remove m.Create(deploymentObject).Should(Succeed()) @@ -121,7 +119,7 @@ var _ = Describe("Wave owner references Suite", func() { } children := []Object{cm1, s1} - err := h.removeOwnerReferences(podControllerDeployment, children) + err := h.removeOwnerReferences(deploymentObject, children) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/core/required_annotation_test.go b/pkg/core/required_annotation_test.go index 6516bbc3..b0217814 100644 --- a/pkg/core/required_annotation_test.go +++ b/pkg/core/required_annotation_test.go @@ -25,11 +25,9 @@ import ( var _ = Describe("Wave required annotation Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject // TODO: remove }) Context("hasRequiredAnnotation", func() { @@ -41,7 +39,7 @@ var _ = Describe("Wave required annotation Suite", func() { annotations[RequiredAnnotation] = requiredAnnotationValue deploymentObject.SetAnnotations(annotations) - Expect(hasRequiredAnnotation(podControllerDeployment)).To(BeTrue()) + Expect(hasRequiredAnnotation(deploymentObject)).To(BeTrue()) }) It("returns false when the annotation has value other than true", func() { @@ -52,11 +50,11 @@ var _ = Describe("Wave required annotation Suite", func() { annotations[RequiredAnnotation] = "false" deploymentObject.SetAnnotations(annotations) - Expect(hasRequiredAnnotation(podControllerDeployment)).To(BeFalse()) + Expect(hasRequiredAnnotation(deploymentObject)).To(BeFalse()) }) It("returns false when the annotation is not set", func() { - Expect(hasRequiredAnnotation(podControllerDeployment)).To(BeFalse()) + Expect(hasRequiredAnnotation(deploymentObject)).To(BeFalse()) }) }) diff --git a/pkg/core/scheduler_test.go b/pkg/core/scheduler_test.go index aba8c94d..21ab5e1a 100644 --- a/pkg/core/scheduler_test.go +++ b/pkg/core/scheduler_test.go @@ -25,49 +25,47 @@ import ( var _ = Describe("Wave scheduler Suite", func() { var deploymentObject *appsv1.Deployment - var podControllerDeployment *appsv1.Deployment BeforeEach(func() { deploymentObject = utils.ExampleDeployment.DeepCopy() - podControllerDeployment = deploymentObject }) Context("When scheduler is disabled", func() { BeforeEach(func() { - disableScheduling(podControllerDeployment) + disableScheduling(deploymentObject) }) It("Sets the annotations and stores the previous scheduler", func() { - annotations := podControllerDeployment.GetAnnotations() + annotations := deploymentObject.GetAnnotations() Expect(annotations[SchedulingDisabledAnnotation]).To(Equal("default-scheduler")) }) It("Disables scheduling", func() { - podTemplate := GetPodTemplate(podControllerDeployment) + podTemplate := GetPodTemplate(deploymentObject) Expect(podTemplate.Spec.SchedulerName).To(Equal(SchedulingDisabledSchedulerName)) }) It("Is reports as disabled", func() { - Expect(isSchedulingDisabled(podControllerDeployment)).To(BeTrue()) + Expect(isSchedulingDisabled(deploymentObject)).To(BeTrue()) }) Context("And Is Restored", func() { BeforeEach(func() { - restoreScheduling(podControllerDeployment) + restoreScheduling(deploymentObject) }) It("Removes the annotations", func() { - annotations := podControllerDeployment.GetAnnotations() + annotations := deploymentObject.GetAnnotations() Expect(annotations).NotTo(HaveKey(SchedulingDisabledAnnotation)) }) It("Restores the scheduler", func() { - podTemplate := GetPodTemplate(podControllerDeployment) + podTemplate := GetPodTemplate(deploymentObject) Expect(podTemplate.Spec.SchedulerName).To(Equal("default-scheduler")) }) It("Is does not report as disabled", func() { - Expect(isSchedulingDisabled(podControllerDeployment)).To(BeFalse()) + Expect(isSchedulingDisabled(deploymentObject)).To(BeFalse()) }) }) From 097b158ac4327af140a69f7b9b500d6c2a106f5f Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Fri, 10 May 2024 01:24:14 +0200 Subject: [PATCH 5/8] handle return and fix lint --- pkg/core/controller_suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/controller_suite.go b/pkg/core/controller_suite.go index 0cdc2697..8d005fd0 100644 --- a/pkg/core/controller_suite.go +++ b/pkg/core/controller_suite.go @@ -206,7 +206,7 @@ func ControllerTestSuite[I InstanceType]( hashMessage := "Configuration hash updated to 421778c325761f51dbf7a23a20eb9c1bc516ffd4aa7362ebec03175d427d7557" Eventually(func() *corev1.EventList { events := &corev1.EventList{} - m.Client.List(context.TODO(), events) + Expect(m.Client.List(context.TODO(), events)).To(Succeed()) return events }, timeout).Should(utils.WithItems(ContainElement(WithTransform(eventMessage, Equal(hashMessage))))) }) From 0a4bbc4a9c4889add9d19fb87ddfa1adb8dd8ccf Mon Sep 17 00:00:00 2001 From: Jan Kantert Date: Mon, 13 May 2024 23:09:26 +0200 Subject: [PATCH 6/8] more robust tests --- .../daemonset_controller_suite_test.go | 70 ++- .../daemonset/daemonset_controller_test.go | 22 +- .../deployment_controller_suite_test.go | 69 ++- .../deployment/deployment_controller_test.go | 531 +----------------- .../statefulset_controller_suite_test.go | 68 ++- .../statefulset_controller_test.go | 22 +- pkg/core/controller_suite.go | 159 +++--- 7 files changed, 224 insertions(+), 717 deletions(-) diff --git a/pkg/controller/daemonset/daemonset_controller_suite_test.go b/pkg/controller/daemonset/daemonset_controller_suite_test.go index a05cca5e..bc73a298 100644 --- a/pkg/controller/daemonset/daemonset_controller_suite_test.go +++ b/pkg/controller/daemonset/daemonset_controller_suite_test.go @@ -24,13 +24,20 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - ctrl "sigs.k8s.io/controller-runtime" - + "github.com/prometheus/client_golang/prometheus" + "github.com/wave-k8s/wave/pkg/core" + "github.com/wave-k8s/wave/test/utils" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,6 +54,11 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) +var requestsStart <-chan reconcile.Request +var requests <-chan reconcile.Request + +var m utils.Matcher + var _ = BeforeSuite(func() { failurePolicy := admissionv1.Ignore sideEffects := admissionv1.SideEffectClassNone @@ -101,30 +113,40 @@ var _ = BeforeSuite(func() { if cfg, err = t.Start(); err != nil { log.Fatal(err) } + + // Reset the Prometheus Registry before each test to avoid errors + metrics.Registry = prometheus.NewRegistry() + + mgr, err := manager.New(cfg, manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: (*t).WebhookInstallOptions.LocalServingHost, + Port: (*t).WebhookInstallOptions.LocalServingPort, + CertDir: (*t).WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + c, cerr := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(cerr).NotTo(HaveOccurred()) + m = utils.Matcher{Client: c} + + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests = core.SetupControllerTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err = AddDaemonSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go core.Run(testCtx, mgr) }) var _ = AfterSuite(func() { + testCancel() t.Stop() }) - -// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and -// writes the request to requests after Reconcile is finished. -func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request, chan reconcile.Request) { - requestsStart := make(chan reconcile.Request) - requests := make(chan reconcile.Request) - fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - requestsStart <- req - result, err := inner.Reconcile(ctx, req) - requests <- req - return result, err - }) - return fn, requestsStart, requests -} - -// Run runs the webhook server. -func Run(ctx context.Context, k8sManager ctrl.Manager) error { - if err := k8sManager.Start(ctx); err != nil { - return err - } - return nil -} diff --git a/pkg/controller/daemonset/daemonset_controller_test.go b/pkg/controller/daemonset/daemonset_controller_test.go index 621572a5..fadbd13c 100644 --- a/pkg/controller/daemonset/daemonset_controller_test.go +++ b/pkg/controller/daemonset/daemonset_controller_test.go @@ -17,36 +17,18 @@ limitations under the License. package daemonset import ( - "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/wave-k8s/wave/pkg/core" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ = Describe("DaemonSet controller Suite", func() { core.ControllerTestSuite( - &t, &cfg, + &t, &cfg, &m, + &requestsStart, &requests, func() *appsv1.DaemonSet { return utils.ExampleDaemonSet.DeepCopy() }, - func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests := SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err := AddDaemonSetWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - return testCancel, requestsStart, requests - }, ) }) diff --git a/pkg/controller/deployment/deployment_controller_suite_test.go b/pkg/controller/deployment/deployment_controller_suite_test.go index 49c3213e..0b87a220 100644 --- a/pkg/controller/deployment/deployment_controller_suite_test.go +++ b/pkg/controller/deployment/deployment_controller_suite_test.go @@ -22,17 +22,23 @@ import ( "path/filepath" "testing" - ctrl "sigs.k8s.io/controller-runtime" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" "github.com/wave-k8s/wave/pkg/apis" + "github.com/wave-k8s/wave/pkg/core" + "github.com/wave-k8s/wave/test/utils" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,6 +55,11 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) +var requestsStart <-chan reconcile.Request +var requests <-chan reconcile.Request + +var m utils.Matcher + var _ = BeforeSuite(func() { failurePolicy := admissionv1.Ignore sideEffects := admissionv1.SideEffectClassNone @@ -104,30 +115,40 @@ var _ = BeforeSuite(func() { if cfg, err = t.Start(); err != nil { log.Fatal(err) } + + // Reset the Prometheus Registry before each test to avoid errors + metrics.Registry = prometheus.NewRegistry() + + mgr, err := manager.New(cfg, manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: (*t).WebhookInstallOptions.LocalServingHost, + Port: (*t).WebhookInstallOptions.LocalServingPort, + CertDir: (*t).WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + c, cerr := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(cerr).NotTo(HaveOccurred()) + m = utils.Matcher{Client: c} + + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests = core.SetupControllerTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err = AddDeploymentWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go core.Run(testCtx, mgr) }) var _ = AfterSuite(func() { + testCancel() t.Stop() }) - -// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and -// writes the request to requests after Reconcile is finished. -func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request, chan reconcile.Request) { - requestsStart := make(chan reconcile.Request) - requests := make(chan reconcile.Request) - fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - requestsStart <- req - result, err := inner.Reconcile(ctx, req) - requests <- req - return result, err - }) - return fn, requestsStart, requests -} - -// Run runs the webhook server. -func Run(ctx context.Context, k8sManager ctrl.Manager) error { - if err := k8sManager.Start(ctx); err != nil { - return err - } - return nil -} diff --git a/pkg/controller/deployment/deployment_controller_test.go b/pkg/controller/deployment/deployment_controller_test.go index 5308453e..9ca55c86 100644 --- a/pkg/controller/deployment/deployment_controller_test.go +++ b/pkg/controller/deployment/deployment_controller_test.go @@ -17,545 +17,18 @@ limitations under the License. package deployment import ( - "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/wave-k8s/wave/pkg/core" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ = Describe("Deployment controller Suite", func() { core.ControllerTestSuite( - &t, &cfg, + &t, &cfg, &m, + &requestsStart, &requests, func() *appsv1.Deployment { return utils.ExampleDeployment.DeepCopy() }, - func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests := SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err := AddDeploymentWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - return testCancel, requestsStart, requests - }, ) }) - -/* -Copyright 2018 Pusher Ltd. and Wave Contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -package deployment - -import ( - "context" - "time" - - "sigs.k8s.io/controller-runtime/pkg/cache" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" - "github.com/wave-k8s/wave/pkg/core" - "github.com/wave-k8s/wave/test/utils" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - webhook "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -var _ = Describe("Deployment controller Suite", func() { - var c client.Client - var m utils.Matcher - - var deployment *appsv1.Deployment - var requestsStart <-chan reconcile.Request - var requests <-chan reconcile.Request - - const timeout = time.Second * 5 - const consistentlyTimeout = time.Second - - var cm1 *corev1.ConfigMap - var cm2 *corev1.ConfigMap - var cm3 *corev1.ConfigMap - var cm4 *corev1.ConfigMap - var cm5 *corev1.ConfigMap - var cm6 *corev1.ConfigMap - var s1 *corev1.Secret - var s2 *corev1.Secret - var s3 *corev1.Secret - var s4 *corev1.Secret - var s5 *corev1.Secret - var s6 *corev1.Secret - - const modified = "modified" - - var waitForDeploymentReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the DaemonSet - Eventually(requestsStart, timeout).Should(Receive(Equal(request))) - Eventually(requests, timeout).Should(Receive(Equal(request))) - } - - var consistentlyDeploymentNotReconciled = func(obj core.Object) { - request := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, - } - // wait for reconcile for creating the DaemonSet - Consistently(requestsStart, consistentlyTimeout).ShouldNot(Receive(Equal(request))) - } - - var clearReconciled = func() { - for len(requestsStart) > 0 { - <-requestsStart - <-requests - } - } - - BeforeEach(func() { - // Reset the Prometheus Registry before each test to avoid errors - metrics.Registry = prometheus.NewRegistry() - - mgr, err := manager.New(cfg, manager.Options{ - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: t.WebhookInstallOptions.LocalServingHost, - Port: t.WebhookInstallOptions.LocalServingPort, - CertDir: t.WebhookInstallOptions.LocalServingCertDir, - }), - Cache: cache.Options{ - DefaultNamespaces: core.BuildCacheDefaultNamespaces(""), - }, - }) - Expect(err).NotTo(HaveOccurred()) - var cerr error - c, cerr = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(cerr).NotTo(HaveOccurred()) - m = utils.Matcher{Client: c} - - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests = SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err = AddDeploymentWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - - // Create some configmaps and secrets - cm1 = utils.ExampleConfigMap1.DeepCopy() - cm2 = utils.ExampleConfigMap2.DeepCopy() - cm3 = utils.ExampleConfigMap3.DeepCopy() - cm4 = utils.ExampleConfigMap4.DeepCopy() - cm5 = utils.ExampleConfigMap5.DeepCopy() - cm6 = utils.ExampleConfigMap6.DeepCopy() - s1 = utils.ExampleSecret1.DeepCopy() - s2 = utils.ExampleSecret2.DeepCopy() - s3 = utils.ExampleSecret3.DeepCopy() - s4 = utils.ExampleSecret4.DeepCopy() - s5 = utils.ExampleSecret5.DeepCopy() - s6 = utils.ExampleSecret6.DeepCopy() - - m.Create(cm1).Should(Succeed()) - m.Create(cm2).Should(Succeed()) - m.Create(cm3).Should(Succeed()) - m.Create(cm4).Should(Succeed()) - m.Create(cm5).Should(Succeed()) - m.Create(cm6).Should(Succeed()) - m.Create(s1).Should(Succeed()) - m.Create(s2).Should(Succeed()) - m.Create(s3).Should(Succeed()) - m.Create(s4).Should(Succeed()) - m.Create(s5).Should(Succeed()) - m.Create(s6).Should(Succeed()) - m.Get(cm1, timeout).Should(Succeed()) - m.Get(cm2, timeout).Should(Succeed()) - m.Get(cm3, timeout).Should(Succeed()) - m.Get(cm4, timeout).Should(Succeed()) - m.Get(cm5, timeout).Should(Succeed()) - m.Get(cm6, timeout).Should(Succeed()) - m.Get(s1, timeout).Should(Succeed()) - m.Get(s2, timeout).Should(Succeed()) - m.Get(s3, timeout).Should(Succeed()) - m.Get(s4, timeout).Should(Succeed()) - m.Get(s5, timeout).Should(Succeed()) - m.Get(s6, timeout).Should(Succeed()) - - deployment = utils.ExampleDeployment.DeepCopy() - }) - - AfterEach(func() { - testCancel() - - utils.DeleteAll(cfg, timeout, - &appsv1.DeploymentList{}, - &corev1.ConfigMapList{}, - &corev1.SecretList{}, - &corev1.EventList{}, - ) - }) - - Context("When a Deployment with all children existing is reconciled", func() { - BeforeEach(func() { - // Create a deployment and wait for it to be reconciled - clearReconciled() - m.Create(deployment).Should(Succeed()) - waitForDeploymentReconciled(deployment) - }) - - Context("And it has the required annotation", func() { - BeforeEach(func() { - addAnnotation := func(obj client.Object) client.Object { - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - obj.SetAnnotations(annotations) - return obj - } - clearReconciled() - m.Update(deployment, addAnnotation).Should(Succeed()) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Has scheduling enabled", func() { - m.Get(deployment, timeout).Should(Succeed()) - Expect(deployment.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(deployment.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - - It("Adds a config hash to the Pod Template", func() { - Eventually(deployment, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - }) - - It("Sends an event when updating the hash", func() { - Eventually(deployment, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - - eventMessage := func(event *corev1.Event) string { - return event.Message - } - hashMessage := "Configuration hash updated to 421778c325761f51dbf7a23a20eb9c1bc516ffd4aa7362ebec03175d427d7557" - Eventually(func() *corev1.EventList { - events := &corev1.EventList{} - m.Client.List(context.TODO(), events) - return events - }, timeout).Should(utils.WithItems(ContainElement(WithTransform(eventMessage, Equal(hashMessage))))) - }) - - Context("And a child is removed", func() { - var originalHash string - BeforeEach(func() { - Eventually(deployment, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - - // Remove "container2" which references Secret example2 and ConfigMap - // example2 - removeContainer2 := func(obj client.Object) client.Object { - dep, _ := obj.(*appsv1.Deployment) - containers := dep.Spec.Template.Spec.Containers - Expect(containers[0].Name).To(Equal("container1")) - dep.Spec.Template.Spec.Containers = []corev1.Container{containers[0]} - return dep - } - clearReconciled() - m.Update(deployment, removeContainer2).Should(Succeed()) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - - It("Changes to the removed children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm2, modifyCM).Should(Succeed()) - consistentlyDeploymentNotReconciled(deployment) - }) - }) - - Context("And a child is updated", func() { - var originalHash string - - BeforeEach(func() { - m.Get(deployment, timeout).Should(Succeed()) - Eventually(deployment, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - originalHash = deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }) - - Context("A ConfigMap volume is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm1, modifyCM).Should(Succeed()) - waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A ConfigMap EnvSource is updated", func() { - BeforeEach(func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = modified - return cm - } - clearReconciled() - m.Update(cm2, modifyCM).Should(Succeed()) - waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret volume is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s1, modifyS).Should(Succeed()) - waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - - Context("A Secret EnvSource is updated", func() { - BeforeEach(func() { - modifyS := func(obj client.Object) client.Object { - s, _ := obj.(*corev1.Secret) - if s.StringData == nil { - s.StringData = make(map[string]string) - } - s.StringData["key1"] = modified - return s - } - clearReconciled() - m.Update(s2, modifyS).Should(Succeed()) - waitForDeploymentReconciled(deployment) - waitForDeploymentReconciled(deployment) - - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Updates the config hash in the Pod Template", func() { - Eventually(func() string { - return deployment.Spec.Template.GetAnnotations()[core.ConfigHashAnnotation] - }, timeout).ShouldNot(Equal(originalHash)) - }) - }) - }) - - Context("And the annotation is removed", func() { - BeforeEach(func() { - removeAnnotations := func(obj client.Object) client.Object { - obj.SetAnnotations(make(map[string]string)) - return obj - } - clearReconciled() - m.Update(deployment, removeAnnotations).Should(Succeed()) - waitForDeploymentReconciled(deployment) - m.Get(deployment).Should(Succeed()) - Eventually(deployment, timeout).ShouldNot(utils.WithAnnotations(HaveKey(core.RequiredAnnotation))) - }) - - It("Removes the config hash annotation", func() { - m.Consistently(deployment, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDeploymentNotReconciled(deployment) - }) - }) - - Context("And is deleted", func() { - BeforeEach(func() { - // Make sure the cache has synced before we run the test - Eventually(deployment, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(core.ConfigHashAnnotation))) - clearReconciled() - m.Delete(deployment).Should(Succeed()) - waitForDeploymentReconciled(deployment) - }) - It("Not longer exists", func() { - m.Get(deployment).Should(MatchError(MatchRegexp(`not found`))) - }) - - It("Changes to children no longer trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDeploymentNotReconciled(deployment) - }) - }) - }) - - Context("And it does not have the required annotation", func() { - BeforeEach(func() { - // Get the updated Deployment - m.Get(deployment, timeout).Should(Succeed()) - }) - - It("Doesn't add a config hash to the Pod Template", func() { - m.Consistently(deployment, consistentlyTimeout).ShouldNot(utils.WithAnnotations(ContainElement(core.ConfigHashAnnotation))) - }) - - It("Changes to children no do not trigger a reconcile", func() { - modifyCM := func(obj client.Object) client.Object { - cm, _ := obj.(*corev1.ConfigMap) - cm.Data["key1"] = "modified" - return cm - } - clearReconciled() - - m.Update(cm1, modifyCM).Should(Succeed()) - consistentlyDeploymentNotReconciled(deployment) - }) - }) - }) - - Context("When a Deployment with missing children is reconciled", func() { - BeforeEach(func() { - m.Delete(cm1).Should(Succeed()) - - annotations := deployment.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[core.RequiredAnnotation] = "true" - deployment.SetAnnotations(annotations) - - // Create a deployment and wait for it to be reconciled - clearReconciled() - m.Create(deployment).Should(Succeed()) - waitForDeploymentReconciled(deployment) - }) - - It("Has scheduling disabled", func() { - m.Get(deployment, timeout).Should(Succeed()) - Expect(deployment.Spec.Template.Spec.SchedulerName).To(Equal(core.SchedulingDisabledSchedulerName)) - Expect(deployment.ObjectMeta.Annotations[core.SchedulingDisabledAnnotation]).To(Equal("default-scheduler")) - }) - - Context("And the missing child is created", func() { - BeforeEach(func() { - clearReconciled() - cm1 = utils.ExampleConfigMap1.DeepCopy() - m.Create(cm1).Should(Succeed()) - waitForDeploymentReconciled(deployment) - }) - - It("Has scheduling renabled", func() { - m.Get(deployment, timeout).Should(Succeed()) - Expect(deployment.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler")) - Expect(deployment.ObjectMeta.Annotations).NotTo(HaveKey(core.SchedulingDisabledAnnotation)) - }) - }) - }) - -}) -*/ diff --git a/pkg/controller/statefulset/statefulset_controller_suite_test.go b/pkg/controller/statefulset/statefulset_controller_suite_test.go index 67fc8705..1a5aff1e 100644 --- a/pkg/controller/statefulset/statefulset_controller_suite_test.go +++ b/pkg/controller/statefulset/statefulset_controller_suite_test.go @@ -24,15 +24,21 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - ctrl "sigs.k8s.io/controller-runtime" - + "github.com/prometheus/client_golang/prometheus" "github.com/wave-k8s/wave/pkg/apis" + "github.com/wave-k8s/wave/pkg/core" + "github.com/wave-k8s/wave/test/utils" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,6 +55,11 @@ var t *envtest.Environment var testCtx, testCancel = context.WithCancel(context.Background()) +var requestsStart <-chan reconcile.Request +var requests <-chan reconcile.Request + +var m utils.Matcher + var _ = BeforeSuite(func() { failurePolicy := admissionv1.Ignore sideEffects := admissionv1.SideEffectClassNone @@ -105,30 +116,39 @@ var _ = BeforeSuite(func() { log.Fatal(err) } + // Reset the Prometheus Registry before each test to avoid errors + metrics.Registry = prometheus.NewRegistry() + + mgr, err := manager.New(cfg, manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: (*t).WebhookInstallOptions.LocalServingHost, + Port: (*t).WebhookInstallOptions.LocalServingPort, + CertDir: (*t).WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + c, cerr := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(cerr).NotTo(HaveOccurred()) + m = utils.Matcher{Client: c} + + var recFn reconcile.Reconciler + r := newReconciler(mgr) + recFn, requestsStart, requests = core.SetupControllerTestReconcile(r) + Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) + + // register mutating pod webhook + err = AddStatefulSetWebhook(mgr) + Expect(err).ToNot(HaveOccurred()) + + testCtx, testCancel = context.WithCancel(context.Background()) + go core.Run(testCtx, mgr) }) var _ = AfterSuite(func() { + testCancel() t.Stop() }) - -// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and -// writes the request to requests after Reconcile is finished. -func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request, chan reconcile.Request) { - requestsStart := make(chan reconcile.Request) - requests := make(chan reconcile.Request) - fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - requestsStart <- req - result, err := inner.Reconcile(ctx, req) - requests <- req - return result, err - }) - return fn, requestsStart, requests -} - -// Run runs the webhook server. -func Run(ctx context.Context, k8sManager ctrl.Manager) error { - if err := k8sManager.Start(ctx); err != nil { - return err - } - return nil -} diff --git a/pkg/controller/statefulset/statefulset_controller_test.go b/pkg/controller/statefulset/statefulset_controller_test.go index bfa6ef30..6b111f09 100644 --- a/pkg/controller/statefulset/statefulset_controller_test.go +++ b/pkg/controller/statefulset/statefulset_controller_test.go @@ -17,36 +17,18 @@ limitations under the License. package statefulset import ( - "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/wave-k8s/wave/pkg/core" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ = Describe("StatefulSet controller Suite", func() { core.ControllerTestSuite( - &t, &cfg, + &t, &cfg, &m, + &requestsStart, &requests, func() *appsv1.StatefulSet { return utils.ExampleStatefulSet.DeepCopy() }, - func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request) { - var recFn reconcile.Reconciler - r := newReconciler(mgr) - recFn, requestsStart, requests := SetupTestReconcile(r) - Expect(add(mgr, recFn, r.handler)).NotTo(HaveOccurred()) - - // register mutating pod webhook - err := AddStatefulSetWebhook(mgr) - Expect(err).ToNot(HaveOccurred()) - - testCtx, testCancel = context.WithCancel(context.Background()) - go Run(testCtx, mgr) - return testCancel, requestsStart, requests - }, ) }) diff --git a/pkg/core/controller_suite.go b/pkg/core/controller_suite.go index 8d005fd0..e0a4d30a 100644 --- a/pkg/core/controller_suite.go +++ b/pkg/core/controller_suite.go @@ -6,34 +6,56 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" "github.com/wave-k8s/wave/test/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/reconcile" - webhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) -func ControllerTestSuite[I InstanceType]( - t **envtest.Environment, cfg **rest.Config, - makeObject func() I, - startController func(mgr manager.Manager) (context.CancelFunc, chan reconcile.Request, chan reconcile.Request)) { - var c client.Client - var m utils.Matcher +// SetupControllerTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupControllerTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request, chan reconcile.Request) { + requestsStart := make(chan reconcile.Request) + requests := make(chan reconcile.Request) + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + requestsStart <- req + result, err := inner.Reconcile(ctx, req) + requests <- req + return result, err + }) + return fn, requestsStart, requests +} - var testCancel context.CancelFunc +// Run runs the webhook server. +func Run(ctx context.Context, k8sManager ctrl.Manager) error { + defer GinkgoRecover() + if err := k8sManager.Start(ctx); err != nil { + return err + } + return nil +} + +func withTimeout(ch <-chan reconcile.Request, timeout time.Duration) (ok bool) { + select { + case <-ch: + return true + case <-time.After(timeout): + } + return false +} + +func ControllerTestSuite[I InstanceType]( + t **envtest.Environment, cfg **rest.Config, mRef *utils.Matcher, + requestsStart *<-chan reconcile.Request, requests *<-chan reconcile.Request, + makeObject func() I) { var instance I - var requestsStart <-chan reconcile.Request - var requests <-chan reconcile.Request + var m utils.Matcher const timeout = time.Second * 5 const consistentlyTimeout = time.Second @@ -53,16 +75,18 @@ func ControllerTestSuite[I InstanceType]( const modified = "modified" - var waitForInstanceReconciled = func(obj Object) { + var waitForInstanceReconciled = func(obj Object, times int) { request := reconcile.Request{ NamespacedName: types.NamespacedName{ Name: obj.GetName(), Namespace: obj.GetNamespace(), }, } - // wait for reconcile for creating the DaemonSet - Eventually(requestsStart, timeout).Should(Receive(Equal(request))) - Eventually(requests, timeout).Should(Receive(Equal(request))) + for range times { + // wait for reconcile for creating the DaemonSet + Eventually(*requestsStart, timeout).Should(Receive(Equal(request))) + Eventually(*requests, timeout).Should(Receive(Equal(request))) + } } var consistentlyInstanceNotReconciled = func(obj Object) { @@ -73,37 +97,15 @@ func ControllerTestSuite[I InstanceType]( }, } // wait for reconcile for creating the DaemonSet - Consistently(requestsStart, consistentlyTimeout).ShouldNot(Receive(Equal(request))) + Consistently(*requestsStart, .1).ShouldNot(Receive(Equal(request))) } - var clearReconciled = func() { - for len(requestsStart) > 0 { - <-requestsStart - <-requests - } + var expectNoReconciles = func() { + consistentlyInstanceNotReconciled(instance) } BeforeEach(func() { - // Reset the Prometheus Registry before each test to avoid errors - metrics.Registry = prometheus.NewRegistry() - - mgr, err := manager.New(*cfg, manager.Options{ - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: (*t).WebhookInstallOptions.LocalServingHost, - Port: (*t).WebhookInstallOptions.LocalServingPort, - CertDir: (*t).WebhookInstallOptions.LocalServingCertDir, - }), - }) - Expect(err).NotTo(HaveOccurred()) - var cerr error - c, cerr = client.New(*cfg, client.Options{Scheme: scheme.Scheme}) - Expect(cerr).NotTo(HaveOccurred()) - m = utils.Matcher{Client: c} - - testCancel, requestsStart, requests = startController(mgr) + m = *mRef // Create some configmaps and secrets cm1 = utils.ExampleConfigMap1.DeepCopy() @@ -148,7 +150,7 @@ func ControllerTestSuite[I InstanceType]( }) AfterEach(func() { - testCancel() + expectNoReconciles() utils.DeleteAll(*cfg, timeout, &appsv1.DaemonSetList{}, @@ -158,14 +160,22 @@ func ControllerTestSuite[I InstanceType]( &corev1.SecretList{}, &corev1.EventList{}, ) + // Let cleanup happen + for { + if ok := withTimeout(*requestsStart, time.Millisecond*100); ok { + <-*requests + } else { + break + } + } }) Context("When a instance with all children existing is reconciled", func() { BeforeEach(func() { // Create a instance and wait for it to be reconciled - clearReconciled() + expectNoReconciles() m.Create(instance).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) }) Context("And it has the required annotation", func() { @@ -179,9 +189,9 @@ func ControllerTestSuite[I InstanceType]( obj.SetAnnotations(annotations) return obj } - clearReconciled() + expectNoReconciles() m.Update(instance, addAnnotation).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -219,14 +229,14 @@ func ControllerTestSuite[I InstanceType]( // Remove "container2" which references Secret example2 and ConfigMap // example2 - clearReconciled() + expectNoReconciles() m.Get(instance, timeout).Should(Succeed()) podTemplate := GetPodTemplate(instance) Expect(podTemplate.Spec.Containers[0].Name).To(Equal("container1")) podTemplate.Spec.Containers = []corev1.Container{podTemplate.Spec.Containers[0]} SetPodTemplate(instance, podTemplate) Expect(m.Client.Update(context.TODO(), instance)).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -244,7 +254,7 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = "modified" return cm } - clearReconciled() + expectNoReconciles() m.Update(cm2, modifyCM).Should(Succeed()) consistentlyInstanceNotReconciled(instance) @@ -267,10 +277,9 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = modified return cm } - clearReconciled() + expectNoReconciles() m.Update(cm1, modifyCM).Should(Succeed()) - waitForInstanceReconciled(instance) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 2) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -290,10 +299,9 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = modified return cm } - clearReconciled() + expectNoReconciles() m.Update(cm2, modifyCM).Should(Succeed()) - waitForInstanceReconciled(instance) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 2) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -316,10 +324,9 @@ func ControllerTestSuite[I InstanceType]( s.StringData["key1"] = modified return s } - clearReconciled() + expectNoReconciles() m.Update(s1, modifyS).Should(Succeed()) - waitForInstanceReconciled(instance) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 2) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -342,10 +349,9 @@ func ControllerTestSuite[I InstanceType]( s.StringData["key1"] = modified return s } - clearReconciled() + expectNoReconciles() m.Update(s2, modifyS).Should(Succeed()) - waitForInstanceReconciled(instance) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 2) // Get the updated instance m.Get(instance, timeout).Should(Succeed()) @@ -365,9 +371,9 @@ func ControllerTestSuite[I InstanceType]( obj.SetAnnotations(make(map[string]string)) return obj } - clearReconciled() + expectNoReconciles() m.Update(instance, removeAnnotations).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) m.Get(instance).Should(Succeed()) Eventually(instance, timeout).ShouldNot(utils.WithAnnotations(HaveKey(RequiredAnnotation))) }) @@ -382,7 +388,7 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = "modified" return cm } - clearReconciled() + expectNoReconciles() m.Update(cm1, modifyCM).Should(Succeed()) consistentlyInstanceNotReconciled(instance) @@ -393,9 +399,9 @@ func ControllerTestSuite[I InstanceType]( BeforeEach(func() { // Make sure the cache has synced before we run the test Eventually(instance, timeout).Should(utils.WithPodTemplateAnnotations(HaveKey(ConfigHashAnnotation))) - clearReconciled() + expectNoReconciles() m.Delete(instance).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) }) It("Not longer exists", func() { m.Get(instance).Should(MatchError(MatchRegexp(`not found`))) @@ -407,7 +413,7 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = "modified" return cm } - clearReconciled() + expectNoReconciles() m.Update(cm1, modifyCM).Should(Succeed()) consistentlyInstanceNotReconciled(instance) @@ -431,7 +437,7 @@ func ControllerTestSuite[I InstanceType]( cm.Data["key1"] = "modified" return cm } - clearReconciled() + expectNoReconciles() m.Update(cm1, modifyCM).Should(Succeed()) consistentlyInstanceNotReconciled(instance) @@ -451,9 +457,9 @@ func ControllerTestSuite[I InstanceType]( instance.SetAnnotations(annotations) // Create a instance and wait for it to be reconciled - clearReconciled() + expectNoReconciles() m.Create(instance).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 1) }) It("Has scheduling disabled", func() { @@ -464,10 +470,10 @@ func ControllerTestSuite[I InstanceType]( Context("And the missing child is created", func() { BeforeEach(func() { - clearReconciled() + expectNoReconciles() cm1 = utils.ExampleConfigMap1.DeepCopy() m.Create(cm1).Should(Succeed()) - waitForInstanceReconciled(instance) + waitForInstanceReconciled(instance, 2) // Two since updating the scheduler self-triggers }) It("Has scheduling renabled", func() { @@ -476,6 +482,7 @@ func ControllerTestSuite[I InstanceType]( Expect(instance.GetAnnotations()).NotTo(HaveKey(SchedulingDisabledAnnotation)) }) }) + }) } From ccd3e20a782af321402993f69002cab24506e235 Mon Sep 17 00:00:00 2001 From: jabdoa2 Date: Mon, 3 Jun 2024 16:08:51 +0200 Subject: [PATCH 7/8] Update pkg/core/controller.go Co-authored-by: Philipp Riederer --- pkg/core/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/controller.go b/pkg/core/controller.go index 8548cb42..9f87a03f 100644 --- a/pkg/core/controller.go +++ b/pkg/core/controller.go @@ -45,7 +45,7 @@ func AddController[I InstanceType](name string, typeInstance I, mgr manager.Mana return err } - // Watch Secrets owned by a DaemonSet + // Watch Secrets owned by Resource err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}), EnqueueRequestForWatcher(h.GetWatchedSecrets())) if err != nil { return err From cfe3509ca6d853ea13704a8987972fd20196db85 Mon Sep 17 00:00:00 2001 From: jabdoa2 Date: Mon, 3 Jun 2024 16:09:06 +0200 Subject: [PATCH 8/8] Update pkg/core/controller.go Co-authored-by: Philipp Riederer --- pkg/core/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/controller.go b/pkg/core/controller.go index 9f87a03f..f4b91b79 100644 --- a/pkg/core/controller.go +++ b/pkg/core/controller.go @@ -39,7 +39,7 @@ func AddController[I InstanceType](name string, typeInstance I, mgr manager.Mana return err } - // Watch ConfigMaps owned by a DaemonSet + // Watch ConfigMaps owned by Resource err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), EnqueueRequestForWatcher(h.GetWatchedConfigmaps())) if err != nil { return err