From 9827ff22f463a1e7ffaeb8ff4125512ad386e732 Mon Sep 17 00:00:00 2001 From: kranurag7 <81210977+kranurag7@users.noreply.github.com> Date: Sat, 16 Mar 2024 01:43:16 +0530 Subject: [PATCH] add webhooks Signed-off-by: kranurag7 <81210977+kranurag7@users.noreply.github.com> --- Makefile | 2 +- Tiltfile | 2 +- api/v1alpha1/cluster_webhook.go | 154 ++++++ api/v1alpha1/clusteraddon_webhook.go | 102 ++++ api/v1alpha1/clusterstack_webhook.go | 193 ++++++++ api/v1alpha1/clusterstackrelease_webhook.go | 120 +++++ api/v1alpha1/webhook.go | 35 ++ api/v1alpha1/zz_generated.deepcopy.go | 2 +- cmd/main.go | 31 +- config/default/kustomization.yaml | 59 +-- config/localmode/kustomization.yaml | 59 +-- config/webhook/manifests.yaml | 42 ++ go.mod | 1 + .../clusteraddon_controller_test.go | 463 ++++++++++++++++++ .../clusteraddoncreate_controller_test.go | 32 +- .../clusterstack_controller_test.go | 222 +++++++++ .../clusterstackrelease_controller_test.go | 84 ++++ internal/controller/controller_suite_test.go | 16 +- internal/test/helpers/envtest.go | 23 + internal/test/helpers/webhook.go | 127 +++++ pkg/kube/mocks/KFactory.go | 22 + pkg/workloadcluster/fake/restconfig.go | 61 +++ vendor/github.com/MakeNowJust/heredoc/LICENSE | 21 + .../github.com/MakeNowJust/heredoc/README.md | 52 ++ .../github.com/MakeNowJust/heredoc/heredoc.go | 105 ++++ vendor/modules.txt | 4 + .../sigs.k8s.io/cluster-api/util/yaml/yaml.go | 279 +++++++++++ 27 files changed, 2239 insertions(+), 74 deletions(-) create mode 100644 api/v1alpha1/cluster_webhook.go create mode 100644 api/v1alpha1/clusteraddon_webhook.go create mode 100644 api/v1alpha1/clusterstack_webhook.go create mode 100644 api/v1alpha1/clusterstackrelease_webhook.go create mode 100644 api/v1alpha1/webhook.go create mode 100644 internal/controller/clusteraddon_controller_test.go create mode 100644 internal/test/helpers/webhook.go create mode 100644 pkg/kube/mocks/KFactory.go create mode 100644 pkg/workloadcluster/fake/restconfig.go create mode 100644 vendor/github.com/MakeNowJust/heredoc/LICENSE create mode 100644 vendor/github.com/MakeNowJust/heredoc/README.md create mode 100644 vendor/github.com/MakeNowJust/heredoc/heredoc.go create mode 100644 vendor/sigs.k8s.io/cluster-api/util/yaml/yaml.go diff --git a/Makefile b/Makefile index 5fced97d6..09ad09331 100644 --- a/Makefile +++ b/Makefile @@ -310,7 +310,7 @@ test-integration: test-integration-workloadcluster test-integration-github .PHONY: test-unit test-unit: $(SETUP_ENVTEST) $(GOTESTSUM) $(HELM) ## Run unit @mkdir -p $(shell pwd)/.coverage - CREATE_KIND_CLUSTER=true KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GOTESTSUM) --junitfile=.coverage/junit.xml --format testname -- -mod=vendor \ + CREATE_KIND_CLUSTER=false KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GOTESTSUM) --junitfile=.coverage/junit.xml --format testname -- -mod=vendor \ -covermode=atomic -coverprofile=.coverage/cover.out -p=4 ./internal/controller/... .PHONY: test-integration-workloadcluster diff --git a/Tiltfile b/Tiltfile index 0fa796f59..2f884969b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -207,7 +207,7 @@ def deploy_cso(): "cso-serving-cert:certificate", "cso-cluster-stack-variables:secret", "cso-selfsigned-issuer:issuer", - #"cso-validating-webhook-configuration:validatingwebhookconfiguration", + "cso-validating-webhook-configuration:validatingwebhookconfiguration", ], new_name = "cso-misc", labels = ["CSO"], diff --git a/api/v1alpha1/cluster_webhook.go b/api/v1alpha1/cluster_webhook.go new file mode 100644 index 000000000..20511792a --- /dev/null +++ b/api/v1alpha1/cluster_webhook.go @@ -0,0 +1,154 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/clusterstack" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/release" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// Cluster implements a validating and defaulting webhook for Cluster. +// +k8s:deepcopy-gen=false +type Cluster struct { + Client client.Client +} + +// SetupWebhookWithManager initializes webhook manager for ClusterStack. +func (r *Cluster) SetupWebhookWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + return ctrl.NewWebhookManagedBy(mgr). + For(&clusterv1.Cluster{}). + WithValidator(r). + Complete() +} + +//+kubebuilder:webhook:path=/validate-cluster-x-k8s-io-v1beta1-cluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=cluster.x-k8s.io,resources=clusters,verbs=create;update,versions=v1beta1,name=validation.cluster.cluster.x-k8s.io,admissionReviewVersions={v1,v1beta1} + +var _ webhook.CustomValidator = &Cluster{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *Cluster) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + cluster, ok := obj.(*clusterv1.Cluster) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", obj)) + } + + return r.isVersionCorrect(ctx, cluster) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *Cluster) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + oldCluster, ok := oldObj.(*clusterv1.Cluster) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected an Cluster but got a %T", oldCluster)) + } + + newCluster, ok := newObj.(*clusterv1.Cluster) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected an Cluster but got a %T", newCluster)) + } + + var allErrs field.ErrorList + + warnings, err := r.isVersionCorrect(ctx, newCluster) + if len(warnings) > 0 || err != nil { + return warnings, err + } + + csOld, err := clusterstack.NewFromClusterClassProperties(oldCluster.Spec.Topology.Class) + if err != nil { + return nil, fmt.Errorf("expected a clusterclass of form --- but got %s: %w", oldCluster.Spec.Topology.Class, err) + } + + csNew, err := clusterstack.NewFromClusterClassProperties(newCluster.Spec.Topology.Class) + if err != nil { + return nil, fmt.Errorf("expected a clusterclass of form --- but got %s: %w", newCluster.Spec.Topology.Class, err) + } + + // provider must not change + if csOld.Provider != csNew.Provider { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "topology", "class"), newCluster.Spec.Topology.Class, fmt.Sprintf("provider name must not change. Got %s, want %s", csNew.Provider, csOld.Provider))) + } + + // cluster stack name must not change + if csOld.Name != csNew.Name { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "topology", "class"), newCluster.Spec.Topology.Class, fmt.Sprintf("cluster stack name must not change. Got %s, want %s", csNew.Name, csOld.Name))) + } + + // kubernetes version must be the same or higher by one + if csNew.KubernetesVersion.Minor-csOld.KubernetesVersion.Minor != 1 && csNew.KubernetesVersion.Minor-csOld.KubernetesVersion.Minor != 0 { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "topology", "class"), newCluster.Spec.Topology.Class, fmt.Sprintf("kubernetes version must be the same or higher by one. Got %s, want %s or 1-%d", csNew.KubernetesVersion, csOld.KubernetesVersion, csOld.KubernetesVersion.Minor+1))) + } + + return nil, aggregateObjErrors(oldCluster.GroupVersionKind().GroupKind(), oldCluster.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (*Cluster) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (r *Cluster) isVersionCorrect(ctx context.Context, cluster *clusterv1.Cluster) (admission.Warnings, error) { + if cluster.Spec.Topology == nil { + return nil, field.Invalid(field.NewPath("spec", "topology"), cluster.Spec.Topology, "topology field cannot be empty") + } + + if cluster.Spec.Topology.Class == "" { + return nil, field.Invalid(field.NewPath("spec", "topology", "class"), cluster.Spec.Topology.Class, "class field cannot be empty") + } + + wantKubernetesVersion, err := r.getClusterStackReleaseVersion(ctx, release.ConvertFromClusterClassToClusterStackFormat(cluster.Spec.Topology.Class), cluster.Namespace) + if err != nil { + return admission.Warnings{fmt.Sprintf("cannot validate clusterClass and Kubernetes version. Getting clusterStackRelease object failed: %s", err.Error())}, nil + } + + if wantKubernetesVersion == "" { + return admission.Warnings{fmt.Sprintf("no Kubernetes version set in status of clusterStackRelease object. Cannot validate Kubernetes version. Check out the ClusterStackReleaseObject %s/%s manually", cluster.Namespace, cluster.Spec.Topology.Class)}, nil + } + + if cluster.Spec.Topology.Version != wantKubernetesVersion { + return nil, field.Invalid(field.NewPath("spec", "topology", "version"), cluster.Spec.Topology.Version, fmt.Sprintf("clusterClass %s expects Kubernetes version %s, but got %s", cluster.Spec.Topology.Class, wantKubernetesVersion, cluster.Spec.Topology.Version)) + } + return nil, nil +} + +func (r *Cluster) getClusterStackReleaseVersion(ctx context.Context, name, namespace string) (string, error) { + clusterStackRelCR := &ClusterStackRelease{} + namespacedName := types.NamespacedName{Name: name, Namespace: namespace} + + if err := r.Client.Get(ctx, namespacedName, clusterStackRelCR); apierrors.IsNotFound(err) { + return "", fmt.Errorf("clusterclass does not exist: %w", err) + } else if err != nil { + return "", fmt.Errorf("failed to get ClusterStackRelease - cannot validate Kubernetes version: %w", err) + } + return clusterStackRelCR.Status.KubernetesVersion, nil +} diff --git a/api/v1alpha1/clusteraddon_webhook.go b/api/v1alpha1/clusteraddon_webhook.go new file mode 100644 index 000000000..3bf6295b1 --- /dev/null +++ b/api/v1alpha1/clusteraddon_webhook.go @@ -0,0 +1,102 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// SetupWebhookWithManager initializes webhook manager for ClusterAddon. +func (r *ClusterAddon) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// SetupWebhookWithManager initializes webhook manager for ClusterAddonList. +func (r *ClusterAddonList) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/validate-clusterstack-x-k8s-io-v1alpha1-clusteraddon,mutating=false,failurePolicy=fail,sideEffects=None,groups=clusterstack.x-k8s.io,resources=clusteraddons,verbs=create;update,versions=v1alpha1,name=validation.clusteraddon.clusterstack.x-k8s.io,admissionReviewVersions={v1,v1alpha1} + +var _ webhook.Validator = &ClusterAddon{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterAddon) ValidateCreate() (admission.Warnings, error) { + var allErrs field.ErrorList + + if r.Spec.ClusterRef == nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterRef"), r.Spec.ClusterRef, "must not be empty")) + } else if r.Spec.ClusterRef.Kind != "Cluster" { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterRef", "kind"), r.Spec.ClusterRef.Kind, "kind must be cluster")) + } + + return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterAddon) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + oldM, ok := old.(*ClusterAddon) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected an ClusterAddon but got a %T", old)) + } + + var allErrs field.ErrorList + + if r.Spec.ClusterRef == nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterRef"), r.Spec.ClusterRef, "must not be empty")) + return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) + } + + // clusterRef.Name is immutable + if oldM.Spec.ClusterRef.Name != r.Spec.ClusterRef.Name { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "clusterRef", "name"), r.Spec.ClusterRef.Name, "field is immutable"), + ) + } + + // namespace needs to always be the same for clusterAddon and cluster + if r.Spec.ClusterRef.Namespace != r.Namespace { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "clusterRef", "namespace"), r.Spec.ClusterRef.Namespace, "cluster and clusterAddon need to be in same namespace"), + ) + } + + // clusterRef.kind is immutable + if oldM.Spec.ClusterRef.Kind != r.Spec.ClusterRef.Kind { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "clusterRef", "kind"), r.Spec.ClusterRef.Kind, "field is immutable"), + ) + } + + return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (*ClusterAddon) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} diff --git a/api/v1alpha1/clusterstack_webhook.go b/api/v1alpha1/clusterstack_webhook.go new file mode 100644 index 000000000..bb9ecbb37 --- /dev/null +++ b/api/v1alpha1/clusterstack_webhook.go @@ -0,0 +1,193 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/clusterstack" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/version" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// ClusterStackWebhook implements validating and defaulting webhook for clusterstack. +// +k8s:deepcopy-gen=false +type ClusterStackWebhook struct { + LocalMode bool + client.Client +} + +// SetupWebhookWithManager initializes webhook manager for ClusterStack. +func (r *ClusterStackWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&ClusterStack{}). + WithValidator(r). + Complete() +} + +// SetupWebhookWithManager initializes webhook manager for ClusterStackList. +func (r *ClusterStackList) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:path=/validate-clusterstack-x-k8s-io-v1alpha1-clusterstack,mutating=false,failurePolicy=fail,sideEffects=None,groups=clusterstack.x-k8s.io,resources=clusterstacks,verbs=create;update;delete,versions=v1alpha1,name=validation.clusterstack.clusterstack.x-k8s.io,admissionReviewVersions={v1,v1alpha1} + +var _ webhook.CustomValidator = &ClusterStackWebhook{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterStackWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + clusterStack, ok := obj.(*ClusterStack) + if !ok { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", obj)) + } + + var allErrs field.ErrorList + + if r.LocalMode && clusterStack.Spec.AutoSubscribe { + field.Invalid(field.NewPath("spec", "autosubscribe"), clusterStack.Spec.AutoSubscribe, "can't autosubscribe in localMode") + } + + // validate versions and validate that versions match with the channel specified. + for _, v := range clusterStack.Spec.Versions { + if _, err := version.New(v); err != nil { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "versions"), clusterStack.Spec.Versions, fmt.Sprintf("invalid version: %s", err.Error())), + ) + } + } + + if clusterStack.Spec.ProviderRef == nil && !clusterStack.Spec.NoProvider { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "providerRef"), clusterStack.Spec.ProviderRef, "empty provider ref, even though noProvider mode is turned off"), + ) + } + + return nil, aggregateObjErrors(clusterStack.GroupVersionKind().GroupKind(), clusterStack.Name, allErrs) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterStackWebhook) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + oldClusterStack, ok := oldObj.(*ClusterStack) + if !ok { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("expected an ClusterStack but got a %T", oldObj)) + } + + newClusterStack, ok := newObj.(*ClusterStack) + if !ok { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("expected an ClusterStack but got a %T", newClusterStack)) + } + + var allErrs field.ErrorList + + if r.LocalMode && newClusterStack.Spec.AutoSubscribe { + field.Invalid(field.NewPath("spec", "autosubscribe"), newClusterStack.Spec.AutoSubscribe, "can't autosubscribe in localMode") + } + + // provider is immutable + if !reflect.DeepEqual(oldClusterStack.Spec.Provider, newClusterStack.Spec.Provider) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "provider"), newClusterStack.Spec.Provider, "field is immutable"), + ) + } + + // name is immutable + if !reflect.DeepEqual(oldClusterStack.Spec.Name, newClusterStack.Spec.Name) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "name"), newClusterStack.Spec.Name, "field is immutable"), + ) + } + + // channel is immutable + if !reflect.DeepEqual(oldClusterStack.Spec.Channel, newClusterStack.Spec.Channel) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "channel"), newClusterStack.Spec.Channel, "field is immutable"), + ) + } + + // KubernetesVersion is immutable + if !reflect.DeepEqual(oldClusterStack.Spec.KubernetesVersion, newClusterStack.Spec.KubernetesVersion) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "kubernetesVersion"), newClusterStack.Spec.KubernetesVersion, "field is immutable"), + ) + } + + return nil, aggregateObjErrors(newClusterStack.GroupVersionKind().GroupKind(), newClusterStack.Name, allErrs) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterStackWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + clusterStack, ok := obj.(*ClusterStack) + if !ok { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterStack but got a %T", obj)) + } + + clusters, err := r.getClustersUsingClusterStack(ctx, clusterStack) + if err != nil { + return admission.Warnings{fmt.Sprintf("cannot validate whether clusterstack is still in use. Watch out for potentially orphaned resources: %s", err.Error())}, nil + } + if len(clusters) > 0 { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("can't delete ClusterStack as there are clusters using it: [%q]", strings.Join(clusters, ", "))) + } + + return nil, nil +} + +func (r *ClusterStackWebhook) getClustersUsingClusterStack(ctx context.Context, clusterStack *ClusterStack) ([]string, error) { + clusterList := &clusterv1.ClusterList{} + if err := r.List(ctx, clusterList, &client.ListOptions{Namespace: clusterStack.Namespace}); err != nil { + return nil, fmt.Errorf("failed to list clusters: %w", err) + } + + clusters := make([]string, 0, len(clusterList.Items)) + + // list the names of all ClusterClasses that are referenced in Cluster objects + for i := range clusterList.Items { + if clusterList.Items[i].Spec.Topology == nil { + continue + } + + clusterClass := clusterList.Items[i].Spec.Topology.Class + if clusterClass == "" { + continue + } + + cs, err := clusterstack.NewFromClusterClassProperties(clusterClass) + if err != nil { + continue + } + if clusterStack.Spec.Provider == cs.Provider && + clusterStack.Spec.Channel == cs.Version.Channel && + clusterStack.Spec.Name == cs.Name && + clusterStack.Spec.KubernetesVersion == cs.KubernetesVersion.StringWithDot() { + clusters = append(clusters, clusterList.Items[i].Name) + } + } + + return clusters, nil +} diff --git a/api/v1alpha1/clusterstackrelease_webhook.go b/api/v1alpha1/clusterstackrelease_webhook.go new file mode 100644 index 000000000..eeee4df02 --- /dev/null +++ b/api/v1alpha1/clusterstackrelease_webhook.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + "strings" + + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/clusterstack" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// ClusterStackReleaseWebhook implements validating and defaulting webhook for ClusterStackRelease. +// +k8s:deepcopy-gen=false +type ClusterStackReleaseWebhook struct { + client.Client +} + +// SetupWebhookWithManager initializes webhook manager for ClusterStackRelease. +func (r *ClusterStackReleaseWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&ClusterStackRelease{}). + WithValidator(r). + Complete() +} + +// SetupWebhookWithManager initializes webhook manager for ClusterStackReleaseList. +func (r *ClusterStackReleaseList) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:path=/validate-clusterstack-x-k8s-io-v1alpha1-clusterstackrelease,mutating=false,failurePolicy=fail,sideEffects=None,groups=clusterstack.x-k8s.io,resources=clusterstackreleases,verbs=delete,versions=v1alpha1,name=validation.clusterstackrelease.clusterstack.x-k8s.io,admissionReviewVersions={v1,v1alpha1} + +var _ webhook.CustomValidator = &ClusterStackReleaseWebhook{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (*ClusterStackReleaseWebhook) ValidateCreate(_ context.Context, _ runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (*ClusterStackReleaseWebhook) ValidateUpdate(_ context.Context, _, _ runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *ClusterStackReleaseWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + clusterStackRelease, ok := obj.(*ClusterStackRelease) + if !ok { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterStackRelease but got a %T", obj)) + } + + clusters, err := r.getClustersUsingClusterStackRelease(ctx, clusterStackRelease) + if err != nil { + return admission.Warnings{fmt.Sprintf("cannot validate whether clusterstackrelease is still in use. Watch out for potentially orphaned resources: %s", err.Error())}, nil + } + if len(clusters) > 0 { + return admission.Warnings{}, apierrors.NewBadRequest(fmt.Sprintf("can't delete ClusterStackRelease as there are clusters using it: [%q]", strings.Join(clusters, ", "))) + } + + return nil, nil +} + +func (r *ClusterStackReleaseWebhook) getClustersUsingClusterStackRelease(ctx context.Context, clusterStackRelease *ClusterStackRelease) ([]string, error) { + clusterList := &clusterv1.ClusterList{} + if err := r.List(ctx, clusterList, &client.ListOptions{Namespace: clusterStackRelease.Namespace}); err != nil { + return nil, fmt.Errorf("failed to list clusters: %w", err) + } + + clusters := make([]string, 0, len(clusterList.Items)) + + // list the names of all ClusterClasses that are referenced in Cluster objects + for i := range clusterList.Items { + if clusterList.Items[i].Spec.Topology == nil { + continue + } + + clusterClass := clusterList.Items[i].Spec.Topology.Class + clusterClassFormated, err := clusterstack.NewFromClusterClassProperties(clusterClass) + if err != nil { + return nil, fmt.Errorf("failed to read properties from clusterClass string: %w", err) + } + clusterStackReleaseFormatted, err := clusterstack.NewFromClusterStackReleaseProperties(clusterStackRelease.Name) + if err != nil { + return nil, fmt.Errorf("failed to read properties from clusterStackRelease object name: %w", err) + } + + if clusterClassFormated.Name == clusterStackReleaseFormatted.Name && + clusterClassFormated.Provider == clusterStackReleaseFormatted.Provider && + clusterClassFormated.KubernetesVersion == clusterStackReleaseFormatted.KubernetesVersion && + clusterClassFormated.Version.String() == clusterStackReleaseFormatted.Version.String() { + clusters = append(clusters, clusterList.Items[i].Name) + } + } + + return clusters, nil +} diff --git a/api/v1alpha1/webhook.go b/api/v1alpha1/webhook.go new file mode 100644 index 000000000..c8662fb0e --- /dev/null +++ b/api/v1alpha1/webhook.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error { + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid( + gk, + name, + allErrs, + ) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a36c9528a..3ed1f4590 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) diff --git a/cmd/main.go b/cmd/main.go index 4b2374f34..4b11bd397 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" controllerruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) var ( @@ -104,8 +105,11 @@ func main() { } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - HealthProbeBindAddress: probeAddr, + Scheme: scheme, + HealthProbeBindAddress: probeAddr, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: 9443, + }), LeaderElection: enableLeaderElection, LeaderElectionID: "clusterstack.x-k8s.io", LeaderElectionNamespace: leaderElectionNamespace, @@ -187,6 +191,7 @@ func main() { os.Exit(1) } + setUpWebhookWithManager(mgr) //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { @@ -208,3 +213,25 @@ func main() { // Wait for all target cluster managers to gracefully shut down. wg.Wait() } + +func setUpWebhookWithManager(mgr ctrl.Manager) { + if err := (&csov1alpha1.ClusterStackWebhook{ + LocalMode: localMode, + Client: mgr.GetClient(), + }).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "ClusterStack") + os.Exit(1) + } + if err := (&csov1alpha1.ClusterAddon{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "ClusterAddon") + os.Exit(1) + } + if err := (&csov1alpha1.Cluster{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Cluster") + os.Exit(1) + } + if err := (&csov1alpha1.ClusterStackReleaseWebhook{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "ClusterStackRelease") + os.Exit(1) + } +} diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 835ccdd63..e54777540 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -3,41 +3,44 @@ namespace: cso-system namePrefix: cso- commonLabels: - cluster.x-k8s.io/provider: "infrastructure-cluster-stack-operator" + cluster.x-k8s.io/provider: "cluster-stack-operator" resources: - ../crd - ../rbac - ../manager + - ../webhook - ../certmanager patchesStrategicMerge: - manager_config_patch.yaml + - manager_webhook_patch.yaml + - webhookcainjection_patch.yaml - manager_pull_policy.yaml -# vars: -# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -# - name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# - name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -# - name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/localmode/kustomization.yaml b/config/localmode/kustomization.yaml index 835ccdd63..e54777540 100644 --- a/config/localmode/kustomization.yaml +++ b/config/localmode/kustomization.yaml @@ -3,41 +3,44 @@ namespace: cso-system namePrefix: cso- commonLabels: - cluster.x-k8s.io/provider: "infrastructure-cluster-stack-operator" + cluster.x-k8s.io/provider: "cluster-stack-operator" resources: - ../crd - ../rbac - ../manager + - ../webhook - ../certmanager patchesStrategicMerge: - manager_config_patch.yaml + - manager_webhook_patch.yaml + - webhookcainjection_patch.yaml - manager_pull_policy.yaml -# vars: -# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -# - name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# - name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -# - name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 27c9f2df6..5864535c0 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -25,6 +25,27 @@ webhooks: resources: - clusters sideEffects: None +- admissionReviewVersions: + - v1 + - v1alpha1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-clusterstack-x-k8s-io-v1alpha1-clusteraddon + failurePolicy: Fail + name: validation.clusteraddon.clusterstack.x-k8s.io + rules: + - apiGroups: + - clusterstack.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - clusteraddons + sideEffects: None - admissionReviewVersions: - v1 - v1alpha1 @@ -43,6 +64,27 @@ webhooks: operations: - CREATE - UPDATE + - DELETE resources: - clusterstacks sideEffects: None +- admissionReviewVersions: + - v1 + - v1alpha1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-clusterstack-x-k8s-io-v1alpha1-clusterstackrelease + failurePolicy: Fail + name: validation.clusterstackrelease.clusterstack.x-k8s.io + rules: + - apiGroups: + - clusterstack.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - DELETE + resources: + - clusterstackreleases + sideEffects: None diff --git a/go.mod b/go.mod index 35f5b35af..942218e31 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( require ( github.com/BurntSushi/toml v1.0.0 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/internal/controller/clusteraddon_controller_test.go b/internal/controller/clusteraddon_controller_test.go new file mode 100644 index 000000000..9525194cf --- /dev/null +++ b/internal/controller/clusteraddon_controller_test.go @@ -0,0 +1,463 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" +) + +var _ = Describe("ClusterAddonReconciler", func() { + var ( + cluster *clusterv1.Cluster + clusterStackRelease *csov1alpha1.ClusterStackRelease + + testNs *corev1.Namespace + key types.NamespacedName + ) + + const clusterAddonName = "cluster-addon-testcluster" + + BeforeEach(func() { + var err error + testNs, err = testEnv.CreateNamespace(ctx, "clusteraddon-reconciler") + Expect(err).NotTo(HaveOccurred()) + + key = types.NamespacedName{Name: clusterAddonName, Namespace: testNs.Name} + + cluster = &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testcluster", + Namespace: testNs.Name, + }, + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Class: testClusterStackName, + Version: testKubernetesVersion, + }, + }, + } + + testEnv.KubeClient.On("Apply", mock.Anything, mock.Anything, mock.Anything).Return([]*csov1alpha1.Resource{}, false, nil) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, testNs, cluster, clusterStackRelease) + }, timeout, interval).Should(BeNil()) + }) + + Context("Basic test", func() { + var clusterStackReleaseKey types.NamespacedName + + BeforeEach(func() { + clusterStackReleaseKey = types.NamespacedName{Name: testClusterStackName, Namespace: testNs.Name} + + clusterStackRelease = &csov1alpha1.ClusterStackRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: testClusterStackName, + Namespace: testNs.Name, + }, + } + Expect(testEnv.Create(ctx, clusterStackRelease)).To(Succeed()) + + Eventually(func() bool { + var foundClusterStackRelease csov1alpha1.ClusterStackRelease + if err := testEnv.Get(ctx, clusterStackReleaseKey, &foundClusterStackRelease); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + + It("creates the clusterAddon object", func() { + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + if foundClusterAddon.Spec.ClusterRef.Name != cluster.Name { + testEnv.GetLogger().Info("wrong cluster ref name", "got", foundClusterAddon.Spec.ClusterRef.Name, "want", cluster.Name) + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + + It("sets ClusterReady condition if cluster has ControlPlaneReadyCondition", func() { + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() bool { + if err := ph.Patch(ctx, cluster); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + if foundClusterAddon.Spec.ClusterRef.Name != cluster.Name { + testEnv.GetLogger().Info("wrong cluster ref name", "got", foundClusterAddon.Spec.ClusterRef.Name, "want", cluster.Name) + return false + } + + return utils.IsPresentAndTrue(ctx, testEnv.Client, key, &foundClusterAddon, csov1alpha1.ClusterReadyCondition) + }, timeout, interval).Should(BeTrue()) + }) + + It("updates the clusteraddon helm chart objects if cluster switches to new cluster stack with new clusteraddon version", func() { + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() bool { + if err := ph.Patch(ctx, cluster); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + By("updating the clusterclass in the cluster") + + ph, err = patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + cluster.Spec.Topology.Class = testClusterStackNameV2 + + Eventually(func() bool { + if err := ph.Patch(ctx, cluster); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + By("ensuring that the spec of clusteraddon is updated and status still synced") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + if foundClusterAddon.Spec.ClusterStack != testClusterStackNameV2 { + testEnv.GetLogger().Info("found wrong cluster stack", "want", testClusterStackNameV2, "got", foundClusterAddon.Spec.ClusterStack) + return false + } + + if foundClusterAddon.Spec.Version != "v2" { + testEnv.GetLogger().Info("found wrong cluster addon version", "want", "v2", "got", foundClusterAddon.Spec.Version) + return false + } + + return true + }, timeout, interval).Should(BeTrue()) + }) + + It("should call update if the cluster addon version changes in the ClusterClass update", func() { + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + By("making the control plane ready") + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() error { + return ph.Patch(ctx, cluster) + }, timeout, interval).Should(BeNil()) + + By("ensuring helm chart is applied") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + return utils.IsPresentAndTrue(ctx, testEnv.GetClient(), key, &foundClusterAddon, csov1alpha1.HelmChartAppliedCondition) + }, timeout, interval).Should(BeTrue()) + + By("updating the cluster class") + ph, err = patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + cluster.Spec.Topology.Class = testClusterStackNameV2 + + Eventually(func() error { + return ph.Patch(ctx, cluster) + }, timeout, interval).Should(BeNil()) + + By("ensuring cluster addon and cluster stack version is updated") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + return foundClusterAddon.Spec.Version == "v2" && foundClusterAddon.Spec.ClusterStack == "docker-ferrol-1-27-v2" + }, timeout, interval).Should(BeTrue()) + + By("checking Update method was called") + Expect(testEnv.KubeClient.AssertCalled(GinkgoT(), "Apply", mock.Anything, mock.Anything, mock.Anything)).To(BeTrue()) + }) + + It("should not call update if the ClusterAddon version does not change in the ClusterClass update", func() { + cluster.Spec.Topology.Class = testClusterStackNameV2 + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + By("making the control plane ready") + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() error { + return ph.Patch(ctx, cluster) + }, timeout, interval).Should(BeNil()) + + By("ensuring helm chart is applied") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + return utils.IsPresentAndTrue(ctx, testEnv.GetClient(), key, &foundClusterAddon, csov1alpha1.HelmChartAppliedCondition) + }, timeout, interval).Should(BeTrue()) + + By("updating the cluster class") + ph, err = patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + cluster.Spec.Topology.Class = testClusterStackNameV3 + + Eventually(func() error { + return ph.Patch(ctx, cluster) + }, timeout, interval).Should(BeNil()) + + By("ensuring cluster addon and cluster stack version is updated") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + return foundClusterAddon.Spec.Version == "v2" && foundClusterAddon.Spec.ClusterStack == "docker-ferrol-1-27-v3" + }, timeout, interval).Should(BeTrue()) + + By("checking Update method was not called") + Expect(testEnv.KubeClient.AssertNotCalled(GinkgoT(), "Apply")).To(BeTrue()) + }) + + It("check that specs are set accordingly after helmchart has been applied", func() { + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + By("making the control plane ready") + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() error { + return ph.Patch(ctx, cluster) + }, timeout, interval).Should(BeNil()) + + By("ensuring helm chart is applied and specs are set") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + return utils.IsPresentAndTrue(ctx, testEnv.GetClient(), key, &foundClusterAddon, csov1alpha1.HelmChartAppliedCondition) && + foundClusterAddon.Status.Ready && foundClusterAddon.Spec.ClusterStack == testClusterStackName + }, timeout, interval).Should(BeTrue()) + }) + }) + + Context("Update test", func() { + var ( + clusterStackRelease *csov1alpha1.ClusterStackRelease + clusterStackReleaseKey types.NamespacedName + ) + + BeforeEach(func() { + clusterStackReleaseKey = types.NamespacedName{Name: testClusterStackNameV2, Namespace: testNs.Name} + + clusterStackRelease = &csov1alpha1.ClusterStackRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: testClusterStackNameV2, + Namespace: testNs.Name, + }, + } + Expect(testEnv.Create(ctx, clusterStackRelease)).To(Succeed()) + + Eventually(func() bool { + var foundClusterStackRelease csov1alpha1.ClusterStackRelease + if err := testEnv.Get(ctx, clusterStackReleaseKey, &foundClusterStackRelease); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, clusterStackRelease) + }, timeout, interval).Should(BeNil()) + }) + + It("does not update the clusteraddon helm chart objects if cluster switches to new cluster stack without new clusteraddon version", func() { + cluster.Spec.Topology.Class = testClusterStackNameV2 + Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + + ph, err := patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + conditions.MarkTrue(cluster, clusterv1.ControlPlaneReadyCondition) + + Eventually(func() bool { + if err := ph.Patch(ctx, cluster); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + By("updating the clusterclass in the cluster") + + ph, err = patch.NewHelper(cluster, testEnv) + Expect(err).ShouldNot(HaveOccurred()) + + cluster.Spec.Topology.Class = testClusterStackNameV3 + + Eventually(func() bool { + if err := ph.Patch(ctx, cluster); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + By("ensuring that the spec of clusteraddon is updated and status still synced") + Eventually(func() bool { + var foundClusterAddon csov1alpha1.ClusterAddon + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + testEnv.GetLogger().Info(err.Error()) + return false + } + + if foundClusterAddon.Spec.ClusterStack != testClusterStackNameV3 { + testEnv.GetLogger().Info("found wrong cluster stack", "want", testClusterStackNameV3, "got", foundClusterAddon.Spec.ClusterStack) + return false + } + + if foundClusterAddon.Spec.Version != "v2" { + testEnv.GetLogger().Info("found wrong cluster addon version", "want", "v2", "got", foundClusterAddon.Spec.Version) + return false + } + + return true + }, timeout, interval).Should(BeTrue()) + + // TODO: Check that mocked function has not been called + }) + }) +}) + +var _ = Describe("ClusterAddon validation", func() { + var ( + clusteraddon *csov1alpha1.ClusterAddon + testNs *corev1.Namespace + key types.NamespacedName + ) + + const clusterAddonName = "cluster-addon-testcluster" + + BeforeEach(func() { + var err error + testNs, err = testEnv.CreateNamespace(ctx, "clusteraddon-validation") + Expect(err).NotTo(HaveOccurred()) + + key = types.NamespacedName{Namespace: testNs.Name, Name: clusterAddonName} + + clusteraddon = &csov1alpha1.ClusterAddon{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterAddonName, + Namespace: testNs.Name, + }, + Spec: csov1alpha1.ClusterAddonSpec{ + ClusterRef: &corev1.ObjectReference{ + Name: "test", + Kind: "Cluster", + }, + }, + } + Expect(testEnv.Create(ctx, clusteraddon)).To(Succeed()) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, testNs, clusteraddon) + }, timeout, interval).Should(BeNil()) + }) + + Context("validate update", func() { + It("Should not allow an update of clusterAddon.Spec.ClusterRef", func() { + var foundClusterAddon csov1alpha1.ClusterAddon + Eventually(func() bool { + if err := testEnv.Get(ctx, key, &foundClusterAddon); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + foundClusterAddon.Spec.ClusterRef.Name = "test2" + Expect(testEnv.Update(ctx, &foundClusterAddon)).NotTo(Succeed()) + }) + }) +}) diff --git a/internal/controller/clusteraddoncreate_controller_test.go b/internal/controller/clusteraddoncreate_controller_test.go index 614a906f0..490e93305 100644 --- a/internal/controller/clusteraddoncreate_controller_test.go +++ b/internal/controller/clusteraddoncreate_controller_test.go @@ -22,6 +22,7 @@ import ( csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -32,22 +33,16 @@ var _ = Describe("ClusterAddonCreateReconciler", func() { var ( cluster *clusterv1.Cluster clusterStackRelease *csov1alpha1.ClusterStackRelease - testNs *corev1.Namespace - key types.NamespacedName + + testNs *corev1.Namespace + key types.NamespacedName ) BeforeEach(func() { var err error testNs, err = testEnv.CreateNamespace(ctx, "clusteraddoncreate-reconciler") Expect(err).NotTo(HaveOccurred()) - - clusterStackRelease = &csov1alpha1.ClusterStackRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: testClusterStackName, - Namespace: testNs.Name, - }, - } - Expect(testEnv.Create(ctx, clusterStackRelease)).To(Succeed()) + testEnv.GetLogger().Info("Namespace", "name", testNs.Name, "namespace", testNs.Namespace) cluster = &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ @@ -57,17 +52,30 @@ var _ = Describe("ClusterAddonCreateReconciler", func() { }, Spec: clusterv1.ClusterSpec{ Topology: &clusterv1.Topology{ - Class: testClusterStackName, + Class: testClusterStackName, + Version: testKubernetesVersion, }, }, } Expect(testEnv.Create(ctx, cluster)).To(Succeed()) + clusterStackRelease = &csov1alpha1.ClusterStackRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: testClusterStackNameV2, + Namespace: testNs.Name, + }, + } + Expect(testEnv.Create(ctx, clusterStackRelease)).To(Succeed()) + key = types.NamespacedName{Name: fmt.Sprintf("cluster-addon-%s", cluster.Name), Namespace: testNs.Name} + + testEnv.KubeClient.On("Apply", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*csov1alpha1.Resource{}, false, nil) }) AfterEach(func() { - Expect(testEnv.Cleanup(ctx, testNs, cluster, clusterStackRelease)).To(Succeed()) + Eventually(func() error { + return testEnv.Cleanup(ctx, testNs, cluster, clusterStackRelease) + }, timeout, interval).Should(BeNil()) }) Context("Basic test", func() { diff --git a/internal/controller/clusterstack_controller_test.go b/internal/controller/clusterstack_controller_test.go index ff33a1615..a6fcca3e7 100644 --- a/internal/controller/clusterstack_controller_test.go +++ b/internal/controller/clusterstack_controller_test.go @@ -23,6 +23,7 @@ import ( csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/clusterstack" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/version" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -31,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/external" "sigs.k8s.io/cluster-api/util/patch" ) @@ -650,3 +652,223 @@ var _ = Describe("ClusterStackReconciler", func() { }) }) }) + +var _ = Describe("clusterStack validation", func() { + var testNs *corev1.Namespace + + BeforeEach(func() { + var err error + testNs, err = testEnv.CreateNamespace(ctx, "clusterstack-validation") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, testNs) + }, timeout, interval).Should(BeNil()) + }) + + Context("validate create", func() { + var clusterStack *csov1alpha1.ClusterStack + + BeforeEach(func() { + clusterStack = &csov1alpha1.ClusterStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-stack", + Namespace: testNs.Name, + }, + Spec: csov1alpha1.ClusterStackSpec{ + Provider: "docker", + Name: "testclusterstack", + KubernetesVersion: "1.27", + NoProvider: true, + AutoSubscribe: false, + ProviderRef: &corev1.ObjectReference{ + APIVersion: "infrastructure.clusterstack.x-k8s.io/v1alpha1", + Kind: "DockerClusterStackReleaseTemplate", + Name: "mytemplate", + Namespace: testNs.Name, + }, + }, + } + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, clusterStack) + }, timeout, interval).Should(BeNil()) + }) + + It("should succeed with correct spec", func() { + Expect(testEnv.Create(ctx, clusterStack)).To(Succeed()) + }) + + It("should succeed with no provider", func() { + clusterStack.Spec.ProviderRef = nil + Expect(testEnv.Create(ctx, clusterStack)).To(Succeed()) + }) + + It("should fail with a wrong version tag", func() { + clusterStack.Spec.Versions = append(clusterStack.Spec.Versions, "v1-alpha") + Expect(testEnv.Create(ctx, clusterStack)).NotTo(Succeed()) + }) + + It("should fail with empty provider", func() { + clusterStack.Spec.Provider = "" + Expect(testEnv.Create(ctx, clusterStack)).NotTo(Succeed()) + }) + + It("should fail with empty clusterStack name", func() { + clusterStack.Spec.Name = "" + Expect(testEnv.Create(ctx, clusterStack)).NotTo(Succeed()) + }) + + It("providerRef is nil and no provider is false", func() { + clusterStack.Spec.ProviderRef = nil + clusterStack.Spec.NoProvider = false + Expect(testEnv.Create(ctx, clusterStack)).ToNot(Succeed()) + }) + }) + + Context("validate update", func() { + var ( + clusterStack *csov1alpha1.ClusterStack + key types.NamespacedName + foundClusterStack csov1alpha1.ClusterStack + ) + + BeforeEach(func() { + key = types.NamespacedName{Namespace: testNs.Name, Name: "cluster-stack"} + clusterStack = &csov1alpha1.ClusterStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-stack", + Namespace: testNs.Name, + }, + Spec: csov1alpha1.ClusterStackSpec{ + Provider: "docker", + Name: "testclusterstack", + Channel: version.ChannelStable, + KubernetesVersion: "1.27", + NoProvider: true, + AutoSubscribe: false, + }, + } + Expect(testEnv.Create(ctx, clusterStack)).To(Succeed()) + + Eventually(func() bool { + if err := testEnv.Get(ctx, key, &foundClusterStack); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, clusterStack) + }, timeout, interval).Should(BeNil()) + }) + + It("should have correct specs", func() { + Expect(foundClusterStack.Spec.AutoSubscribe).To(Equal(false)) + Expect(foundClusterStack.Spec.NoProvider).To(Equal(true)) + Expect(foundClusterStack.Spec.Provider).To(Equal("docker")) + Expect(foundClusterStack.Spec.Name).To(Equal("testclusterstack")) + Expect(foundClusterStack.Spec.KubernetesVersion).To(Equal("1.27")) + Expect(foundClusterStack.Spec.Channel).To(Equal(version.ChannelStable)) + }) + + It("Should not allow an update of ClusterStack.Spec.Provider", func() { + foundClusterStack.Spec.Provider = "otherprovider" + Expect(testEnv.Update(ctx, &foundClusterStack)).NotTo(Succeed()) + }) + + It("Should not allow an update of ClusterStack.Spec.Name", func() { + foundClusterStack.Spec.Name = "testclusterstack2" + Expect(testEnv.Update(ctx, &foundClusterStack)).NotTo(Succeed()) + }) + + It("Should not allow an update of ClusterStack.Spec.Channel", func() { + foundClusterStack.Spec.Channel = version.ChannelCustom + Expect(testEnv.Update(ctx, &foundClusterStack)).NotTo(Succeed()) + }) + + It("Should not allow an update of ClusterStack.Spec.KubernetesVersion", func() { + foundClusterStack.Spec.KubernetesVersion = "1.25" + Expect(testEnv.Update(ctx, &foundClusterStack)).NotTo(Succeed()) + }) + }) + + Context("validate delete", func() { + var ( + clusterStack *csov1alpha1.ClusterStack + key types.NamespacedName + foundClusterStack csov1alpha1.ClusterStack + cluster clusterv1.Cluster + ) + + BeforeEach(func() { + key = types.NamespacedName{Namespace: testNs.Name, Name: "cluster-stack"} + By("creating clusterstack") + clusterStack = &csov1alpha1.ClusterStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-stack", + Namespace: testNs.Name, + Finalizers: []string{clusterv1.ClusterFinalizer}, + }, + Spec: csov1alpha1.ClusterStackSpec{ + Provider: "docker", + Name: "ferrol", + Channel: version.ChannelStable, + KubernetesVersion: "1.27", + NoProvider: true, + AutoSubscribe: false, + }, + } + Expect(testEnv.Create(ctx, clusterStack)).To(Succeed()) + + By("checking if clusterstack is created properly") + Eventually(func() error { + return testEnv.Get(ctx, key, &foundClusterStack) + }, timeout, interval).Should(BeNil()) + + By("creating cluster") + cluster = clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: testNs.Name, + }, + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Class: "docker-ferrol-1-27-v6", + }, + }, + } + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, &cluster, clusterStack) + }, timeout, interval).Should(BeNil()) + }) + + It("should not allow delete if ClusterStack is in use by Cluster", func() { + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + Expect(testEnv.Delete(ctx, clusterStack)).ToNot(Succeed()) + }) + + It("should allow delete if existing Clusters reference ClusterClasses that do not follow the cluster stack naming convention", func() { + cluster.Spec.Topology.Class = "test-cluster-class" + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + + Expect(testEnv.Delete(ctx, clusterStack)).To(Succeed()) + }) + + It("should allow delete if existing Clusters reference different ClusterClasses", func() { + cluster.Spec.Topology.Class = "docker-ferrol-1-25-v1" + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + + Expect(testEnv.Delete(ctx, clusterStack)).To(Succeed()) + }) + }) +}) diff --git a/internal/controller/clusterstackrelease_controller_test.go b/internal/controller/clusterstackrelease_controller_test.go index 8e60f235a..70c5ab06e 100644 --- a/internal/controller/clusterstackrelease_controller_test.go +++ b/internal/controller/clusterstackrelease_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/SovereignCloudStack/cluster-stack-operator/pkg/test/utils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -256,3 +257,86 @@ var _ = Describe("ClusterStackReleaseReconciler", func() { }) }) }) + +var _ = Describe("ClusterStackRelease validation", func() { + var testNs *corev1.Namespace + + BeforeEach(func() { + var err error + testNs, err = testEnv.CreateNamespace(ctx, "clusterstack-validation") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, testNs) + }, timeout, interval).Should(BeNil()) + }) + + Context("validate delete", func() { + var ( + clusterStackRelease *csov1alpha1.ClusterStackRelease + key types.NamespacedName + foundClusterStackRelease csov1alpha1.ClusterStackRelease + cluster clusterv1.Cluster + ) + + BeforeEach(func() { + key = types.NamespacedName{Namespace: testNs.Name, Name: "docker-ferrol-1-27-v1"} + By("creating ClusterStackRelease") + clusterStackRelease = &csov1alpha1.ClusterStackRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "docker-ferrol-1-27-v1", + Namespace: testNs.Name, + }, + } + Expect(testEnv.Create(ctx, clusterStackRelease)).To(Succeed()) + + By("checking if ClusterStackRelease is created properly") + Eventually(func() error { + return testEnv.Get(ctx, key, &foundClusterStackRelease) + }, timeout, interval).Should(BeNil()) + + By("creating cluster") + cluster = clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: testNs.Name, + }, + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Class: "docker-ferrol-1-27-v1", + Version: "v1.27.3", + }, + }, + } + + testEnv.KubeClient.On("Apply", mock.Anything, mock.Anything, mock.Anything).Return([]*csov1alpha1.Resource{}, false, nil) + }) + + AfterEach(func() { + Eventually(func() error { + return testEnv.Cleanup(ctx, &cluster, clusterStackRelease) + }, timeout, interval).Should(BeNil()) + }) + + It("should not allow delete if ClusterStackRelease is in use by Cluster", func() { + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + Expect(testEnv.Delete(ctx, clusterStackRelease)).ToNot(Succeed()) + }) + + It("should allow delete if existing Clusters reference ClusterClasses that do not follow the cluster stack naming convention", func() { + cluster.Spec.Topology.Class = "test-cluster-class" + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + + Expect(testEnv.Delete(ctx, clusterStackRelease)).To(Succeed()) + }) + + It("should allow delete if existing Clusters reference different ClusterClasses", func() { + cluster.Spec.Topology.Class = "docker-ferrol-1-26-v5" + Expect(testEnv.Create(ctx, &cluster)).To(Succeed()) + + Expect(testEnv.Delete(ctx, clusterStackRelease)).To(Succeed()) + }) + }) +}) diff --git a/internal/controller/controller_suite_test.go b/internal/controller/controller_suite_test.go index ccd8340dd..1c4804cad 100644 --- a/internal/controller/controller_suite_test.go +++ b/internal/controller/controller_suite_test.go @@ -22,6 +22,7 @@ import ( "github.com/SovereignCloudStack/cluster-stack-operator/internal/test/helpers" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" + fakeworkloadcluster "github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster/fake" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ctrl "sigs.k8s.io/controller-runtime" @@ -34,7 +35,11 @@ const ( ) const ( - testClusterStackName = "docker-ferrol-1-27-v1" + testClusterStackName = "docker-ferrol-1-27-v1" + testClusterStackNameV2 = "docker-ferrol-1-27-v2" + testClusterStackNameV3 = "docker-ferrol-1-27-v3" + + testKubernetesVersion = "v1.27.3" ) func TestControllers(t *testing.T) { @@ -64,6 +69,13 @@ var _ = BeforeSuite(func() { ReleaseDirectory: "./../../test/releases", }).SetupWithManager(ctx, testEnv.Manager, c.Options{})).To(Succeed()) + Expect((&ClusterAddonReconciler{ + Client: testEnv.Manager.GetClient(), + ReleaseDirectory: "./../../test/releases", + KubeClientFactory: testEnv.KubeClientFactory, + WorkloadClusterFactory: fakeworkloadcluster.NewFactory(), + }).SetupWithManager(ctx, testEnv.Manager, c.Options{})).To(Succeed()) + Expect((&ClusterAddonCreateReconciler{ Client: testEnv.Manager.GetClient(), }).SetupWithManager(ctx, testEnv.Manager, c.Options{})).To(Succeed()) @@ -74,6 +86,8 @@ var _ = BeforeSuite(func() { }() <-testEnv.Manager.Elected() + // wait for webhook port to be open prior to running tests + testEnv.WaitForWebhooks() }) var _ = AfterSuite(func() { diff --git a/internal/test/helpers/envtest.go b/internal/test/helpers/envtest.go index 046aefef4..45ca8d45e 100644 --- a/internal/test/helpers/envtest.go +++ b/internal/test/helpers/envtest.go @@ -33,6 +33,8 @@ import ( "github.com/SovereignCloudStack/cluster-stack-operator/internal/test/helpers/builder" githubclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client" githubmocks "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client/mocks" + kubeclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" + kubemocks "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube/mocks" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/test/utils" g "github.com/onsi/ginkgo/v2" corev1 "k8s.io/api/core/v1" @@ -146,12 +148,17 @@ type ( kind *kind.Provider WorkloadClusterClient *kubernetes.Clientset GitHubClientFactory githubclient.Factory + KubeClientFactory kubeclient.Factory GitHubClient *githubmocks.Client + KubeClient *kubemocks.Client } ) // NewTestEnvironment creates a new environment spinning up a local api-server. func NewTestEnvironment() *TestEnvironment { + // initialize webhook here to be able to test the envtest install via webhookOptions + initializeWebhookInEnvironment() + config, err := env.Start() if err != nil { klog.Fatalf("unable to start env: %s", err) @@ -172,6 +179,19 @@ func NewTestEnvironment() *TestEnvironment { klog.Fatalf("unable to create manager: %s", err) } + if err := (&csov1alpha1.ClusterStackWebhook{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil { + klog.Fatalf("failed to set up webhook with manager for ClusterStack: %s", err) + } + if err := (&csov1alpha1.ClusterAddon{}).SetupWebhookWithManager(mgr); err != nil { + klog.Fatalf("failed to set up webhook with manager for ClusterAddon: %s", err) + } + if err := (&csov1alpha1.Cluster{}).SetupWebhookWithManager(mgr); err != nil { + klog.Fatalf("failed to set up webhook with manager for Cluster: %s", err) + } + if err := (&csov1alpha1.ClusterStackReleaseWebhook{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil { + klog.Fatalf("failed to set up webhook with manager for ClusterStackRelease: %s", err) + } + // create manager pod namespace ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -184,13 +204,16 @@ func NewTestEnvironment() *TestEnvironment { } githubClient := &githubmocks.Client{} + kubeClient := &kubemocks.Client{} testEnv := &TestEnvironment{ Manager: mgr, Client: mgr.GetClient(), Config: mgr.GetConfig(), GitHubClientFactory: githubmocks.NewGitHubFactory(githubClient), + KubeClientFactory: kubemocks.NewKubeFactory(kubeClient), GitHubClient: githubClient, + KubeClient: kubeClient, } if ifCreateKind() { diff --git a/internal/test/helpers/webhook.go b/internal/test/helpers/webhook.go new file mode 100644 index 000000000..35f84b2c9 --- /dev/null +++ b/internal/test/helpers/webhook.go @@ -0,0 +1,127 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helpers + +import ( + "fmt" + "net" + "os" + "path" + "path/filepath" + goruntime "runtime" + "strconv" + "time" + + v1 "k8s.io/api/admissionregistration/v1" + "k8s.io/klog/v2" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const ( + mutatingWebhookKind = "MutatingWebhookConfiguration" + validatingWebhookKind = "ValidatingWebhookConfiguration" + mutatingwebhook = "mutating-webhook-configuration" + validatingwebhook = "validating-webhook-configuration" +) + +// Mutate the name of each webhook, because kubebuilder generates the same name for all controllers. +// In normal usage, kustomize will prefix the controller name, which we have to do manually here. +func appendWebhookConfiguration(configyamlFile []byte, tag string) ([]*v1.MutatingWebhookConfiguration, []*v1.ValidatingWebhookConfiguration, error) { + var mutatingWebhooks []*v1.MutatingWebhookConfiguration + var validatingWebhooks []*v1.ValidatingWebhookConfiguration + objs, err := utilyaml.ToUnstructured(configyamlFile) + if err != nil { + klog.Fatalf("failed to parse yaml") + } + // look for resources of kind MutatingWebhookConfiguration + for i := range objs { + o := objs[i] + if o.GetKind() == mutatingWebhookKind { + // update the name in metadata + if o.GetName() == mutatingwebhook { + var m v1.MutatingWebhookConfiguration + o.SetName(mutatingwebhook + "-" + tag) + if err := scheme.Convert(&o, &m, nil); err != nil { + return nil, nil, fmt.Errorf("failed to convert scheme: %w", err) + } + mutatingWebhooks = append(mutatingWebhooks, &m) + } + } + if o.GetKind() == validatingWebhookKind { + // update the name in metadata + if o.GetName() == validatingwebhook { + var v v1.ValidatingWebhookConfiguration + o.SetName(validatingwebhook + "-" + tag) + if err := scheme.Convert(&o, &v, nil); err != nil { + return nil, nil, fmt.Errorf("failed to convert scheme: %w", err) + } + validatingWebhooks = append(validatingWebhooks, &v) + } + } + } + return mutatingWebhooks, validatingWebhooks, nil +} + +func initializeWebhookInEnvironment() { + // Get the root of the current file to use in CRD paths. + _, filename, _, _ := goruntime.Caller(0) + root := path.Join(path.Dir(filename), "..", "..", "..") + corepath := filepath.Join(root, "config", "webhook", "manifests.yaml") + configyamlFile, err := os.ReadFile(corepath) //#nosec + if err != nil { + klog.Fatalf("Failed to read core webhook configuration file: %v", err) + } + if err != nil { + klog.Fatalf("failed to parse yaml") + } + // append the webhook with suffix to avoid clashing webhooks. repeated for every webhook + mutatingWebhooks, validatingWebhooks, err := appendWebhookConfiguration(configyamlFile, "config") + if err != nil { + klog.Fatalf("Failed to append core controller webhook config: %v", err) + } + + env.WebhookInstallOptions = envtest.WebhookInstallOptions{ + LocalServingPort: 9443, + LocalServingHost: "localhost", + MaxTime: 20 * time.Second, + PollInterval: time.Second, + ValidatingWebhooks: validatingWebhooks, + MutatingWebhooks: mutatingWebhooks, + } +} + +// WaitForWebhooks waits for webhook port to be ready. +func (*TestEnvironment) WaitForWebhooks() { + port := env.WebhookInstallOptions.LocalServingPort + + klog.V(2).Infof("Waiting for webhook port %d to be open prior to running tests", port) + timeout := 1 * time.Second + for { + time.Sleep(1 * time.Second) + conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), timeout) + if err != nil { + klog.V(2).Infof("Webhook port is not ready, will retry in %v: %s", timeout, err) + continue + } + if err := conn.Close(); err != nil { + klog.Fatalf("failed to close connection: %s", err) + } + klog.V(2).Info("Webhook port is now open. Continuing with tests...") + return + } +} diff --git a/pkg/kube/mocks/KFactory.go b/pkg/kube/mocks/KFactory.go new file mode 100644 index 000000000..67fa03a22 --- /dev/null +++ b/pkg/kube/mocks/KFactory.go @@ -0,0 +1,22 @@ +// Package mocks implement important mocking interface of kube. +package mocks + +import ( + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" + "k8s.io/client-go/rest" +) + +type kubeFactory struct { + client *Client +} + +// NewKubeFactory returns packer factory interface. +func NewKubeFactory(client *Client) kube.Factory { + return &kubeFactory{client: client} +} + +var _ = kube.Factory(&kubeFactory{}) + +func (f *kubeFactory) NewClient(_ string, _ *rest.Config) kube.Client { + return f.client +} diff --git a/pkg/workloadcluster/fake/restconfig.go b/pkg/workloadcluster/fake/restconfig.go new file mode 100644 index 000000000..829f7c48f --- /dev/null +++ b/pkg/workloadcluster/fake/restconfig.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package fake includes important implementation of interfaces like Workload cluster. +package fake + +import ( + "context" + + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type restClient struct{} + +type factory struct{} + +// NewClient creates new workload cluster clients. +func (*factory) NewClient(_, _ string, _ client.Client) workloadcluster.Client { + return &restClient{} +} + +// NewFactory creates new fake workload factory. +func NewFactory() workloadcluster.Factory { + return &factory{} +} + +// RestConfig returns the empty rest config and nil error. +func (*restClient) RestConfig(_ context.Context) (*rest.Config, error) { + return nil, nil +} diff --git a/vendor/github.com/MakeNowJust/heredoc/LICENSE b/vendor/github.com/MakeNowJust/heredoc/LICENSE new file mode 100644 index 000000000..6d0eb9d5d --- /dev/null +++ b/vendor/github.com/MakeNowJust/heredoc/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2019 TSUYUSATO Kitsune + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/MakeNowJust/heredoc/README.md b/vendor/github.com/MakeNowJust/heredoc/README.md new file mode 100644 index 000000000..e9924d297 --- /dev/null +++ b/vendor/github.com/MakeNowJust/heredoc/README.md @@ -0,0 +1,52 @@ +# heredoc + +[![Build Status](https://circleci.com/gh/MakeNowJust/heredoc.svg?style=svg)](https://circleci.com/gh/MakeNowJust/heredoc) [![GoDoc](https://godoc.org/github.com/MakeNowJusti/heredoc?status.svg)](https://godoc.org/github.com/MakeNowJust/heredoc) + +## About + +Package heredoc provides the here-document with keeping indent. + +## Install + +```console +$ go get github.com/MakeNowJust/heredoc +``` + +## Import + +```go +// usual +import "github.com/MakeNowJust/heredoc" +``` + +## Example + +```go +package main + +import ( + "fmt" + "github.com/MakeNowJust/heredoc" +) + +func main() { + fmt.Println(heredoc.Doc(` + Lorem ipsum dolor sit amet, consectetur adipisicing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna + aliqua. Ut enim ad minim veniam, ... + `)) + // Output: + // Lorem ipsum dolor sit amet, consectetur adipisicing elit, + // sed do eiusmod tempor incididunt ut labore et dolore magna + // aliqua. Ut enim ad minim veniam, ... + // +} +``` + +## API Document + + - [heredoc - GoDoc](https://godoc.org/github.com/MakeNowJust/heredoc) + +## License + +This software is released under the MIT License, see LICENSE. diff --git a/vendor/github.com/MakeNowJust/heredoc/heredoc.go b/vendor/github.com/MakeNowJust/heredoc/heredoc.go new file mode 100644 index 000000000..1fc046955 --- /dev/null +++ b/vendor/github.com/MakeNowJust/heredoc/heredoc.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014-2019 TSUYUSATO Kitsune +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +// Package heredoc provides creation of here-documents from raw strings. +// +// Golang supports raw-string syntax. +// +// doc := ` +// Foo +// Bar +// ` +// +// But raw-string cannot recognize indentation. Thus such content is an indented string, equivalent to +// +// "\n\tFoo\n\tBar\n" +// +// I dont't want this! +// +// However this problem is solved by package heredoc. +// +// doc := heredoc.Doc(` +// Foo +// Bar +// `) +// +// Is equivalent to +// +// "Foo\nBar\n" +package heredoc + +import ( + "fmt" + "strings" + "unicode" +) + +const maxInt = int(^uint(0) >> 1) + +// Doc returns un-indented string as here-document. +func Doc(raw string) string { + skipFirstLine := false + if len(raw) > 0 && raw[0] == '\n' { + raw = raw[1:] + } else { + skipFirstLine = true + } + + lines := strings.Split(raw, "\n") + + minIndentSize := getMinIndent(lines, skipFirstLine) + lines = removeIndentation(lines, minIndentSize, skipFirstLine) + + return strings.Join(lines, "\n") +} + +// getMinIndent calculates the minimum indentation in lines, excluding empty lines. +func getMinIndent(lines []string, skipFirstLine bool) int { + minIndentSize := maxInt + + for i, line := range lines { + if i == 0 && skipFirstLine { + continue + } + + indentSize := 0 + for _, r := range []rune(line) { + if unicode.IsSpace(r) { + indentSize += 1 + } else { + break + } + } + + if len(line) == indentSize { + if i == len(lines)-1 && indentSize < minIndentSize { + lines[i] = "" + } + } else if indentSize < minIndentSize { + minIndentSize = indentSize + } + } + return minIndentSize +} + +// removeIndentation removes n characters from the front of each line in lines. +// Skips first line if skipFirstLine is true, skips empty lines. +func removeIndentation(lines []string, n int, skipFirstLine bool) []string { + for i, line := range lines { + if i == 0 && skipFirstLine { + continue + } + + if len(lines[i]) >= n { + lines[i] = line[n:] + } + } + return lines +} + +// Docf returns unindented and formatted string as here-document. +// Formatting is done as for fmt.Printf(). +func Docf(raw string, args ...interface{}) string { + return fmt.Sprintf(Doc(raw), args...) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1f1a67392..c43c38c46 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,6 +2,9 @@ ## explicit; go 1.16 github.com/BurntSushi/toml github.com/BurntSushi/toml/internal +# github.com/MakeNowJust/heredoc v1.0.0 +## explicit; go 1.12 +github.com/MakeNowJust/heredoc # github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 ## explicit; go 1.13 github.com/ProtonMail/go-crypto/bitcurves @@ -739,6 +742,7 @@ sigs.k8s.io/cluster-api/util/patch sigs.k8s.io/cluster-api/util/predicates sigs.k8s.io/cluster-api/util/record sigs.k8s.io/cluster-api/util/secret +sigs.k8s.io/cluster-api/util/yaml # sigs.k8s.io/cluster-api/test v1.7.2 ## explicit; go 1.21 sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1 diff --git a/vendor/sigs.k8s.io/cluster-api/util/yaml/yaml.go b/vendor/sigs.k8s.io/cluster-api/util/yaml/yaml.go new file mode 100644 index 000000000..e539da9ac --- /dev/null +++ b/vendor/sigs.k8s.io/cluster-api/util/yaml/yaml.go @@ -0,0 +1,279 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package yaml implements yaml utility functions. +package yaml + +import ( + "bufio" + "bytes" + "io" + "os" + "strings" + + "github.com/MakeNowJust/heredoc" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/streaming" + apiyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/yaml" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" +) + +// ExtractClusterReferences returns the references in a Cluster object. +func ExtractClusterReferences(out *ParseOutput, c *clusterv1.Cluster) (res []*unstructured.Unstructured) { + if c.Spec.InfrastructureRef == nil { + return nil + } + if obj := out.FindUnstructuredReference(c.Spec.InfrastructureRef); obj != nil { + res = append(res, obj) + } + return +} + +// ExtractMachineReferences returns the references in a Machine object. +func ExtractMachineReferences(out *ParseOutput, m *clusterv1.Machine) (res []*unstructured.Unstructured) { + if obj := out.FindUnstructuredReference(&m.Spec.InfrastructureRef); obj != nil { + res = append(res, obj) + } + if m.Spec.Bootstrap.ConfigRef != nil { + if obj := out.FindUnstructuredReference(m.Spec.Bootstrap.ConfigRef); obj != nil { + res = append(res, obj) + } + } + return +} + +// ParseOutput is the output given from the Parse function. +type ParseOutput struct { + Clusters []*clusterv1.Cluster + Machines []*clusterv1.Machine + MachineSets []*clusterv1.MachineSet + MachineDeployments []*clusterv1.MachineDeployment + UnstructuredObjects []*unstructured.Unstructured +} + +// Add adds the other ParseOutput slices to this instance. +func (p *ParseOutput) Add(o *ParseOutput) *ParseOutput { + p.Clusters = append(p.Clusters, o.Clusters...) + p.Machines = append(p.Machines, o.Machines...) + p.MachineSets = append(p.MachineSets, o.MachineSets...) + p.MachineDeployments = append(p.MachineDeployments, o.MachineDeployments...) + p.UnstructuredObjects = append(p.UnstructuredObjects, o.UnstructuredObjects...) + return p +} + +// FindUnstructuredReference takes in an ObjectReference and tries to find an Unstructured object. +func (p *ParseOutput) FindUnstructuredReference(ref *corev1.ObjectReference) *unstructured.Unstructured { + for _, obj := range p.UnstructuredObjects { + if obj.GroupVersionKind() == ref.GroupVersionKind() && + ref.Namespace == obj.GetNamespace() && + ref.Name == obj.GetName() { + return obj + } + } + return nil +} + +// ParseInput is an input struct for the Parse function. +type ParseInput struct { + File string +} + +// Parse extracts runtime objects from a file. +func Parse(input ParseInput) (*ParseOutput, error) { + output := &ParseOutput{} + + // Open the input file. + reader, err := os.Open(input.File) + if err != nil { + return nil, err + } + + // Create a new decoder. + decoder := NewYAMLDecoder(reader) + defer decoder.Close() + + for { + u := &unstructured.Unstructured{} + _, gvk, err := decoder.Decode(nil, u) + if errors.Is(err, io.EOF) { + break + } + if runtime.IsNotRegisteredError(err) { + continue + } + if err != nil { + return nil, err + } + + switch gvk.Kind { + case "Cluster": + obj := &clusterv1.Cluster{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) + } + output.Clusters = append(output.Clusters, obj) + case "Machine": + obj := &clusterv1.Machine{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) + } + output.Machines = append(output.Machines, obj) + case "MachineSet": + obj := &clusterv1.MachineSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) + } + output.MachineSets = append(output.MachineSets, obj) + case "MachineDeployment": + obj := &clusterv1.MachineDeployment{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) + } + output.MachineDeployments = append(output.MachineDeployments, obj) + default: + output.UnstructuredObjects = append(output.UnstructuredObjects, u) + } + } + + return output, nil +} + +type yamlDecoder struct { + reader *apiyaml.YAMLReader + decoder runtime.Decoder + close func() error +} + +func (d *yamlDecoder) Decode(defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { + for { + doc, err := d.reader.Read() + if err != nil { + return nil, nil, err + } + + // Skip over empty documents, i.e. a leading `---` + if len(bytes.TrimSpace(doc)) == 0 { + continue + } + + return d.decoder.Decode(doc, defaults, into) + } +} + +func (d *yamlDecoder) Close() error { + return d.close() +} + +// NewYAMLDecoder returns a new streaming Decoded that supports YAML. +func NewYAMLDecoder(r io.ReadCloser) streaming.Decoder { + return &yamlDecoder{ + reader: apiyaml.NewYAMLReader(bufio.NewReader(r)), + decoder: scheme.Codecs.UniversalDeserializer(), + close: r.Close, + } +} + +// ToUnstructured takes a YAML and converts it to a list of Unstructured objects. +func ToUnstructured(rawyaml []byte) ([]unstructured.Unstructured, error) { + var ret []unstructured.Unstructured + + reader := apiyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(rawyaml))) + count := 1 + for { + // Read one YAML document at a time, until io.EOF is returned + b, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, errors.Wrapf(err, "failed to read yaml") + } + if len(b) == 0 { + break + } + + var m map[string]interface{} + if err := yaml.Unmarshal(b, &m); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal the %s yaml document: %q", util.Ordinalize(count), string(b)) + } + + var u unstructured.Unstructured + u.SetUnstructuredContent(m) + + // Ignore empty objects. + // Empty objects are generated if there are weird things in manifest files like e.g. two --- in a row without a yaml doc in the middle + if u.Object == nil { + continue + } + + ret = append(ret, u) + count++ + } + + return ret, nil +} + +// JoinYaml takes a list of YAML files and join them ensuring +// each YAML that the yaml separator goes on a new line by adding \n where necessary. +func JoinYaml(yamls ...[]byte) []byte { + var yamlSeparator = []byte("---") + + var cr = []byte("\n") + var b [][]byte //nolint:prealloc + for _, y := range yamls { + if !bytes.HasPrefix(y, cr) { + y = append(cr, y...) + } + if !bytes.HasSuffix(y, cr) { + y = append(y, cr...) + } + b = append(b, y) + } + + r := bytes.Join(b, yamlSeparator) + r = bytes.TrimPrefix(r, cr) + r = bytes.TrimSuffix(r, cr) + + return r +} + +// FromUnstructured takes a list of Unstructured objects and converts it into a YAML. +func FromUnstructured(objs []unstructured.Unstructured) ([]byte, error) { + var ret [][]byte //nolint:prealloc + for _, o := range objs { + content, err := yaml.Marshal(o.UnstructuredContent()) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal yaml for %s, %s/%s", o.GroupVersionKind(), o.GetNamespace(), o.GetName()) + } + ret = append(ret, content) + } + + return JoinYaml(ret...), nil +} + +// Raw returns un-indented yaml string; it also remove the first empty line, if any. +// While writing yaml, always use space instead of tabs for indentation. +func Raw(raw string) string { + return strings.TrimPrefix(heredoc.Doc(raw), "\n") +}