Add envtest for testing controllers

Signed-off-by: Adrian Pedriza <[email protected]>

cmd/main.go

@@ -231,7 +231,6 @@ func main() {
 
 		if err = (&controlplane.K0sController{
 			Client:     mgr.GetClient(),
-			Scheme:     mgr.GetScheme(),
 			ClientSet:  clientSet,
 			RESTConfig: restConfig,
 		}).SetupWithManager(mgr); err != nil {

go.mod

@@ -5,6 +5,7 @@ go 1.22.0
 require (
 	github.com/cloudflare/cfssl v1.6.4
 	github.com/go-logr/logr v1.4.2
+	github.com/gobuffalo/flect v1.0.2
 	github.com/google/uuid v1.6.0
 	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/k0s v1.27.2-0.20230504131248-94378e521a29
@@ -16,8 +17,11 @@ require (
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.30.3
+	k8s.io/apiextensions-apiserver v0.30.3
 	k8s.io/apimachinery v0.30.3
 	k8s.io/client-go v0.30.3
+	k8s.io/klog/v2 v2.120.1
+	k8s.io/kubectl v0.30.3
 	k8s.io/kubernetes v1.30.3
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	sigs.k8s.io/controller-runtime v0.18.5
@@ -57,7 +61,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.24.0
 	gotest.tools/v3 v3.4.0 // indirect
 	helm.sh/helm/v3 v3.11.3 // indirect
 	k8s.io/kube-aggregator v0.27.2 // indirect
@@ -90,7 +94,6 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -152,6 +155,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
@@ -166,14 +170,12 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.30.3 // indirect
 	k8s.io/apiserver v0.30.3 // indirect
 	k8s.io/cloud-provider v0.27.1 // indirect
 	k8s.io/cluster-bootstrap v0.30.3 // indirect
 	k8s.io/component-base v0.30.3 // indirect
 	k8s.io/component-helpers v0.30.3 // indirect
 	k8s.io/controller-manager v0.30.3 // indirect
-	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/kms v0.30.3 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kubelet v0.27.1 // indirect

go.sum

@@ -521,6 +521,8 @@ k8s.io/kube-aggregator v0.30.3 h1:hy5zfQ7p6BuJgc/XtGp3GBh2MPfOj6b1n3raKKMHOQE=
 k8s.io/kube-aggregator v0.30.3/go.mod h1:2SP0IckvQoOwwZN8lmtWUnTZTgIpwOWvidWtxyqLwuk=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI=
+k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo=
 k8s.io/kubelet v0.30.3 h1:KvGWDdhzD0vEyDyGTCjsDc8D+0+lwRMw3fJbfQgF7ys=
 k8s.io/kubelet v0.30.3/go.mod h1:D9or45Vkzcqg55CEiqZ8dVbwP3Ksj7DruEVRS9oq3Ys=
 k8s.io/kubernetes v1.30.3 h1:A0qoXI1YQNzrQZiff33y5zWxYHFT/HeZRK98/sRDJI0=

internal/controller/controlplane/helper.go

@@ -50,7 +50,7 @@ func (c *K0sController) createMachine(ctx context.Context, name string, cluster
 	if err != nil {
 		return nil, fmt.Errorf("error generating machine: %w", err)
 	}
-	_ = ctrl.SetControllerReference(kcp, machine, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, machine, c.Client.Scheme())
 
 	err = c.Client.Patch(ctx, machine, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",
@@ -226,7 +226,11 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		return nil, err
 	}
 
-	_ = ctrl.SetControllerReference(kcp, infraMachineTemplate, c.Scheme)
+	_ = ctrl.SetControllerReference(cluster, infraMachineTemplate, c.Client.Scheme())
+	err = c.Client.Patch(ctx, infraMachineTemplate, client.Merge, &client.PatchOptions{FieldManager: "k0smotron"})
+	if err != nil {
+		return nil, err
+	}
 
 	template, found, err := unstructured.NestedMap(infraMachineTemplate.UnstructuredContent(), "spec", "template")
 	if !found {

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -36,7 +36,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -48,7 +47,6 @@ import (
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/certs"
 	"sigs.k8s.io/cluster-api/util/collections"
-	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/failuredomains"
 	"sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/secret"
@@ -67,15 +65,20 @@ const (
 )
 
 var (
-	ErrNotReady            = fmt.Errorf("waiting for the state")
-	ErrNewMachinesNotReady = fmt.Errorf("waiting for new machines: %w", ErrNotReady)
+	ErrNotReady               = fmt.Errorf("waiting for the state")
+	ErrNewMachinesNotReady    = fmt.Errorf("waiting for new machines: %w", ErrNotReady)
+	FRPTokenNameTemplate      = "%s-frp-token"
+	FRPConfigMapNameTemplate  = "%s-frps-config"
+	FRPDeploymentNameTemplate = "%s-frps"
+	FRPServiceNameTemplate    = "%s-frps"
 )
 
 type K0sController struct {
 	client.Client
-	Scheme     *runtime.Scheme
 	ClientSet  *kubernetes.Clientset
 	RESTConfig *rest.Config
+	// workloadClusterKubeClient is used during testing to inject a fake client
+	workloadClusterKubeClient *kubernetes.Clientset
 }
 
 // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=k0scontrolplanes/status,verbs=get;list;watch;create;update;patch;delete
@@ -128,6 +131,11 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 		return ctrl.Result{}, nil
 	}
 
+	if annotations.IsPaused(cluster, kcp) {
+		log.Info("Reconciliation is paused for this object or owning cluster")
+		return ctrl.Result{}, nil
+	}
+
 	// Always patch the object to update the status
 	defer func() {
 		log.Info("Updating status")
@@ -171,13 +179,6 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 
 	log = log.WithValues("cluster", cluster.Name)
 
-	// TODO: Use paused.EnsurePausedCondition from "sigs.k8s.io/cluster-api/util/paused" when upgrading to v1.9.0.
-	if annotations.IsPaused(cluster, kcp) {
-		log.Info("Reconciliation is paused for this object or owning cluster")
-		conditions.MarkTrue(kcp, cpv1beta1.ControlPlanePausedCondition)
-		return ctrl.Result{}, nil
-	}
-
 	if err := c.ensureCertificates(ctx, cluster, kcp); err != nil {
 		log.Error(err, "Failed to ensure certificates")
 		return ctrl.Result{}, err
@@ -349,7 +350,7 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 	log.Log.Info("Got current cluster version", "version", currentVersion)
 
 	machineNamesToDelete := make(map[string]bool)
-	desiredMachineNames := make(map[string]bool)
+	desiredMachineNamesSlice := []string{}
 
 	var clusterIsUpdating bool
 	var clusterIsMutating bool
@@ -358,19 +359,28 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 			clusterIsUpdating = true
 			clusterIsMutating = true
 			if kcp.Spec.UpdateStrategy == cpv1beta1.UpdateInPlace {
-				desiredMachineNames[m.Name] = true
+				desiredMachineNamesSlice = append(desiredMachineNamesSlice, m.Name)
 			} else {
 				machineNamesToDelete[m.Name] = true
 			}
 		} else if !matchesTemplateClonedFrom(infraMachines, kcp, m) {
 			clusterIsMutating = true
 			machineNamesToDelete[m.Name] = true
-		} else if machines.Len() > int(kcp.Spec.Replicas)+len(machineNamesToDelete) {
-			machineNamesToDelete[m.Name] = true
 		} else {
-			desiredMachineNames[m.Name] = true
+			desiredMachineNamesSlice = append(desiredMachineNamesSlice, m.Name)
 		}
 	}
+
+	desiredMachineNames := make(map[string]bool)
+	for i := range desiredMachineNamesSlice {
+		desiredMachineNames[desiredMachineNamesSlice[i]] = true
+	}
+
+	// if it is necessary to reduce the number of replicas even counting the replicas to be eliminated
+	// because they are outdated, we choose the oldest among the valid ones.
+	if machines.Len() > int(kcp.Spec.Replicas)+len(machineNamesToDelete) && len(desiredMachineNamesSlice) > 0 {
+		machineNamesToDelete[desiredMachineNamesSlice[0]] = true
+	}
 	log.Log.Info("Collected machines", "count", machines.Len(), "desired", kcp.Spec.Replicas, "updating", clusterIsUpdating, "deleting", len(machineNamesToDelete), "desiredMachines", desiredMachineNames)
 
 	if clusterIsUpdating {
@@ -399,10 +409,10 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 
 	if len(machineNamesToDelete)+len(desiredMachineNames) > int(kcp.Spec.Replicas) {
 
-		m := machines.Newest().Name
-		err := c.checkMachineIsReady(ctx, m, cluster)
+		newestMachine := machines.Newest().Name
+		err := c.checkMachineIsReady(ctx, newestMachine, cluster)
 		if err != nil {
-			logger.Error(err, "Error checking machine left", "machine", m)
+			logger.Error(err, "Error checking machine left", "machine", newestMachine)
 			return err
 		}
 
@@ -689,7 +699,7 @@ token = ` + frpToken + `
 `
 	}
 
-	frpsCMName := kcp.GetName() + "-frps-config"
+	frpsCMName := fmt.Sprintf(FRPConfigMapNameTemplate, kcp.GetName())
 	cm := corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",
@@ -704,7 +714,7 @@ token = ` + frpToken + `
 		},
 	}
 
-	_ = ctrl.SetControllerReference(kcp, &cm, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &cm, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &cm, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating ConfigMap: %w", err)
@@ -716,7 +726,7 @@ token = ` + frpToken + `
 			Kind:       "Deployment",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPDeploymentNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: appsv1.DeploymentSpec{
@@ -773,7 +783,7 @@ token = ` + frpToken + `
 				}},
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsDeployment, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Deployment: %w", err)
@@ -785,7 +795,7 @@ token = ` + frpToken + `
 			Kind:       "Service",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPServiceNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: corev1.ServiceSpec{
@@ -809,7 +819,7 @@ token = ` + frpToken + `
 			Type: corev1.ServiceTypeNodePort,
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsService, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Service: %w", err)
@@ -828,7 +838,7 @@ func (c *K0sController) detectNodeIP(ctx context.Context, _ *cpv1beta1.K0sContro
 }
 
 func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (string, error) {
-	secretName := cluster.Name + "-frp-token"
+	secretName := fmt.Sprintf(FRPTokenNameTemplate, cluster.Name)
 
 	var existingSecret corev1.Secret
 	err := c.Client.Get(ctx, client.ObjectKey{Name: secretName, Namespace: cluster.Namespace}, &existingSecret)
@@ -857,7 +867,7 @@ func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.C
 		Type: clusterv1.ClusterSecretType,
 	}
 
-	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Client.Scheme())
 
 	return frpToken, c.Client.Patch(ctx, frpSecret, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",