K0sControlPlane update strategies

.github/workflows/go.yml

@@ -159,6 +159,7 @@ jobs:
           - check-capi-controlplane-docker-worker
           - check-capi-remote-machine-template-update
           - check-capi-docker-machine-template-update
+          - check-capi-docker-machine-template-update-recreate
           - check-capi-remote-machine-job-provision
 
     uses: ./.github/workflows/capi-smoke-tests.yml

.github/workflows/prepare-build-env.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+set -eu
+
+goVersion="$(./vars.sh go_version)"
+golangciLintVersion="$(./vars.sh FROM=hack/tools golangci-lint_version)"
+pythonVersion="$(./vars.sh FROM=docs python_version)"
+
+cat <<EOF >>"$GITHUB_ENV"
+GO_VERSION=$goVersion
+GOLANGCI_LINT_VERSION=$golangciLintVersion
+PYTHON_VERSION=$pythonVersion
+EOF
+
+# shellcheck disable=SC1090
+. "$GITHUB_ENV"
+
+echo ::group::OS Environment
+env | sort
+echo ::endgroup::

api/controlplane/v1beta1/k0s_types.go

@@ -31,7 +31,8 @@ func init() {
 type UpdateStrategy string
 
 const (
-	UpdateInPlace UpdateStrategy = "InPlace"
+	UpdateInPlace  UpdateStrategy = "InPlace"
+	UpdateRecreate UpdateStrategy = "Recreate"
 )
 
 // +kubebuilder:object:root=true
@@ -55,7 +56,7 @@ type K0sControlPlaneSpec struct {
 	Replicas int32 `json:"replicas,omitempty"`
 	// UpdateStrategy defines the strategy to use when updating the control plane. Currently only InPlace is supported.
 	//+kubebuilder:validation:Optional
-	//+kubebuilder:validation:Enum:InPlace
+	//+kubebuilder:validation:Enum:InPlace,Recreate
 	//+kubebuilder:default=InPlace
 	UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"`
 	// Version defines the k0s version to be deployed. You can use a specific k0s version (e.g. v1.27.1+k0s.0) or

internal/controller/bootstrap/controlplane_bootstrap_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	capiutil "sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -213,14 +215,23 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, fmt.Errorf("control plane endpoint is not set")
 	}
 
-	if strings.HasSuffix(config.Name, "-0") {
+	machines, err := collections.GetFilteredMachinesForCluster(ctx, c.Client, cluster, collections.ControlPlaneMachines(cluster.Name), collections.ActiveMachines)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error collecting machines: %w", err)
+	}
+
+	if machines.Oldest().Name == config.Name {
 		files, err = c.genInitialControlPlaneFiles(ctx, scope, files)
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error generating initial control plane files: %v", err)
 		}
 		installCmd = createCPInstallCmd(scope)
 	} else {
-		files, err = c.genControlPlaneJoinFiles(ctx, scope, files)
+		oldest := getFirstRunningMachineWithLatestVersion(machines)
+		if oldest == nil {
+			return ctrl.Result{}, fmt.Errorf("wait for initial control plane provisioning")
+		}
+		files, err = c.genControlPlaneJoinFiles(ctx, scope, files, oldest)
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error generating control plane join files: %v", err)
 		}
@@ -323,7 +334,7 @@ func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context
 	return files, nil
 }
 
-func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File) ([]cloudinit.File, error) {
+func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File, firstControllerMachine *clusterv1.Machine) ([]cloudinit.File, error) {
 	log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name)
 
 	_, ca, err := c.getCerts(ctx, scope)
@@ -350,7 +361,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
 		return nil, err
 	}
 
-	host, err := c.findFirstControllerIP(ctx, scope.Config)
+	host, err := c.findFirstControllerIP(ctx, firstControllerMachine)
 	if err != nil {
 		log.Error(err, "Failed to get controller IP")
 		return nil, err
@@ -576,22 +587,9 @@ func createCPInstallCmdWithJoinToken(scope *ControllerScope, tokenPath string) s
 	return strings.Join(installCmd, " ")
 }
 
-func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, config *bootstrapv1.K0sControllerConfig) (string, error) {
-	// Dirty first controller name generation
-	nameParts := strings.Split(config.Name, "-")
-	nameParts[len(nameParts)-1] = "0"
-	name := strings.Join(nameParts, "-")
-	machine, machineImpl, err := c.getMachineImplementation(ctx, name, config)
-	if err != nil {
-		return "", fmt.Errorf("error getting machine implementation: %w", err)
-	}
-	addresses, found, err := unstructured.NestedSlice(machineImpl.UnstructuredContent(), "status", "addresses")
-	if err != nil {
-		return "", err
-	}
-
+func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, firstControllerMachine *clusterv1.Machine) (string, error) {
 	extAddr, intAddr := "", ""
-	for _, addr := range machine.Status.Addresses {
+	for _, addr := range firstControllerMachine.Status.Addresses {
 		if addr.Type == clusterv1.MachineExternalIP {
 			extAddr = addr.Address
 			break
@@ -602,16 +600,29 @@ func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, conf
 		}
 	}
 
-	if found {
-		for _, addr := range addresses {
-			addrMap, _ := addr.(map[string]interface{})
-			if addrMap["type"] == string(v1.NodeExternalIP) {
-				extAddr = addrMap["address"].(string)
-				break
-			}
-			if addrMap["type"] == string(v1.NodeInternalIP) {
-				intAddr = addrMap["address"].(string)
-				break
+	name := firstControllerMachine.Name
+
+	if extAddr == "" && intAddr == "" {
+		machineImpl, err := c.getMachineImplementation(ctx, firstControllerMachine)
+		if err != nil {
+			return "", fmt.Errorf("error getting machine implementation: %w", err)
+		}
+		addresses, found, err := unstructured.NestedSlice(machineImpl.UnstructuredContent(), "status", "addresses")
+		if err != nil {
+			return "", err
+		}
+
+		if found {
+			for _, addr := range addresses {
+				addrMap, _ := addr.(map[string]interface{})
+				if addrMap["type"] == string(v1.NodeExternalIP) {
+					extAddr = addrMap["address"].(string)
+					break
+				}
+				if addrMap["type"] == string(v1.NodeInternalIP) {
+					intAddr = addrMap["address"].(string)
+					break
+				}
 			}
 		}
 	}
@@ -627,13 +638,7 @@ func (c *ControlPlaneController) findFirstControllerIP(ctx context.Context, conf
 	return "", fmt.Errorf("no address found for machine %s", name)
 }
 
-func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, name string, config *bootstrapv1.K0sControllerConfig) (*clusterv1.Machine, *unstructured.Unstructured, error) {
-	var machine clusterv1.Machine
-	err := c.Get(ctx, client.ObjectKey{Name: name, Namespace: config.Namespace}, &machine)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error getting machine object: %w", err)
-	}
-
+func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, machine *clusterv1.Machine) (*unstructured.Unstructured, error) {
 	infRef := machine.Spec.InfrastructureRef
 
 	machineImpl := new(unstructured.Unstructured)
@@ -643,11 +648,11 @@ func (c *ControlPlaneController) getMachineImplementation(ctx context.Context, n
 
 	key := client.ObjectKey{Name: infRef.Name, Namespace: infRef.Namespace}
 
-	err = c.Get(ctx, key, machineImpl)
+	err := c.Get(ctx, key, machineImpl)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error getting machine implementation object: %w", err)
+		return nil, fmt.Errorf("error getting machine implementation object: %w", err)
 	}
-	return &machine, machineImpl, nil
+	return machineImpl, nil
 }
 
 func genShutdownServiceFiles() []cloudinit.File {
@@ -700,3 +705,31 @@ command="/etc/bin/k0sleave.sh"
 		},
 	}
 }
+
+func getFirstRunningMachineWithLatestVersion(machines collections.Machines) *clusterv1.Machine {
+	res := make(machinesByVersionAndCreationTimestamp, 0, len(machines))
+	for _, value := range machines {
+		if value.Status.Phase == string(clusterv1.MachinePhasePending) {
+			continue
+		}
+		res = append(res, value)
+	}
+	if len(res) == 0 {
+		return nil
+	}
+	sort.Sort(res)
+	return res[0]
+}
+
+// machinesByCreationTimestamp sorts a list of Machine by creation timestamp, using their names as a tie breaker.
+type machinesByVersionAndCreationTimestamp []*clusterv1.Machine
+
+func (o machinesByVersionAndCreationTimestamp) Len() int      { return len(o) }
+func (o machinesByVersionAndCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
+func (o machinesByVersionAndCreationTimestamp) Less(i, j int) bool {
+
+	if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) {
+		return o[i].Name < o[j].Name
+	}
+	return *o[i].Spec.Version < *o[j].Spec.Version && o[i].CreationTimestamp.Before(&o[j].CreationTimestamp)
+}

internal/controller/bootstrap/providerid_controller.go

@@ -48,7 +48,8 @@ func (p *ProviderIDController) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	if machine.Spec.ProviderID == nil || *machine.Spec.ProviderID == "" {
-		return ctrl.Result{}, fmt.Errorf("waiting for providerID for the machine %s/%s", machine.Namespace, machine.Name)
+		log.Info("waiting for providerID for the machine " + machine.Name)
+		return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 	}
 
 	cluster, err := capiutil.GetClusterByName(ctx, p.Client, machine.Namespace, machine.Spec.ClusterName)

internal/controller/controlplane/helper.go

@@ -53,7 +53,7 @@
 	return nil
 }
 
 func (c *K0sController) machineExist(ctx context.Context, name string, kcp *cpv1beta1.K0sControlPlane) (bool, error) {
 	var machine clusterv1.Machine
 
 	err := c.Client.Get(ctx, client.ObjectKey{Name: name, Namespace: kcp.Namespace}, &machine)
@@ -189,12 +189,20 @@
 
 	err := clientset.RESTClient().
 		Patch(types.MergePatchType).
-		AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes/" + name).
-		Body([]byte(`{"metadata":{"annotations":{"k0smotron.io/leave":"true"}}}`)).
+		AbsPath("/apis/etcd.k0sproject.io/v1beta1/etcdmembers/" + name).
+		Body([]byte(`{"spec":{"leave":"true"}}`)).
 		Do(ctx).
 		Error()
-	if err != nil && !apierrors.IsNotFound(err) {
-		return fmt.Errorf("error marking control node to leave: %w", err)
+	if err != nil {
+		err := clientset.RESTClient().
+			Patch(types.MergePatchType).
+			AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes/" + name).
+			Body([]byte(`{"metadata":{"annotations":{"k0smotron.io/leave":"true"}}}`)).
+			Do(ctx).
+			Error()
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("error marking control node to leave: %w", err)
+		}
 	}
 
 	return nil