[release-1.26] version bumps and backports for 2023-08 release

Dockerfile

@@ -131,17 +131,17 @@ RUN CHART_VERSION="0.2.200"                   CHART_FILE=/charts/harvester-cloud
 RUN CHART_VERSION="0.1.1600"                  CHART_FILE=/charts/harvester-csi-driver.yaml     CHART_BOOTSTRAP=true /charts/build-chart.sh
 RUN CHART_VERSION="1.7.202"                   CHART_FILE=/charts/rke2-snapshot-controller.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh
 RUN CHART_VERSION="1.7.202"                   CHART_FILE=/charts/rke2-snapshot-controller-crd.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh
-RUN CHART_VERSION="1.7.101"                   CHART_FILE=/charts/rke2-snapshot-validation-webhook.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh
+RUN CHART_VERSION="1.7.300"                   CHART_FILE=/charts/rke2-snapshot-validation-webhook.yaml CHART_BOOTSTRAP=false /charts/build-chart.sh
 RUN rm -vf /charts/*.sh /charts/*.md
 
 # rke2-runtime image
 # This image includes any host level programs that we might need. All binaries
 # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation.
 # This means bin/foo/bar will become bin/bar when rke2 installs this to the host
 FROM rancher/hardened-kubernetes:v1.26.7-rke2r1-build20230719 AS kubernetes
-FROM rancher/hardened-containerd:v1.7.1-k3s1-build20230511 AS containerd
+FROM rancher/hardened-containerd:v1.7.3-k3s1-build20230802 AS containerd
 FROM rancher/hardened-crictl:v1.26.1-build20230406 AS crictl
-FROM rancher/hardened-runc:v1.1.7-build20230511 AS runc
+FROM rancher/hardened-runc:v1.1.8-build20230802 AS runc
 
 FROM scratch AS runtime-collect
 COPY --from=runc \

Dockerfile.windows

@@ -38,7 +38,7 @@ RUN curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/ins
 WORKDIR /source
 # End Dapper stuff
 
-FROM rancher/hardened-containerd:v1.7.1-k3s1-build20230511-amd64-windows AS containerd
+FROM rancher/hardened-containerd:v1.7.3-k3s1-build20230802-amd64-windows AS containerd
 FROM build as windows-runtime-collect
 ARG KUBERNETES_VERSION=dev
 

pkg/cli/cmds/cert.go

@@ -45,9 +45,11 @@ func NewCertCommand() cli.Command {
 			},
 		},
 		"rotate-ca": {
-			"server": copyFlag,
-			"path":   copyFlag,
-			"force":  copyFlag,
+			"server": {
+				Default: "https://127.0.0.1:9345",
+			},
+			"path":  copyFlag,
+			"force": copyFlag,
 			"data-dir": {
 				Usage:   "(data) Folder to hold state",
 				Default: rke2Path,

pkg/rke2/rke2.go

@@ -103,7 +103,7 @@ func Server(clx *cli.Context, cfg Config) error {
 	}
 	dataDir := clx.String("data-dir")
 	cmds.ServerConfig.StartupHooks = append(cmds.ServerConfig.StartupHooks,
-		checkStaticManifests(cmds.AgentConfig.ContainerRuntimeEndpoint, dataDir),
+		reconcileStaticPods(cmds.AgentConfig.ContainerRuntimeEndpoint, dataDir),
 		setNetworkPolicies(cisMode, defaultNamespaces),
 		setClusterRoles(),
 		restrictServiceAccounts(cisMode, defaultNamespaces),

pkg/rke2/spw.go

@@ -2,7 +2,6 @@ package rke2
 
 import (
 	"context"
-	"encoding/json"
 	"os"
 	"path/filepath"
 	"sync"
@@ -22,11 +21,11 @@ type containerInfo struct {
 	Config *runtimeapi.ContainerConfig `json:"config,omitempty"`
 }
 
-// checkStaticManifests validates that the pods started with rke2 match the static manifests
-// provided in /var/lib/rancher/rke2/agent/pod-manifests. When restarting rke2, it takes time
-// for any changes to static manifests to be pulled by kubelet. Additionally this prevents errors
-// where something is wrong with the static manifests and RKE2 starts anyways.
-func checkStaticManifests(containerRuntimeEndpoint, dataDir string) cmds.StartupHook {
+// reconcileStaticPods validates that the running pods for etcd and kube-apiserver match the static pod
+// manifests provided in /var/lib/rancher/rke2/agent/pod-manifests. If any old pods are found, they are
+// manually terminated, as the kubelet cannot be relied upon to terminate old pod when the apiserver is
+// not available.
+func reconcileStaticPods(containerRuntimeEndpoint, dataDir string) cmds.StartupHook {
 	return func(ctx context.Context, wg *sync.WaitGroup, args cmds.StartupHookArgs) error {
 		go func() {
 			defer wg.Done()
@@ -51,22 +50,22 @@ func checkStaticManifests(containerRuntimeEndpoint, dataDir string) cmds.Startup
 							// Since split-role servers exist, we don't care if no manifest is found
 							continue
 						}
-						logrus.Infof("Container for %s not found (%v), retrying", pod, err)
+						logrus.Infof("Pod for %s not synced (%v), retrying", pod, err)
 						return false, nil
 					}
-					logrus.Infof("Container for %s is running", pod)
+					logrus.Infof("Pod for %s is synced", pod)
 				}
 				return true, nil
 			}); err != nil {
-				logrus.Fatalf("Failed waiting for static pods to deploy: %v", err)
+				logrus.Fatalf("Failed waiting for static pods to sync: %v", err)
 			}
 		}()
 		return nil
 	}
 }
 
-// checkManifestDeployed returns an error if the static pod's manifest cannot be decoded and
-// verified as present and running with the current pod hash in the container runtime.
+// checkManifestDeployed returns an error if the static pod's manifest cannot be decoded and verified as present
+// and exclusively running with the current pod uid. If old pods are found, they will be terminated and an error returned.
 func checkManifestDeployed(ctx context.Context, cRuntime runtimeapi.RuntimeServiceClient, manifestFile string) error {
 	f, err := os.Open(manifestFile)
 	if err != nil {
@@ -81,43 +80,38 @@ func checkManifestDeployed(ctx context.Context, cRuntime runtimeapi.RuntimeServi
 		return errors.Wrap(err, "failed to decode manifest")
 	}
 
-	var podHash string
-	for _, env := range podManifest.Spec.Containers[0].Env {
-		if env.Name == "POD_HASH" {
-			podHash = env.Value
-			break
-		}
-	}
-
-	filter := &runtimeapi.ContainerFilter{
-		State: &runtimeapi.ContainerStateValue{
-			State: runtimeapi.ContainerState_CONTAINER_RUNNING,
-		},
+	filter := &runtimeapi.PodSandboxFilter{
 		LabelSelector: map[string]string{
-			"io.kubernetes.pod.uid": string(podManifest.UID),
+			"component":                   podManifest.Labels["component"],
+			"io.kubernetes.pod.namespace": podManifest.Namespace,
+			"tier":                        podManifest.Labels["tier"],
 		},
 	}
-
-	resp, err := cRuntime.ListContainers(ctx, &runtimeapi.ListContainersRequest{Filter: filter})
+	resp, err := cRuntime.ListPodSandbox(ctx, &runtimeapi.ListPodSandboxRequest{Filter: filter})
 	if err != nil {
-		return errors.Wrap(err, "failed to list containers")
+		return errors.Wrap(err, "failed to list pods")
 	}
 
-	for _, container := range resp.Containers {
-		resp, err := cRuntime.ContainerStatus(ctx, &runtimeapi.ContainerStatusRequest{ContainerId: container.Id, Verbose: true})
-		if err != nil {
-			return errors.Wrap(err, "failed to get container status")
+	var currentPod, stalePod bool
+	for _, pod := range resp.Items {
+		if pod.Annotations["kubernetes.io/config.source"] != "file" {
+			continue
 		}
-		info := &containerInfo{}
-		err = json.Unmarshal([]byte(resp.Info["info"]), &info)
-		if err != nil || info.Config == nil {
-			return errors.Wrap(err, "failed to unmarshal container config")
-		}
-		for _, env := range info.Config.Envs {
-			if env.Key == "POD_HASH" && env.Value == podHash {
-				return nil
+		if pod.Labels["io.kubernetes.pod.uid"] == string(podManifest.UID) {
+			currentPod = pod.State == runtimeapi.PodSandboxState_SANDBOX_READY
+		} else {
+			stalePod = true
+			if _, err := cRuntime.RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{PodSandboxId: pod.Id}); err != nil {
+				logrus.Warnf("Failed to terminate old %s pod: %v", pod.Metadata.Name, err)
 			}
 		}
 	}
-	return errors.New("no matching container found")
+
+	if stalePod {
+		return errors.New("waiting for termination of old pod")
+	}
+	if !currentPod {
+		return errors.New("no current running pod found")
+	}
+	return nil
 }

pkg/staticpod/staticpod.go

@@ -94,22 +94,15 @@ func Run(dir string, args Args) error {
 
 	manifestPath := filepath.Join(dir, args.Command+".yaml")
 
-	// Generate a stable UID based on the manifest path. This allows the kubelet to reconcile the pod's
-	// containers even when the apiserver is unavailable. If the UID is not stable, the kubelet
-	// will consider the manifest change as two separate add/remove operations, and may start the new pod
-	// before terminating the old one. Cleanup of removed pods is disabled until all sources have synced,
-	// so if the apiserver is down, the newly added pod may get stuck in a crash loop due to the old pod
-	// still using its ports. See https://github.com/rancher/rke2/issues/3387
+	// We hash the completed pod manifest use that as the UID; this mimics what upstream does:
+	// https://github.com/kubernetes/kubernetes/blob/v1.24.0/pkg/kubelet/config/common.go#L58-68
+	// We manually terminate static pods with incorrect UIDs, as the kubelet cannot be relied
+	// upon to clean up the old one while the apiserver is down.
+	// See https://github.com/rancher/rke2/issues/3387 and https://github.com/rancher/rke2/issues/3725
 	hasher := md5.New()
-	fmt.Fprint(hasher, manifestPath)
-	pod.UID = types.UID(hex.EncodeToString(hasher.Sum(nil)[0:]))
-
-	// Append a hash of the completed pod manifest to the container environment for later use when checking
-	// to see if the pod has been updated. It's fine that setting this changes the actual hash; we
-	// just need a stable values that we can compare between the file on disk and the running
-	// container to see if the kubelet has reconciled yet.
 	hash.DeepHashObject(hasher, pod)
-	pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, v1.EnvVar{Name: "POD_HASH", Value: hex.EncodeToString(hasher.Sum(nil)[0:])})
+	fmt.Fprintf(hasher, "file:%s", manifestPath)
+	pod.UID = types.UID(hex.EncodeToString(hasher.Sum(nil)[0:]))
 
 	b, err := yaml.Marshal(pod)
 	if err != nil {

scripts/build-binary

@@ -24,7 +24,7 @@ VERSION_FLAGS="
         -X ${K3S_PKG}/pkg/version.Program=${PROG}
         -X ${K3S_PKG}/pkg/version.Version=${VERSION}
         -X ${RKE2_PKG}/pkg/images.DefaultRegistry=${REGISTRY}
-        -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20230406
+        -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20230802
         -X ${RKE2_PKG}/pkg/images.DefaultKubernetesImage=rancher/hardened-kubernetes:${KUBERNETES_IMAGE_TAG}
         -X ${RKE2_PKG}/pkg/images.DefaultPauseImage=rancher/pause:${PAUSE_VERSION}
         -X ${RKE2_PKG}/pkg/images.DefaultRuntimeImage=${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}

scripts/build-images

@@ -16,7 +16,7 @@ xargs -n1 -t docker image pull --quiet << EOF >> build/images-core.txt
     ${REGISTRY}/rancher/hardened-coredns:v1.10.1-build20230406
     ${REGISTRY}/rancher/hardened-cluster-autoscaler:v1.8.6-build20230406
     ${REGISTRY}/rancher/hardened-dns-node-cache:1.22.20-build20230406
-    ${REGISTRY}/rancher/hardened-etcd:${ETCD_VERSION}-build20230406
+    ${REGISTRY}/rancher/hardened-etcd:${ETCD_VERSION}-build20230802
     ${REGISTRY}/rancher/hardened-k8s-metrics-server:v0.6.3-build20230515
     ${REGISTRY}/rancher/klipper-helm:v0.8.0-build20230510
     ${REGISTRY}/rancher/klipper-lb:v0.4.4
@@ -25,7 +25,7 @@ xargs -n1 -t docker image pull --quiet << EOF >> build/images-core.txt
     ${REGISTRY}/rancher/nginx-ingress-controller:nginx-1.7.1-hardened1
     ${REGISTRY}/rancher/rke2-cloud-provider:${CCM_VERSION}
     ${REGISTRY}/rancher/mirrored-sig-storage-snapshot-controller:v6.2.1
-    ${REGISTRY}/rancher/mirrored-sig-storage-snapshot-validation-webhook:v6.2.1
+    ${REGISTRY}/rancher/mirrored-sig-storage-snapshot-validation-webhook:v6.2.2
 EOF
 
 xargs -n1 -t docker image pull --quiet << EOF > build/images-canal.txt

scripts/build-windows-binary

@@ -30,7 +30,7 @@ VERSION_FLAGS="
         -X ${K3S_PKG}/pkg/version.Program=${PROG}
         -X ${K3S_PKG}/pkg/version.Version=${VERSION}
         -X ${RKE2_PKG}/pkg/images.DefaultRegistry=${REGISTRY}
-        -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20221129
+        -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20230802
         -X ${RKE2_PKG}/pkg/images.DefaultKubernetesImage=${REPO}/hardened-kubernetes:${KUBERNETES_IMAGE_TAG}
         -X ${RKE2_PKG}/pkg/images.DefaultPauseImage=rancher/pause:${PAUSE_VERSION}
         -X ${RKE2_PKG}/pkg/images.DefaultRuntimeImage=${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-${GOOS}-${GOARCH}

scripts/validate-release

@@ -27,10 +27,14 @@ function parse_tag() {
     fi
 }
 
+function get-module-version() {
+  go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $1
+}
+
 function check_release_branch() {
     TAG_BRANCH=$(git branch --all -q --contains $GIT_TAG | grep origin | grep -v origin$ | grep -v "HEAD" | sed -e 's/^[[:space:]]*//')
     if [ "$TAG_BRANCH" == "remotes/origin/master" ]; then
-        K8S_VERSION_GO_MOD=$(grep 'k8s.io/kubernetes v' go.mod | head -n1 | awk '{print $2}' | cut -d. -f1-2)
+        K8S_VERSION_GO_MOD=$(get-module-version k8s.io/kubernetes | cut -d. -f1-2)
         if [ "v$MAJOR.$MINOR" == "$K8S_VERSION_GO_MOD" ]; then
             info "Tag $GIT_TAG is cut from master"
             return

scripts/version.sh

@@ -33,7 +33,7 @@ RELEASE=${PROG}.${PLATFORM}
 # hardcode versions unless set specifically
 KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.26.7}
 KUBERNETES_IMAGE_TAG=${KUBERNETES_IMAGE_TAG:-v1.26.7-rke2r1-build20230719}
-ETCD_VERSION=${ETCD_VERSION:-v3.5.7-k3s1}
+ETCD_VERSION=${ETCD_VERSION:-v3.5.9-k3s1}
 PAUSE_VERSION=${PAUSE_VERSION:-3.6}
 CCM_VERSION=${CCM_VERSION:-v1.26.3-build20230406}