Fix Kubernetes control-plane restart after IP rotation

Author: mxschmitt

Sources/ContainerK8s/Commands/K8sCreate.swift

@@ -102,6 +102,10 @@ public struct K8sCreate: AsyncParsableCommand {
             try await K8sHelper.prepareNode(nodeID: name, client: client, log: log)
             try await K8sHelper.bootstrapControlPlane(
                 nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP,
+                // All in-VM Kubernetes clients (including host-networked kube-proxy)
+                // can reach the single-node API through loopback. This endpoint is
+                // independent of the container's rotating vmnet address.
+                controlPlaneEndpoint: K8sHelper.nodeLocalControlPlaneEndpoint,
                 schedulable: provisioner.roles.contains(StandardRoles.worker),
                 client: client, log: log)
 

Sources/ContainerK8s/K8sHelper.swift

@@ -51,6 +51,8 @@ public struct K8sHelper {
     static let proxyEnvVars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]
 
     public static let clusterContainerPort: UInt16 = 6443
+    /// Endpoint used by clients inside the single-node cluster container.
+    static let nodeLocalControlPlaneEndpoint = "127.0.0.1:\(clusterContainerPort)"
 
     // MARK: - Resource defaults
 

Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift

@@ -35,12 +35,15 @@ extension K8sHelper {
 
     static func bootstrapControlPlane(
         nodeID: String, apiServerSANs: [String], advertiseAddress: String,
+        controlPlaneEndpoint: String,
         schedulable: Bool, client: ContainerClient, log: Logger
     ) async throws {
-        let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
+        let configYAML = initConfigYAML(
+            advertiseAddress: advertiseAddress, certSANs: apiServerSANs,
+            controlPlaneEndpoint: controlPlaneEndpoint)
         var r = try await execCapture(
             containerId: nodeID, executable: "/bin/sh",
-            arguments: ["-c", "cat > /etc/kubernetes/kubeadm-config.yaml <<'EOF'\n\(configYAML)\nEOF"],
+            arguments: ["-c", "mkdir -p /kind && cat > /kind/kubeadm.conf <<'EOF'\n\(configYAML)\nEOF"],
             client: client)
         guard r.code == 0 else {
             throw ContainerizationError(.internalError, message: "write kubeadm config failed on \(nodeID): \(r.output)")
@@ -50,7 +53,7 @@ extension K8sHelper {
         r = try await execCapture(
             containerId: nodeID, executable: kubeadmPath,
             arguments: [
-                "init", "--config", "/etc/kubernetes/kubeadm-config.yaml",
+                "init", "--config", "/kind/kubeadm.conf",
                 "--ignore-preflight-errors", ignorePreflightErrors,
             ],
             client: client)
@@ -135,7 +138,9 @@ extension K8sHelper {
         """
     }
 
-    private static func initConfigYAML(advertiseAddress: String, certSANs: [String]) -> String {
+    static func initConfigYAML(
+        advertiseAddress: String, certSANs: [String], controlPlaneEndpoint: String
+    ) -> String {
         let sans = certSANs.map { "  - \($0)" }.joined(separator: "\n")
         return """
             apiVersion: kubeadm.k8s.io/v1beta4
@@ -148,6 +153,7 @@ extension K8sHelper {
             ---
             apiVersion: kubeadm.k8s.io/v1beta4
             kind: ClusterConfiguration
+            controlPlaneEndpoint: \(controlPlaneEndpoint)
             kubernetesVersion: \(kubernetesVersion())
             networking:
               podSubnet: \(podSubnet)

Tests/IntegrationTests/K8s/TestK8sRunSerial.swift

@@ -115,4 +115,53 @@ struct TestK8sRunSerial {
             #expect(server1 != server2)
         }
     }
+
+    @Test func testRestartAfterAddressRotation() async throws {
+        try await ContainerFixture.with { f in
+            let name = "k8s-\(f.testID)"
+            let bumper = "\(name)-bumper"
+            f.addCleanup { try f.doRemoveIfExists(bumper, force: true, ignoreFailure: true) }
+            f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }
+
+            try f.restoreWarmupImage(.kindestNodeV1_35_5)
+            try f.run(["k8s", "create", "--name", name]).check()
+
+            let originalAddress = try f.run(["exec", name, "cat", "/kind/old-ipv4"])
+            try originalAddress.check()
+            #expect(try f.run(["exec", name, "test", "-f", "/kind/kubeadm.conf"]).status == 0)
+            try f.run([
+                "exec", name, "grep", "-F", "controlPlaneEndpoint: 127.0.0.1:6443", "/kind/kubeadm.conf",
+            ]).check()
+
+            try f.run(["stop", name]).check()
+            // Advance the rotating allocator while the node is stopped so its
+            // restart cannot accidentally reuse the same address.
+            try f.restoreWarmupImage(.alpine320)
+            try f.run([
+                "run", "--name", bumper, "-d", WarmupImage.alpine320.rawValue,
+                "sleep", "infinity",
+            ]).check()
+
+            let restart = try f.run(["k8s", "start", "--name", name])
+            if restart.status != 0 {
+                print("[k8s-run] restart stderr: \(restart.error)")
+                f.dumpNodeDiagnostics(node: name)
+            }
+            try restart.check()
+
+            let rotatedAddress = try f.run(["exec", name, "cat", "/kind/old-ipv4"])
+            try rotatedAddress.check()
+            let originalIP = originalAddress.output.trimmingCharacters(in: .whitespacesAndNewlines)
+            let rotatedIP = rotatedAddress.output.trimmingCharacters(in: .whitespacesAndNewlines)
+            print("[k8s-run] restart rotated address \(originalIP) -> \(rotatedIP)")
+            #expect(originalIP != rotatedIP)
+            #expect(try f.getContainerStatus(name) == "running")
+            try f.run([
+                "exec", name, "grep", "-F", "advertiseAddress: \(rotatedIP)", "/kind/kubeadm.conf",
+            ]).check()
+            try f.run([
+                "exec", name, "kubectl", "wait", "--for=condition=Ready", "node", "--all", "--timeout=30s",
+            ]).check()
+        }
+    }
 }

Tests/K8sPluginTests/K8sBootstrapTests.swift

@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Testing
+import Yams
+
+@testable import ContainerK8s
+
+@Suite("K8s bootstrap configuration")
+struct K8sBootstrapTests {
+    @Test func nodeLocalEndpointUsesClusterPort() {
+        #expect(K8sHelper.nodeLocalControlPlaneEndpoint == "127.0.0.1:6443")
+    }
+
+    @Test func nodeLocalControlPlaneEndpointIsRendered() {
+        let yaml = K8sHelper.initConfigYAML(
+            advertiseAddress: "192.168.64.2",
+            certSANs: ["127.0.0.1"],
+            controlPlaneEndpoint: K8sHelper.nodeLocalControlPlaneEndpoint)
+
+        #expect(yaml.contains("controlPlaneEndpoint: 127.0.0.1:6443"))
+        #expect(yaml.contains("advertiseAddress: 192.168.64.2"))
+        #expect(yaml.contains("  - 127.0.0.1"))
+        for document in yaml.components(separatedBy: "\n---\n") {
+            #expect((try? Yams.load(yaml: document)) != nil)
+        }
+    }
+}