diff --git a/.github/workflows/k8s-check.yaml b/.github/workflows/k8s-check.yaml deleted file mode 100644 index d2954887d..000000000 --- a/.github/workflows/k8s-check.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: Check Native Kubernetes - -on: - pull_request: - branches: - - master - - development - - k8s-new - paths: - - ".github/workflows/k8s-check.yaml" - - "MANIFEST.in" - - "development.env" - - "requirements.txt" - - "setup.py" - - "docker_images/multiarch/**" - - "seedemu/compiler/Docker.py" - - "seedemu/compiler/kubernetes.py" - - "seedemu/compiler/__init__.py" - - "seedemu/k8sTools/**" - - "examples/internet/b61_k8s_compile/**" - - "tests/k8s/**" - workflow_dispatch: - inputs: - build_images: - description: "Build and push native K8s workload images to a local registry" - type: boolean - required: false - default: true - -permissions: - contents: read - -env: - PYTHON_VERSION: "3.10" - K8S_NAMESPACE: seedemu-k8s-b61 - K8S_IMAGE_REGISTRY_PREFIX: seedemu - K8S_OUTPUT_DIR: /tmp/seedemu-k8s-output - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - k8s-static: - name: K8s Static - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install static-check dependencies - run: python -m pip install PyYAML - - name: Validate K8s source files - run: python3 tests/k8s/validateK8sStatic.py - - k8s-compile: - name: K8s Compile - runs-on: ubuntu-latest - needs: k8s-static - timeout-minutes: 20 - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install compile dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Compile native K8s example - run: | - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Archive native K8s compile output - uses: actions/upload-artifact@v6 - with: - name: native-k8s-output - path: ${{ env.K8S_OUTPUT_DIR }} - if-no-files-found: error - - k8s-build-images: - name: K8s Build Images - runs-on: ubuntu-latest - needs: k8s-compile - timeout-minutes: 45 - if: ${{ github.event_name == 'pull_request' || github.event.inputs.build_images == 'true' }} - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Check Docker buildx - run: | - docker version - docker buildx version - - name: Compile native K8s example for image build - run: | - rm -rf "${K8S_OUTPUT_DIR}" - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s build output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Build and push images to local registry - run: | - bash tests/k8s/buildNativeK8sImages.sh \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --registry-prefix 127.0.0.1:5000 \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" diff --git a/examples/internet/b61_k8s_compile/README.md b/examples/internet/b61_k8s_compile/README.md index 7e2d9b4a3..9c4095e99 100644 --- a/examples/internet/b61_k8s_compile/README.md +++ b/examples/internet/b61_k8s_compile/README.md @@ -35,9 +35,16 @@ The default output directory is `./output`. To write elsewhere: python3 ./mini_internet_k8s.py --output-dir /tmp/seedemu-b61-output ``` -The compiler writes `k8s.kube-ovn.yaml`, `images.yaml`, and one Docker build -context per compiled SeedEMU node. The default logical image prefix is -`seedemu`, and the default namespace is `seedemu-k8s-b61`. +To compile macvlan manifests instead of the default Kube-OVN/OVS manifests: + +```bash +python3 ./mini_internet_k8s.py --cni-type macvlan +``` + +The default compiler mode writes `k8s.kube-ovn.yaml`, `images.yaml`, and one +Docker build context per compiled SeedEMU node. `--cni-type macvlan` writes +`k8s.yaml` with macvlan NetworkAttachmentDefinitions. `k8sTools.py up` also +accepts `images.txt` metadata when present. The default namespace is `seedemu`. ## Common Deploy Commands @@ -47,17 +54,19 @@ Every setup mode follows the same command shape: python3 ./k8sTools.py build \ --input \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml python3 ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml -python3 ./k8sTools.py down -f ./output -k kubeconfig.yaml +python3 ./k8sTools.py clean -f ./output -k kubeconfig.yaml python3 ./k8sTools.py destroy -d configK3s.yaml ``` -`build` creates or prepares infrastructure and writes `configK3s.yaml` plus -`kubeconfig.yaml`. `up` builds/pushes workload images and applies the SeedEMU -namespace. `down` removes only the workload namespace. `destroy` removes the -K3s/OVN setup and any KVM infrastructure recorded in `configK3s.yaml`. +`build` creates or prepares infrastructure and writes `configK3s.yaml`, +`kubeconfig.yaml`, and `inventory.yaml`. `up` builds/pushes workload images and +applies the SeedEMU namespace. `clean` removes only the workload namespace. +`destroy` removes the K3s/OVN setup and any KVM infrastructure recorded in +`configK3s.yaml`. Add `--keep-temp` to any `k8sTools.py` command when you need to inspect its temporary setup/running directory after a failure. @@ -72,7 +81,8 @@ Use this when you want the shortest local KVM configuration. python3 ./k8sTools.py build \ --input configKvmOvnSimply.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` This file is a minimal template for `kind: kvmOvn`. It only specifies the @@ -89,7 +99,8 @@ Use this for a more explicit local KVM deployment. python3 ./k8sTools.py build \ --input configKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Compared with `configKvmOvnSimply.yaml`, this file pins VM names, IP ranges, @@ -106,7 +117,8 @@ Use this when Kubernetes should be built on existing physical machines. python3 ./k8sTools.py build \ --input configK3sOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Each `nodes[]` entry describes a real machine: `name`, `role`, management `ip`, @@ -126,7 +138,8 @@ K3s cluster. python3 ./k8sTools.py build \ --input configMultiHostKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` The `hypervisors[]` section describes each physical KVM host. Each hypervisor @@ -175,10 +188,58 @@ ssh -i ~/.ssh/seedemu_k8s seed@192.0.2.11 'sudo -n true' ## Runtime Notes - `k8sTools.py up` infers the logical compiler image prefix from - `output/images.yaml`; use `--image-registry-prefix` only for custom outputs. -- `k8sTools.py down` keeps the cluster infrastructure and only removes the + `output/images.yaml` or `output/images.txt`; use `--image-registry-prefix` + only for custom outputs. +- During `up`, generated Dockerfile `FROM` images such as + `handsonsecurity/seedemu-base:2.0` and `handsonsecurity/seedemu-router:2.0` + are prepared on the local host first and then loaded into the registry/master + Docker daemon before remote buildx runs. +- `k8sTools.py clean` keeps the cluster infrastructure and only removes the deployed SeedEMU workload. - `k8sTools.py destroy` uses metadata embedded in generated `configK3s.yaml`; do not delete that file before cleanup. - Real `build`, `up`, `down`, and `destroy` commands modify local or remote infrastructure. The compile step alone is non-destructive. + +## Quick Kubernetes Debug Commands + +Run these commands from this directory. Use the local stable kubeconfig +explicitly so a stale `KUBECONFIG` from another experiment is not used by +mistake: + +```bash +export B61_NS=seedemu +export B61_KUBECONFIG=./examples/internet/b61_k8s_compile/kubeconfig.yaml +``` + +Check namespace, nodes, and pod status: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" get ns +kubectl --kubeconfig "${B61_KUBECONFIG}" get namespace "${B61_NS}" +kubectl --kubeconfig "${B61_KUBECONFIG}" get nodes -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get deploy -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get events --sort-by='.lastTimestamp' +``` + +Inspect or enter one pod/deployment. Replace the object name with a real name +from `kubectl get pods` or `kubectl get deploy`: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" describe pod +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" logs --tail=80 +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it -- bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it deploy/ -- sh +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec deploy/ -- birdc show protocols +``` +as1862brd-r13-1.13.7.70-7564dbb49c-9wctj + +Large post-BIRD runs can intermittently return `TLS handshake timeout`, +`Client.Timeout exceeded`, or `http2: client connection lost` while VM CPUs are +saturated. First retry the same command with an explicit request timeout: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" --request-timeout=120s -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +``` diff --git a/examples/internet/b61_k8s_compile/SKILL.md b/examples/internet/b61_k8s_compile/SKILL.md index 5bdec745e..b6b1839a2 100644 --- a/examples/internet/b61_k8s_compile/SKILL.md +++ b/examples/internet/b61_k8s_compile/SKILL.md @@ -36,7 +36,7 @@ unless the task specifically concerns runtime artifacts. Think of the workflow as three separate contracts: 1. Compile contract: - - `NativeKubernetesCompiler` turns a rendered SeedEMU emulator into + - `KubernetesCompiler` turns a rendered SeedEMU emulator into Kubernetes manifests and Docker build contexts. - The B61 example writes compiler output to `./output` by default. - Important files are `k8s.kube-ovn.yaml` or `k8s.yaml`, `images.yaml`, diff --git a/examples/internet/b61_k8s_compile/mini_internet_k8s.py b/examples/internet/b61_k8s_compile/mini_internet_k8s.py index 7b65d2f50..243ca8868 100644 --- a/examples/internet/b61_k8s_compile/mini_internet_k8s.py +++ b/examples/internet/b61_k8s_compile/mini_internet_k8s.py @@ -6,6 +6,7 @@ NetworkAttachmentDefinitions, Deployments. - images.yaml: image build contexts consumed by the generated running/ stage. - per-node Docker build contexts and optional base_images/ contexts. +Default CNI is kube-ovn. Use --cni-type macvlan for macvlan manifests. """ from __future__ import annotations @@ -34,7 +35,7 @@ def findRepoRoot(start: Path) -> Path: from seedemu.utilities import Makers from seedemu.compiler import Platform -from seedemu.compiler import NativeKubernetesCompiler +from seedemu.compiler import KubernetesCompiler HOSTS_PER_AS = 2 @@ -56,6 +57,12 @@ def parse_args() -> argparse.Namespace: default="amd64", help="Target platform for Docker build contexts.", ) + parser.add_argument( + "--cni-type", + choices=("kube-ovn", "ovn", "macvlan"), + default="kube-ovn", + help="Secondary CNI backend for SeedEMU links. Defaults to kube-ovn.", + ) return parser.parse_args() @@ -127,7 +134,7 @@ def main() -> int: output_dir = (SCRIPT_DIR / output_dir).resolve() platform = Platform.AMD64 if args.platform == "amd64" else Platform.ARM64 - compiler = NativeKubernetesCompiler(platform=platform) + compiler = KubernetesCompiler(platform=platform, cni_type=args.cni_type) emu = build_mini_internet(hosts_per_as=HOSTS_PER_AS) emu.compile(compiler, str(output_dir), override=True) @@ -138,7 +145,8 @@ def main() -> int: print("Config file: not used") print(f"Output directory: {output_dir}") print("Image registry prefix: seedemu") - print("Namespace: seedemu-k8s-b61") + print("Namespace: seedemu") + print(f"CNI type: {args.cni_type}") print("Runtime workflow: seedemu.k8sTools") print("Inventory required for compile: no") return 0 diff --git a/seedemu/compiler/__init__.py b/seedemu/compiler/__init__.py index 5dad3a90a..552e5d449 100644 --- a/seedemu/compiler/__init__.py +++ b/seedemu/compiler/__init__.py @@ -5,4 +5,4 @@ from .DistributedDocker import DistributedDocker from .Graphviz import Graphviz from .GcpDistributedDocker import GcpDistributedDocker -from .kubernetes import NativeKubernetesCompiler +from .kubernetes import KubernetesCompiler, NativeKubernetesCompiler, SchedulingStrategy diff --git a/seedemu/compiler/kubernetes.py b/seedemu/compiler/kubernetes.py index d5ad8d962..46b13749a 100644 --- a/seedemu/compiler/kubernetes.py +++ b/seedemu/compiler/kubernetes.py @@ -22,10 +22,23 @@ SEEDEMU_NODE_TYPES = ["rnode", "csnode", "hnode", "rs", "snode"] -class NativeKubernetesCompiler(Docker): +class SchedulingStrategy: + """Compatibility constants for older Kubernetes compiler callers.""" + + NONE = "none" + AUTO = "auto" + BY_AS = "by_as" + BY_AS_HARD = "by_as_hard" + BY_ROLE = "by_role" + CUSTOM = "custom" + + +class KubernetesCompiler(Docker): """Minimal Kubernetes compiler for a k8s-native baseline. Design goals: + - public compiler name is KubernetesCompiler + - kube-ovn is the default secondary CNI, macvlan is the only alternative - no inventory dependency during compile - no nodeSelector / affinity / topology spreading - no node-aware preload planning @@ -45,28 +58,37 @@ def __init__( self, image_registry_prefix: str = "seedemu", registry_prefix: str | None = None, - namespace: str = "seedemu-k8s-b61", + namespace: str = "seedemu", + use_multus: bool = True, cni_type: str = "kube-ovn", + local_link_cni_type: str | None = None, cni_master_interface: str = "ens2", image_pull_policy: str = "Always", - # use_multus: bool = True, - # create_namespace: bool = True, + scheduling_strategy: str = SchedulingStrategy.NONE, + node_labels: Dict[str, Dict[str, str]] | None = None, + default_resources: Dict[str, Dict[str, str]] | None = None, + generate_services: bool = False, + service_type: str = "ClusterIP", **kwargs: Any, ) -> None: kwargs["selfManagedNetwork"] = True super().__init__(**kwargs) + if not use_multus: + raise ValueError("KubernetesCompiler requires Multus network attachments.") if registry_prefix is not None: image_registry_prefix = registry_prefix self.__image_registry_prefix = image_registry_prefix.strip().strip("/") self.__namespace = namespace.strip() or "seedemu" - self.__cni_type = cni_type.strip().lower() or "bridge" + self.__cni_type = self._normalizeCniType(cni_type) + if local_link_cni_type is not None and str(local_link_cni_type).strip(): + self._normalizeCniType(str(local_link_cni_type)) self.__cni_master_interface = cni_master_interface.strip() or "eth0" self.__image_pull_policy = image_pull_policy.strip() or "Always" self.__manifests = [] self.__image_entries = [] def getName(self) -> str: - return "NativeKubernetes" + return "Kubernetes" def getManifestName(self) -> str: """Return the manifest filename generated by this compiler.""" @@ -74,13 +96,26 @@ def getManifestName(self) -> str: return "k8s.kube-ovn.yaml" return "k8s.yaml" + @staticmethod + def _normalizeCniType(cni_type: str | None) -> str: + """Normalize and validate the secondary CNI mode.""" + value = (cni_type or "kube-ovn").strip().lower().replace("_", "-") + if value in {"kube-ovn", "ovn"}: + return "kube-ovn" + if value == "macvlan": + return "macvlan" + raise ValueError( + f"KubernetesCompiler supports only cni_type \"kube-ovn\" or " + f"cni_type \"macvlan\"; got {cni_type!r}." + ) + @staticmethod def _safeBridgeName(name: str) -> str: return f"br-{md5(name.encode()).hexdigest()[:12]}" def _usesKubeOvn(self) -> bool: """Return true when the compiler should emit Kube-OVN resources.""" - return self.__cni_type in {"kube-ovn", "kube_ovn", "ovn"} + return self.__cni_type == "kube-ovn" def _doCompile(self, emulator) -> None: registry = emulator.getRegistry() @@ -116,39 +151,24 @@ def _doCompile(self, emulator) -> None: def _compileNetK8s(self, net: Network) -> Dict[str, Any]: name = self._getRealNetName(net).replace("_", "-").lower() - prefix = str(net.getPrefix()) - if self.__cni_type == "macvlan": + if self._usesKubeOvn(): config = { "cniVersion": "0.3.1", - "type": "macvlan", - "master": self.__cni_master_interface, - "mode": "bridge", - "ipam": {"type": "static"}, + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": f"{name}.{self.__namespace}.ovn", } - elif self.__cni_type == "ipvlan": + elif self.__cni_type == "macvlan": config = { "cniVersion": "0.3.1", - "type": "ipvlan", + "type": "macvlan", "master": self.__cni_master_interface, - "mode": "l2", + "mode": "bridge", "ipam": {"type": "static"}, } - elif self.__cni_type == "host-local": - config = { - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "isGateway": False, - "ipam": {"type": "host-local", "subnet": prefix}, - } else: - config = { - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "ipam": {"type": "static"}, - } + raise ValueError(f"unsupported Kubernetes CNI type: {self.__cni_type}") return { "apiVersion": "k8s.cni.cncf.io/v1", @@ -567,3 +587,5 @@ def _stage_base_image_contexts(self) -> None: dockerfile = os.path.join(dst_root, "Dockerfile") with open(dockerfile, "w", encoding="utf-8") as handle: handle.write(f"FROM {image}\n") + +NativeKubernetesCompiler = KubernetesCompiler diff --git a/seedemu/k8sTools/README.md b/seedemu/k8sTools/README.md index 67ed4da21..7012b7a01 100644 --- a/seedemu/k8sTools/README.md +++ b/seedemu/k8sTools/README.md @@ -13,7 +13,8 @@ Build infrastructure: python k8sTools.py build \ --input configKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Deploy workload: @@ -22,16 +23,19 @@ Deploy workload: python k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml ``` -`up` infers the logical compiler image prefix from `images.yaml`. Pass +`up` infers the logical compiler image prefix from `images.yaml` or +`images.txt`. Pass `--image-registry-prefix ` only when the compile output intentionally uses a non-standard or mixed prefix. Clean workload only: ```bash -python k8sTools.py down -f ./output -k kubeconfig.yaml +python k8sTools.py clean -f ./output -k kubeconfig.yaml ``` +`down` remains accepted as a compatibility alias for `clean`. + Destroy infrastructure: ```bash @@ -60,13 +64,17 @@ Python entrypoints there: `up` copies bundled `resources/running/` into a temporary directory and invokes `manageRunningStage.py preflight`, `build`, and `up`. The running stage renders -`kustomization.yaml`, rewrites OVN manifests when needed, builds/pushes images -to the registry resolved from `configK3s.yaml`, applies the manifest, and waits -for readiness. +`kustomization.yaml`, rewrites OVN manifests when needed, reads image metadata +from `images.yaml` or `images.txt`, builds/pushes images to the registry +resolved from `configK3s.yaml`, applies the manifest, and waits for readiness. +Before a remote registry-host build, it scans generated Dockerfiles for external +`FROM` images, ensures those images exist on the local Docker daemon, and loads +them into the registry host Docker daemon so buildx does not need to resolve +compiler base images through Docker Hub from the master node. No persistent setup/running working directory is created by default. Persistent outputs are limited to the user-requested `configK3s.yaml`, `kubeconfig.yaml`, -and the compiler output directory. +optional `inventory.yaml`, and the compiler output directory. All commands accept `--keep-temp` to leave the copied setup/running resources on disk for debugging. diff --git a/seedemu/k8sTools/k8sTools.py b/seedemu/k8sTools/k8sTools.py index e6830a1f5..8a2d64014 100644 --- a/seedemu/k8sTools/k8sTools.py +++ b/seedemu/k8sTools/k8sTools.py @@ -22,6 +22,7 @@ def build( configK3s: str | Path, kubeconfig: str | Path, *, + inventory: str | Path | None = None, keepTemp: bool = False, ) -> None: """Create infrastructure and write K3s access files. @@ -30,9 +31,10 @@ def build( inputConfig: YAML with kind=kvmOvn, physicalOvn, or multiHostKvmOvn. configK3s: Output configK3s.yaml path. kubeconfig: Output kubeconfig path. + inventory: Optional output inventory.yaml path. keepTemp: Keep temporary setup resources for debugging. """ - buildCluster(inputConfig, configK3s, kubeconfig, keep_temp=keepTemp) + buildCluster(inputConfig, configK3s, kubeconfig, inventory=inventory, keep_temp=keepTemp) def up( self, @@ -71,6 +73,16 @@ def down(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool """ cleanWorkload(outputDir, kubeconfig, keep_temp=keepTemp) + def clean(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool = False) -> None: + """Delete workload resources while keeping cluster infrastructure. + + Args: + outputDir: Compile output directory. + kubeconfig: Kubeconfig path. + keepTemp: Keep temporary cleanup resources for debugging. + """ + self.down(outputDir, kubeconfig, keepTemp=keepTemp) + def destroy(self, configK3s: str | Path, *, keepTemp: bool = False) -> None: """Destroy K3s/OVN and optional KVM infrastructure. @@ -93,6 +105,7 @@ def runCli(self, argv: list[str] | None = None) -> int: build.add_argument("--input", required=True, help="input YAML with explicit kind") build.add_argument("--config-k3s", required=True, help="output configK3s.yaml") build.add_argument("--kubeconfig", required=True, help="output kubeconfig.yaml") + build.add_argument("--inventory", help="optional output inventory.yaml with node resources") build.add_argument("--keep-temp", action="store_true", help="keep temporary setup resources") up = sub.add_parser("up", help="build images and deploy workload") @@ -110,13 +123,18 @@ def runCli(self, argv: list[str] | None = None) -> int: down.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") down.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + clean = sub.add_parser("clean", help="delete workload resources only") + clean.add_argument("-f", "--folder", required=True, help="compile output directory") + clean.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") + clean.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + destroy = sub.add_parser("destroy", help="destroy cluster and optional KVM resources") destroy.add_argument("-d", "--config-k3s", required=True, help="configK3s.yaml") destroy.add_argument("--keep-temp", action="store_true", help="keep temporary destroy resources") args = parser.parse_args(argv) if args.command == "build": - self.build(args.input, args.config_k3s, args.kubeconfig, keepTemp=args.keep_temp) + self.build(args.input, args.config_k3s, args.kubeconfig, inventory=args.inventory, keepTemp=args.keep_temp) elif args.command == "up": self.up( args.folder, @@ -127,6 +145,8 @@ def runCli(self, argv: list[str] | None = None) -> int: ) elif args.command == "down": self.down(args.folder, args.kubeconfig, keepTemp=args.keep_temp) + elif args.command == "clean": + self.clean(args.folder, args.kubeconfig, keepTemp=args.keep_temp) elif args.command == "destroy": self.destroy(args.config_k3s, keepTemp=args.keep_temp) else: diff --git a/seedemu/k8sTools/resources/running/_embeddedShell.py b/seedemu/k8sTools/resources/running/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/running/_embeddedShell.py +++ b/seedemu/k8sTools/resources/running/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.py b/seedemu/k8sTools/resources/running/buildRegistryImages.py index 13a24552a..e4de2feeb 100755 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.py +++ b/seedemu/k8sTools/resources/running/buildRegistryImages.py @@ -1,21 +1,185 @@ #!/usr/bin/env python3 -"""Python entrypoint for buildRegistryImages.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for buildRegistryImages with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Build SeedEMU workload images with BuildKit/buildx and push them to the +# registry selected by command-line arguments or configRunning.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +OUTPUT_DIR="" +REGISTRY_PREFIX="" +IMAGE_REGISTRY_PREFIX="" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +fi +if [ -z "${REGISTRY_PREFIX}" ]; then + REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +fi +if [ -z "${IMAGE_REGISTRY_PREFIX}" ]; then + IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +fi + +IMAGES_YAML="${OUTPUT_DIR}/images.yaml" +[ -s "${IMAGES_YAML}" ] || IMAGES_YAML="${OUTPUT_DIR}/images.txt" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +cd "${OUTPUT_DIR}" + +buildImageWithBuildx() { + # Args: + # $1: full destination image reference + # $2: Docker build context directory + local image="$1" + local context="$2" + local log_file + log_file="$(mktemp)" + echo "+ DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t ${image} ${context}" + if DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" 2>&1 | tee "${log_file}"; then + rm -f "${log_file}" + return 0 + fi + + if grep -Eq 'parent snapshot .* does not exist|failed to prepare extraction snapshot' "${log_file}"; then + echo "[k8s_build] buildx export/cache failure while building ${image}; pruning buildx cache and retrying once" >&2 + docker buildx prune -af >/dev/null 2>&1 || true + rm -f "${log_file}" + DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" + return $? + fi + + rm -f "${log_file}" + return 1 +} + +firstFromImage() { + # Args: + # $1: Dockerfile path. + # Prints the first FROM image reference, handling optional --platform. + awk ' + toupper($1) == "FROM" { + if ($2 ~ /^--platform=/) { + print $3 + } else { + print $2 + } + exit + } + ' "$1" +} + +isCompilerBaseTag() { + # Args: + # $1: image reference to check. + # Returns true for Docker compiler hash tags used in generated FROM lines. + [[ "$1" =~ ^[0-9a-f]{32}(:latest)?$ ]] +} + +ensureCompilerBaseImages() { + # Build every hash-tagged base image required by generated node Dockerfiles. + # A missing base_images//Dockerfile means the compiler output is + # incomplete; without this check Docker would try pulling library/. + local tmpfile + local dockerfile + local from_image + local base_tag + local context + local missing=0 + + tmpfile="$(mktemp)" + while IFS= read -r dockerfile; do + from_image="$(firstFromImage "${dockerfile}")" + if isCompilerBaseTag "${from_image}"; then + printf '%s\n' "${from_image%:latest}" >> "${tmpfile}" + fi + done < <(find . -path './base_images/*' -prune -o -name Dockerfile -print | sort) + + while IFS= read -r base_tag; do + [ -n "${base_tag}" ] || continue + context="base_images/${base_tag}" + if [ -f "${context}/Dockerfile" ]; then + buildImageWithBuildx "${base_tag}" "${context}" + elif docker image inspect "${base_tag}" >/dev/null 2>&1; then + continue + else + echo "Missing compiler base image context: ${context}/Dockerfile" >&2 + echo "Re-run compile with the current KubernetesCompiler so base_images/ is generated." >&2 + missing=1 + fi + done < <(sort -u "${tmpfile}") + + rm -f "${tmpfile}" + return "${missing}" +} + +ensureCompilerBaseImages + +python3 "${HELPER}" mapped-images \ + --images-yaml "${IMAGES_YAML}" \ + --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" \ + --registry-prefix "${REGISTRY_PREFIX}" | +while IFS=$'\t' read -r image context; do + [ -n "${image}" ] || continue + [ -n "${context}" ] || { echo "Missing context for ${image}" >&2; exit 1; } + buildImageWithBuildx "${image}" "${context}" + echo "+ docker push ${image}" + docker push "${image}" +done +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.sh b/seedemu/k8sTools/resources/running/buildRegistryImages.sh deleted file mode 100755 index fb88d64ed..000000000 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# Build SeedEMU workload images with BuildKit/buildx and push them to the -# registry selected by command-line arguments or configRunning.yaml. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" -OUTPUT_DIR="" -REGISTRY_PREFIX="" -IMAGE_REGISTRY_PREFIX="" -HELPER="${SCRIPT_DIR}/manageK8sManifest.py" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" -fi -if [ -z "${REGISTRY_PREFIX}" ]; then - REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" -fi -if [ -z "${IMAGE_REGISTRY_PREFIX}" ]; then - IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" -fi - -IMAGES_YAML="${OUTPUT_DIR}/images.yaml" -OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" -cd "${OUTPUT_DIR}" - -buildImageWithBuildx() { - # Args: - # $1: full destination image reference - # $2: Docker build context directory - local image="$1" - local context="$2" - local log_file - log_file="$(mktemp)" - echo "+ DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t ${image} ${context}" - if DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" 2>&1 | tee "${log_file}"; then - rm -f "${log_file}" - return 0 - fi - - if grep -Eq 'parent snapshot .* does not exist|failed to prepare extraction snapshot' "${log_file}"; then - echo "[k8s_build] buildx export/cache failure while building ${image}; pruning buildx cache and retrying once" >&2 - docker buildx prune -af >/dev/null 2>&1 || true - rm -f "${log_file}" - DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" - return $? - fi - - rm -f "${log_file}" - return 1 -} - -firstFromImage() { - # Args: - # $1: Dockerfile path. - # Prints the first FROM image reference, handling optional --platform. - awk ' - toupper($1) == "FROM" { - if ($2 ~ /^--platform=/) { - print $3 - } else { - print $2 - } - exit - } - ' "$1" -} - -isCompilerBaseTag() { - # Args: - # $1: image reference to check. - # Returns true for Docker compiler hash tags used in generated FROM lines. - [[ "$1" =~ ^[0-9a-f]{32}(:latest)?$ ]] -} - -ensureCompilerBaseImages() { - # Build every hash-tagged base image required by generated node Dockerfiles. - # A missing base_images//Dockerfile means the compiler output is - # incomplete; without this check Docker would try pulling library/. - local tmpfile - local dockerfile - local from_image - local base_tag - local context - local missing=0 - - tmpfile="$(mktemp)" - while IFS= read -r dockerfile; do - from_image="$(firstFromImage "${dockerfile}")" - if isCompilerBaseTag "${from_image}"; then - printf '%s\n' "${from_image%:latest}" >> "${tmpfile}" - fi - done < <(find . -path './base_images/*' -prune -o -name Dockerfile -print | sort) - - while IFS= read -r base_tag; do - [ -n "${base_tag}" ] || continue - context="base_images/${base_tag}" - if [ -f "${context}/Dockerfile" ]; then - buildImageWithBuildx "${base_tag}" "${context}" - elif docker image inspect "${base_tag}" >/dev/null 2>&1; then - continue - else - echo "Missing compiler base image context: ${context}/Dockerfile" >&2 - echo "Re-run compile with the current NativeKubernetesCompiler so base_images/ is generated." >&2 - missing=1 - fi - done < <(sort -u "${tmpfile}") - - rm -f "${tmpfile}" - return "${missing}" -} - -ensureCompilerBaseImages - -python3 "${HELPER}" mapped-images \ - --images-yaml "${IMAGES_YAML}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" \ - --registry-prefix "${REGISTRY_PREFIX}" | -while IFS=$'\t' read -r image context; do - [ -n "${image}" ] || continue - [ -n "${context}" ] || { echo "Missing context for ${image}" >&2; exit 1; } - buildImageWithBuildx "${image}" "${context}" - echo "+ docker push ${image}" - docker push "${image}" -done diff --git a/seedemu/k8sTools/resources/running/manageK8sManifest.py b/seedemu/k8sTools/resources/running/manageK8sManifest.py index 3d5cb4b5e..157181521 100755 --- a/seedemu/k8sTools/resources/running/manageK8sManifest.py +++ b/seedemu/k8sTools/resources/running/manageK8sManifest.py @@ -18,6 +18,7 @@ import hashlib import ipaddress import json +import os import subprocess from pathlib import Path from typing import Any @@ -266,11 +267,18 @@ def running_context(config_path: str) -> dict[str, str]: if fabric_type in {"linux-vxlan", "vxlan", "linux_vxlan"} else "ens2" ) + attached_cni_type = str( + get_nested( + setup, + "cni.localLinkCniType", + get_nested(setup, "fabric.attachedCniType", os.environ.get("SEED_LOCAL_LINK_CNI_TYPE", "kube-ovn")), + ) + ) return { "setupConfig": str(setup_config_path), "outputDir": str(output_dir), "manifest": str(manifest_path), - "imagesYaml": str(output_dir / "images.yaml"), + "imagesYaml": str(resolveImagesPath(output_dir)), "kustomization": str(output_dir / "kustomization.yaml"), "imageRegistryPrefix": str(running.get("imageRegistryPrefix") or "seedemu"), "registryPrefix": f"{registry_host}:{registry_port}", @@ -290,6 +298,7 @@ def running_context(config_path: str) -> dict[str, str]: "masterConnection": str((master_node or {}).get("connection") or "ssh"), "cniMasterInterface": str(get_nested(setup, "cni.defaultMasterInterface", default_cni_master)), "networkBackend": network_backend, + "attachedCniType": attached_cni_type, "rolloutTimeoutSeconds": str(running.get("rolloutTimeoutSeconds") or "1800"), } @@ -320,6 +329,18 @@ def resolveManifestPath(running: dict[str, Any], output_dir: Path, network_backe return default_manifest +def resolveImagesPath(output_dir: Path) -> Path: + """Return the generated image metadata file for the compile output. + + Args: + output_dir: Compile output directory. + """ + images_yaml = output_dir / "images.yaml" + if images_yaml.exists(): + return images_yaml + return output_dir / "images.txt" + + def config_value(args: argparse.Namespace) -> None: """Print one resolved value from configRunning.yaml.""" values = running_context(args.config) @@ -370,8 +391,48 @@ def strip_prefix(image: str, prefix: str) -> str: def load_images(path: str) -> list[dict[str, str]]: - data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} - return data.get("images", []) + """Load image metadata from images.yaml or legacy images.txt. + + Args: + path: Compiler image metadata file. + """ + image_path = Path(path) + text = image_path.read_text(encoding="utf-8") + if image_path.name == "images.txt": + return imagesFromText(text) + data = yaml.safe_load(text) or {} + if isinstance(data, dict): + images = data.get("images", []) + if isinstance(images, list): + return images + return imagesFromText(text) + + +def imagesFromText(text: str) -> list[dict[str, str]]: + """Parse one-image-reference-per-line metadata from KubernetesCompiler. + + Args: + text: Contents of images.txt. + """ + images: list[dict[str, str]] = [] + for line in text.splitlines(): + image = line.strip() + if not image or image.startswith("#"): + continue + images.append({"name": image, "context": contextFromImage(image)}) + return images + + +def contextFromImage(image: str) -> str: + """Infer a Docker build context directory from an image reference. + + Args: + image: Full image reference generated by the compiler. + """ + repo = image.rsplit("/", 1)[-1] + if ":" in repo: + repo = repo.rsplit(":", 1)[0] + return repo def mapped_images(args: argparse.Namespace) -> None: @@ -409,57 +470,128 @@ def render_kustomization(args: argparse.Namespace) -> None: if network_backend in {"kube-ovn", "ovn"}: rendered_manifest = output_path.parent / "k8s.kube-ovn.yaml" if manifest_path.resolve() != rendered_manifest.resolve(): - renderKubeOvnManifest(args.manifest, rendered_manifest) + renderKubeOvnManifest( + args.manifest, + rendered_manifest, + attached_cni_type=args.attached_cni_type, + cni_master_interface=args.cni_master_interface, + ) payload = {"resources": [rendered_manifest.name], "images": images} else: - payload = {"resources": ["k8s.yaml"], "images": images} + payload = {"resources": namespaceResources(manifest_path, output_path.parent) + ["k8s.yaml"], "images": images} patches = networkAttachmentPatches(args.manifest, args.cni_master_interface) if patches: payload["patches"] = patches output_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") -def renderKubeOvnManifest(manifest_path: str, output_path: Path) -> None: - """Render a SeedEMU manifest that uses Kube-OVN secondary L2 networks. +def renderKubeOvnManifest( + manifest_path: str, + output_path: Path, + attached_cni_type: str | None = None, + cni_master_interface: str | None = None, +) -> None: + """Render a SeedEMU manifest that uses Kube-OVN managed secondary networks. Args: manifest_path: Source k8s.yaml generated by the compiler. output_path: Destination manifest referenced by kustomization.yaml. + attached_cni_type: Secondary CNI type. ``kube-ovn`` creates overlay + attached NICs; ``macvlan`` keeps macvlan dataplane and uses + Kube-OVN only for IPAM. + cni_master_interface: Optional macvlan master interface override. The compiler emits macvlan NADs plus Multus network-selection annotations - with static `ips` values in CIDR form. Kube-OVN attached NICs use provider - annotations such as `..ovn.kubernetes.io/ip_address` and - expect plain IP strings. The renderer therefore converts NADs to - type=kube-ovn, creates matching Subnets, and rewrites Pod template - annotations without changing the compiler output. + with static `ips` values in CIDR form. The renderer creates matching + Subnets and rewrites Pod template annotations without changing compiler + output. In macvlan mode, Kube-OVN acts as the centralized IPAM plugin and + avoids creating thousands of OVN logical switches/routes for scale tests. """ - with open(manifest_path, "r", encoding="utf-8") as fh: - docs = [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] + docs = loadManifestDocs(Path(manifest_path)) namespace_name = findNamespaceName(docs) + attached_cni = resolveAttachedCniType(attached_cni_type) + use_ovn_attached = isKubeOvnAttachedCni(attached_cni) vpc_name = kubeOvnResourceName("vpc", namespace_name) rendered: list[dict[str, Any]] = [] vpc_written = False + namespace_written = any(doc.get("kind") == "Namespace" for doc in docs) + if not namespace_written: + rendered.append(namespaceResource(namespace_name)) for doc in docs: - if doc.get("kind") == "Namespace" and not vpc_written: + if use_ovn_attached and doc.get("kind") == "Namespace" and not vpc_written: rendered.append(doc) rendered.append(kubeOvnVpc(namespace_name, vpc_name)) vpc_written = True continue if doc.get("kind") == "NetworkAttachmentDefinition": - converted = convertNetworkAttachmentToKubeOvn(doc, namespace_name, vpc_name) + converted = convertNetworkAttachmentToKubeOvn( + doc, + namespace_name, + vpc_name, + attached_cni, + cni_master_interface, + ) rendered.extend(converted) continue - convertWorkloadAnnotationsToKubeOvn(doc, namespace_name) + convertWorkloadAnnotationsToKubeOvn(doc, namespace_name, attached_cni) rendered.append(doc) - if not vpc_written: - rendered.insert(0, kubeOvnVpc(namespace_name, vpc_name)) + if use_ovn_attached and not vpc_written: + insert_at = 1 if rendered and rendered[0].get("kind") == "Namespace" else 0 + rendered.insert(insert_at, kubeOvnVpc(namespace_name, vpc_name)) output_path.write_text(yaml.safe_dump_all(rendered, sort_keys=False), encoding="utf-8") +def namespaceResource(namespace_name: str) -> dict[str, Any]: + """Return a Kubernetes Namespace resource for compiler outputs without one.""" + return {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace_name}} + + +def namespaceResources(manifest_path: Path, output_dir: Path) -> list[str]: + """Write namespace.yaml when the manifest lacks an explicit Namespace. + + Args: + manifest_path: Source manifest path. + output_dir: Directory that will contain kustomization.yaml. + """ + docs = loadManifestDocs(manifest_path) + if any(doc.get("kind") == "Namespace" for doc in docs): + return [] + namespace_name = findNamespaceName(docs) + namespace_path = output_dir / "namespace.yaml" + namespace_path.write_text(yaml.safe_dump(namespaceResource(namespace_name), sort_keys=False), encoding="utf-8") + return [namespace_path.name] + + +def resolveAttachedCniType(attached_cni_type: str | None = None) -> str: + """Return the configured secondary CNI type for Kube-OVN rendering.""" + value = ( + attached_cni_type + or os.environ.get("SEED_LOCAL_LINK_CNI_TYPE") + or os.environ.get("SEED_CNI_TYPE") + or "kube-ovn" + ) + return str(value).strip().lower().replace("_", "-") + + +def loadManifestDocs(manifest_path: Path) -> list[dict[str, Any]]: + """Load Kubernetes documents from a manifest path. + + Args: + manifest_path: YAML manifest path. + """ + with open(manifest_path, "r", encoding="utf-8") as fh: + return [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] + + +def isKubeOvnAttachedCni(attached_cni_type: str) -> bool: + """Return true when secondary NICs should use the Kube-OVN CNI directly.""" + return attached_cni_type in {"kube-ovn", "ovn"} + + def findNamespaceName(docs: list[dict[str, Any]]) -> str: """Return the manifest namespace used by SeedEMU workload resources.""" for doc in docs: @@ -497,6 +629,8 @@ def convertNetworkAttachmentToKubeOvn( doc: dict[str, Any], default_namespace: str, vpc_name: str, + attached_cni_type: str, + cni_master_interface: str | None, ) -> list[dict[str, Any]]: """Return [Subnet, NAD] for one compiler-generated macvlan NAD. @@ -504,6 +638,8 @@ def convertNetworkAttachmentToKubeOvn( doc: Original NetworkAttachmentDefinition document. default_namespace: Namespace to use when the NAD omits metadata.namespace. vpc_name: Namespace-scoped Kube-OVN VPC name. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. """ metadata = doc.get("metadata") or {} nad_name = str(metadata.get("name") or "") @@ -515,7 +651,8 @@ def convertNetworkAttachmentToKubeOvn( if not prefix: raise SystemExit(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") - provider = f"{nad_name}.{namespace_name}.ovn" + use_ovn_attached = isKubeOvnAttachedCni(attached_cni_type) + provider = kubeOvnProviderName(nad_name, namespace_name, attached_cni_type) subnet_name = kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") subnet = { "apiVersion": "kubeovn.io/v1", @@ -524,43 +661,86 @@ def convertNetworkAttachmentToKubeOvn( "spec": { "protocol": "IPv4", "provider": provider, - "vpc": vpc_name, "cidrBlock": str(prefix), "gateway": firstUsableIp(str(prefix)), - "gatewayType": "distributed", "natOutgoing": False, "private": False, }, } + if use_ovn_attached: + subnet["spec"]["vpc"] = vpc_name + subnet["spec"]["gatewayType"] = "distributed" converted = dict(doc) - converted["spec"] = { - "config": json.dumps( - { - "cniVersion": "0.3.1", - "type": "kube-ovn", - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", - "provider": provider, - }, - separators=(",", ":"), - ) - } + converted["spec"] = {"config": renderConvertedNadConfig(doc, provider, attached_cni_type, cni_master_interface)} return [subnet, converted] -def convertWorkloadAnnotationsToKubeOvn(doc: dict[str, Any], default_namespace: str) -> None: +def renderConvertedNadConfig( + doc: dict[str, Any], + provider: str, + attached_cni_type: str, + cni_master_interface: str | None, +) -> str: + """Return NetworkAttachmentDefinition spec.config for the selected CNI. + + Args: + doc: Original NetworkAttachmentDefinition document. + provider: Kube-OVN provider string used to match the Subnet. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. + """ + if isKubeOvnAttachedCni(attached_cni_type): + config = { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + spec = doc.get("spec") or {} + raw_config = spec.get("config") + try: + config = json.loads(raw_config) if isinstance(raw_config, str) else {} + except json.JSONDecodeError: + config = {} + if not isinstance(config, dict): + config = {} + config.setdefault("cniVersion", "0.3.1") + config["type"] = attached_cni_type + if attached_cni_type == "macvlan": + if cni_master_interface: + config["master"] = cni_master_interface + config.setdefault("mode", "bridge") + config["ipam"] = { + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + +def convertWorkloadAnnotationsToKubeOvn( + doc: dict[str, Any], + default_namespace: str, + attached_cni_type: str, +) -> None: """Rewrite Multus static IP annotations for Kube-OVN attached NICs. Args: doc: Manifest document to mutate in place. Deployments and raw Pods are supported because both can contain Multus network annotations. default_namespace: Namespace to use when a network selection omits it. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. SeedEMU's compiler writes Multus `ips: ["10.x.x.x/24"]`, which is correct - for macvlan with static IPAM. Kube-OVN non-primary mode uses provider - annotations and expects plain IP values, so each `ips` item is converted to - `..ovn.kubernetes.io/ip_address: 10.x.x.x`. + for static CNI IPAM. Kube-OVN IPAM uses per-provider annotations and + expects plain IP values. Raw Pods use `ip_address`; workload templates use + `ip_pool`, otherwise kube-ovn-controller rejects Deployment pods during + address allocation. """ + kind = str(doc.get("kind") or "") annotations = getPodTemplateAnnotations(doc) if not annotations: return @@ -587,13 +767,36 @@ def convertWorkloadAnnotationsToKubeOvn(doc: dict[str, Any], default_namespace: ip_values = [stripCidr(str(ip_value)) for ip_value in ips if str(ip_value).strip()] if not ip_values: continue - annotations[f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address"] = ",".join(ip_values) + static_field = kubeOvnStaticAddressField(kind) + provider = kubeOvnProviderName(nad_name, nad_namespace, attached_cni_type) + subnet_name = kubeOvnResourceName("subnet", f"{nad_namespace}-{nad_name}") + annotations[f"{provider}.kubernetes.io/{static_field}"] = ",".join(ip_values) + annotations[f"{provider}.kubernetes.io/logical_switch"] = subnet_name changed = True if changed: annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) +def kubeOvnProviderName(nad_name: str, namespace_name: str, attached_cni_type: str) -> str: + """Return the provider name used by Kube-OVN for one NetworkAttachmentDefinition.""" + provider = f"{nad_name}.{namespace_name}" + if isKubeOvnAttachedCni(attached_cni_type): + provider = f"{provider}.ovn" + return provider + + +def kubeOvnStaticAddressField(kind: str) -> str: + """Return the Kube-OVN fixed-address annotation field for a resource kind. + + Args: + kind: Kubernetes resource kind containing the Multus annotation. + """ + if kind == "Pod": + return "ip_address" + return "ip_pool" + + def getPodTemplateAnnotations(doc: dict[str, Any]) -> dict[str, str] | None: """Return pod-level annotations from a workload or Pod document. @@ -709,14 +912,7 @@ def deployment_names(args: argparse.Namespace) -> None: def namespace(args: argparse.Namespace) -> None: - with open(args.manifest, "r", encoding="utf-8") as fh: - for doc in yaml.safe_load_all(fh): - if isinstance(doc, dict) and doc.get("kind") == "Namespace": - name = (doc.get("metadata") or {}).get("name") - if name: - print(name) - return - raise SystemExit(f"No Namespace object found in {args.manifest}") + print(findNamespaceName(loadManifestDocs(Path(args.manifest)))) def validate_manifest(args: argparse.Namespace) -> None: @@ -793,6 +989,7 @@ def main() -> int: kustomization.add_argument("--registry-prefix", required=True) kustomization.add_argument("--network-backend", default="macvlan") kustomization.add_argument("--cni-master-interface", default="") + kustomization.add_argument("--attached-cni-type", default=None) kustomization.add_argument("--output", required=True) kustomization.set_defaults(func=render_kustomization) diff --git a/seedemu/k8sTools/resources/running/manageRunningStage.py b/seedemu/k8sTools/resources/running/manageRunningStage.py index a261ffb93..3896579f7 100755 --- a/seedemu/k8sTools/resources/running/manageRunningStage.py +++ b/seedemu/k8sTools/resources/running/manageRunningStage.py @@ -70,6 +70,7 @@ def context(config: Path) -> dict[str, str]: "masterConnection", "cniMasterInterface", "networkBackend", + "attachedCniType", "rolloutTimeoutSeconds", ] return {key: helperOutput(["config-value", "--config", str(config), "--key", key]) for key in keys} @@ -141,6 +142,171 @@ def streamTarToRemote(source: Path, ssh_command: list[str], remote_extract_comma raise subprocess.CalledProcessError(return_code, ["tar", "-C", str(source), "-czf", "-", "."]) +def firstFromImage(dockerfile: Path) -> str | None: + """Return the first Dockerfile FROM image, handling optional flags. + + Args: + dockerfile: Dockerfile to inspect. + """ + for raw_line in dockerfile.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if not parts or parts[0].upper() != "FROM": + continue + index = 1 + while index < len(parts) and parts[index].startswith("--"): + index += 1 + if index < len(parts): + return parts[index] + return None + + +def isCompilerHashBaseTag(image: str) -> bool: + """Return true for generated compiler hash base tags. + + Args: + image: Docker image reference from a FROM line. + """ + tag = image.removesuffix(":latest") + return len(tag) == 32 and all(char in "0123456789abcdef" for char in tag) + + +def externalBuildBaseImages(output_dir: Path) -> list[str]: + """Collect non-generated Docker FROM images needed by remote buildx. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + """ + images: list[str] = [] + seen: set[str] = set() + for dockerfile in sorted(output_dir.rglob("Dockerfile")): + image = firstFromImage(dockerfile) + if not image: + continue + if image == "scratch" or image.startswith("$") or "${" in image: + continue + if isCompilerHashBaseTag(image): + continue + if image not in seen: + seen.add(image) + images.append(image) + return images + + +def dockerImageExists(image: str) -> bool: + """Return true when the local Docker daemon has one image reference. + + Args: + image: Docker image reference. + """ + return subprocess.run( + ["docker", "image", "inspect", image], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def dockerIoMirrorRef(image: str) -> str | None: + """Return a Docker Hub mirror reference for unqualified Docker Hub images. + + Args: + image: Docker image reference. + """ + if "/" in image: + first = image.split("/", 1)[0] + if "." in first or ":" in first or first == "localhost": + return None + return f"docker.m.daocloud.io/{image}" + return f"docker.m.daocloud.io/library/{image}" + + +def ensureLocalDockerImage(image: str) -> None: + """Ensure the local Docker daemon has one image, pulling if required. + + Args: + image: Docker image reference. + """ + if dockerImageExists(image): + print(f"[k8s_build] local Docker already has {image}") + return + + print(f"[k8s_build] pulling build base image on local host: {image}") + result = subprocess.run(["docker", "pull", image], check=False) + if result.returncode == 0: + return + + mirror_image = dockerIoMirrorRef(image) + if mirror_image is None: + raise subprocess.CalledProcessError(result.returncode, ["docker", "pull", image]) + + print(f"[k8s_build] pulling build base image from mirror: {mirror_image}") + runCommand(["docker", "pull", mirror_image]) + runCommand(["docker", "tag", mirror_image, image]) + + +def remoteDockerImageExists(image: str, ssh_command: list[str]) -> bool: + """Return true when the remote Docker daemon has one image reference. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + return subprocess.run( + [*ssh_command, f"sudo -n docker image inspect {shlex.quote(image)} >/dev/null 2>&1"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def streamDockerImageToRemote(image: str, ssh_command: list[str]) -> None: + """Load one local Docker image into the remote Docker daemon over SSH. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + if remoteDockerImageExists(image, ssh_command): + print(f"[k8s_build] registry host Docker already has {image}") + return + + ensureLocalDockerImage(image) + print("+ " + " ".join(["docker", "save", image]) + " | " + " ".join(ssh_command + ["sudo -n docker load"])) + producer = subprocess.Popen(["docker", "save", image], stdout=subprocess.PIPE) + try: + assert producer.stdout is not None + subprocess.run([*ssh_command, "sudo -n docker load"], stdin=producer.stdout, check=True) + finally: + if producer.stdout is not None: + producer.stdout.close() + return_code = producer.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, ["docker", "save", image]) + + +def ensureBuildBaseImages(output_dir: Path, ssh_command: list[str] | None) -> None: + """Ensure Dockerfile FROM images are available where buildx will run. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + ssh_command: Base SSH argv for a remote registry host, or None when the + registry/build host is local. + """ + images = externalBuildBaseImages(output_dir) + if not images: + return + print("[k8s_build] ensuring Docker build base images: " + ", ".join(images)) + if ssh_command is None: + for image in images: + ensureLocalDockerImage(image) + return + for image in images: + streamDockerImageToRemote(image, ssh_command) + + def buildImages(config: Path) -> None: """Stage compiler output on the registry node and build/push images. @@ -172,6 +338,7 @@ def buildImages(config: Path) -> None: (remote_root / "running").mkdir(parents=True) copyTreeContents(output_dir, remote_root / "output") copyTreeContents(SCRIPT_DIR, remote_root / "running") + ensureBuildBaseImages(output_dir, None) print("[k8s_build] running local build") runCommand(["sudo", "-n", *build_command], cwd=remote_root / "running") return @@ -192,6 +359,7 @@ def buildImages(config: Path) -> None: ssh_target, ] print(f"[k8s_build] uploading compile output to {ssh_target}:{remote_dir}/output") + ensureBuildBaseImages(output_dir, ssh_command) runCommand( [ *ssh_command, @@ -229,6 +397,8 @@ def renderKustomization(config: Path) -> dict[str, str]: values["networkBackend"], "--cni-master-interface", values["cniMasterInterface"], + "--attached-cni-type", + values["attachedCniType"], "--output", values["kustomization"], ] diff --git a/seedemu/k8sTools/resources/running/validateClusterPreflight.py b/seedemu/k8sTools/resources/running/validateClusterPreflight.py index 7efab1733..7bc9d8eb5 100755 --- a/seedemu/k8sTools/resources/running/validateClusterPreflight.py +++ b/seedemu/k8sTools/resources/running/validateClusterPreflight.py @@ -1,21 +1,226 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateClusterPreflight.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateClusterPreflight with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that compile output, kubeconfig, registry, remote Docker/buildx, and +# all Kubernetes nodes are ready before running make build/up. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +MANIFEST="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key manifest)" +IMAGES_YAML="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imagesYaml)" +KUBECONFIG_PATH="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key kubeconfig)" +REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +SSH_USER="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshUser)" +SSH_KEY="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshKey)" +MASTER_CONNECTION="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key masterConnection)" +NETWORK_BACKEND="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key networkBackend)" + +ssh_args=(-n -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8) + +fail() { + echo "[preflight][ERROR] $*" >&2 + exit 1 +} + +requireCommand() { + # Args: + # $1: command name expected in PATH + command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" +} + +requireFile() { + # Args: + # $1: required file path + [ -f "$1" ] || fail "missing file: $1" +} + +registry_ref="${REGISTRY_PREFIX#http://}" +registry_ref="${registry_ref#https://}" +registry_ref="${registry_ref%%/*}" +registry_host="${registry_ref%%:*}" +registry_url="http://${registry_ref}" + +declare -A node_config_ip=() +declare -A node_connection=() +declare -A node_user=() +declare -A node_key=() +while IFS=$'\t' read -r access_name access_ip access_connection access_user access_key; do + [ -n "${access_name}" ] || continue + node_config_ip["${access_name}"]="${access_ip}" + node_connection["${access_name}"]="${access_connection}" + node_user["${access_name}"]="${access_user}" + node_key["${access_name}"]="${access_key}" +done < <(python3 "${HELPER}" node-access --config "${CONFIG_PATH}") + +echo "=== native-k8s preflight ===" +echo "config=${CONFIG_PATH}" +echo "output_dir=${OUTPUT_DIR}" +echo "manifest=${MANIFEST}" +echo "images_yaml=${IMAGES_YAML}" +echo "kubeconfig=${KUBECONFIG_PATH}" +echo "registry_prefix=${REGISTRY_PREFIX}" +echo "image_registry_prefix=${IMAGE_REGISTRY_PREFIX}" +echo "network_backend=${NETWORK_BACKEND}" + +echo "[1/7] Local tools and compile output" +requireCommand python3 +requireCommand kubectl +requireCommand curl +requireCommand ssh +requireFile "${HELPER}" +requireFile "${MANIFEST}" +requireFile "${IMAGES_YAML}" +requireFile "${KUBECONFIG_PATH}" +[ -d "${OUTPUT_DIR}" ] || fail "missing output directory: ${OUTPUT_DIR}" +[ -s "${MANIFEST}" ] || fail "empty manifest: ${MANIFEST}" +[ -s "${IMAGES_YAML}" ] || fail "empty images yaml: ${IMAGES_YAML}" + +namespace="$(python3 "${HELPER}" namespace --manifest "${MANIFEST}")" +[ -n "${namespace}" ] || fail "cannot determine namespace from ${MANIFEST}" +image_count="$(python3 "${HELPER}" mapped-images --images-yaml "${IMAGES_YAML}" --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" --registry-prefix "${REGISTRY_PREFIX}" | wc -l)" +[ "${image_count}" -gt 0 ] || fail "no images found in ${IMAGES_YAML}" +echo "namespace=${namespace}" +echo "image_count=${image_count}" + +echo "[2/7] Kubeconfig and API server" +kubectl --kubeconfig "${KUBECONFIG_PATH}" version --client=true >/dev/null +kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide + +echo "[3/7] Node readiness" +not_ready="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ + | awk '$2 != "True" {print $1}' +)" +[ -z "${not_ready}" ] || fail "not-ready nodes: ${not_ready//$'\n'/ }" + +echo "[4/7] kube-system baseline" +kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -o wide +bad_system_pods="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods --no-headers 2>/dev/null \ + | awk '$3 !~ /^(Running|Completed)$/ {print $1 ":" $3}' +)" +[ -z "${bad_system_pods}" ] || fail "kube-system has unhealthy pods: ${bad_system_pods//$'\n'/ }" +if [ "${NETWORK_BACKEND}" = "kube-ovn" ]; then + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status deployment/kube-ovn-controller --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/kube-ovn-cni --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/ovs-ovn --timeout=300s +fi + +echo "[5/7] Namespace baseline" +if kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; then + workload_count="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" \ + get pods,deployments.apps,replicasets.apps,statefulsets.apps,daemonsets.apps,jobs.batch,cronjobs.batch \ + --ignore-not-found --no-headers 2>/dev/null | wc -l + )" + if [ "${workload_count}" -gt 0 ]; then + fail "namespace ${namespace} already has ${workload_count} workload resources; run make clean or wait for previous cleanup before make up" + fi + echo "namespace ${namespace} already exists without workload resources; continuing" +else + echo "namespace ${namespace} is absent" +fi + +echo "[6/7] Registry health" +code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" +case "${code}" in + 200|401) echo "local registry_http=${code}" ;; + *) fail "registry ${registry_url}/v2/ is not healthy from local host, http=${code}" ;; +esac + +ssh_target="${SSH_USER}@${registry_host}" +if [ "${MASTER_CONNECTION}" = "local" ]; then + echo "registry-host local registry_http=${code}" +else + requireFile "${SSH_KEY}" + ssh "${ssh_args[@]}" "${ssh_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/'" \ + | awk '{print "registry-host registry_http="$0; if ($0 != "200" && $0 != "401") exit 1}' \ + || fail "registry ${registry_url}/v2/ is not healthy from ${ssh_target}" +fi + +echo "[7/7] Remote build prerequisites and node registry reachability" +if [ "${MASTER_CONNECTION}" = "local" ]; then + sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null \ + || fail "docker/buildx is not ready on local registry host" +else + ssh "${ssh_args[@]}" "${ssh_target}" "sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null" \ + || fail "docker/buildx is not ready on ${ssh_target}" +fi +echo "registry-host docker/buildx ok" + +while IFS=$'\t' read -r node_name node_ip; do + [ -n "${node_name}" ] || continue + node_access="${node_connection[$node_name]:-}" + [ -n "${node_access}" ] || fail "node ${node_name} is not present in configK3s.yaml" + target_ip="${node_ip:-${node_config_ip[$node_name]}}" + [ -n "${target_ip}" ] || fail "cannot determine InternalIP for node ${node_name}" + if [ "${node_access}" = "local" ]; then + code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" + else + requireFile "${node_key[$node_name]}" + node_target="${node_user[$node_name]}@${target_ip}" + node_ssh_args=(-n -i "${node_key[$node_name]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=8) + code="$(ssh "${node_ssh_args[@]}" "${node_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/' || true")" + fi + printf '%s\t%s\tregistry_http=%s\n' "${node_name}" "${node_ip}" "${code}" + case "${code}" in + 200|401) ;; + *) fail "registry ${registry_url}/v2/ is not reachable from ${node_name} (${node_ip}), http=${code}" ;; + esac +done < <( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' +) + +echo "Preflight completed" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/running/validateClusterPreflight.sh b/seedemu/k8sTools/resources/running/validateClusterPreflight.sh deleted file mode 100755 index 226a52468..000000000 --- a/seedemu/k8sTools/resources/running/validateClusterPreflight.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -# Validate that compile output, kubeconfig, registry, remote Docker/buildx, and -# all Kubernetes nodes are ready before running make build/up. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" -HELPER="${SCRIPT_DIR}/manageK8sManifest.py" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" -MANIFEST="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key manifest)" -IMAGES_YAML="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imagesYaml)" -KUBECONFIG_PATH="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key kubeconfig)" -REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" -IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" -SSH_USER="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshUser)" -SSH_KEY="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshKey)" -MASTER_CONNECTION="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key masterConnection)" -NETWORK_BACKEND="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key networkBackend)" - -ssh_args=(-n -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8) - -fail() { - echo "[preflight][ERROR] $*" >&2 - exit 1 -} - -requireCommand() { - # Args: - # $1: command name expected in PATH - command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" -} - -requireFile() { - # Args: - # $1: required file path - [ -f "$1" ] || fail "missing file: $1" -} - -registry_ref="${REGISTRY_PREFIX#http://}" -registry_ref="${registry_ref#https://}" -registry_ref="${registry_ref%%/*}" -registry_host="${registry_ref%%:*}" -registry_url="http://${registry_ref}" - -declare -A node_config_ip=() -declare -A node_connection=() -declare -A node_user=() -declare -A node_key=() -while IFS=$'\t' read -r access_name access_ip access_connection access_user access_key; do - [ -n "${access_name}" ] || continue - node_config_ip["${access_name}"]="${access_ip}" - node_connection["${access_name}"]="${access_connection}" - node_user["${access_name}"]="${access_user}" - node_key["${access_name}"]="${access_key}" -done < <(python3 "${HELPER}" node-access --config "${CONFIG_PATH}") - -echo "=== native-k8s preflight ===" -echo "config=${CONFIG_PATH}" -echo "output_dir=${OUTPUT_DIR}" -echo "manifest=${MANIFEST}" -echo "images_yaml=${IMAGES_YAML}" -echo "kubeconfig=${KUBECONFIG_PATH}" -echo "registry_prefix=${REGISTRY_PREFIX}" -echo "image_registry_prefix=${IMAGE_REGISTRY_PREFIX}" -echo "network_backend=${NETWORK_BACKEND}" - -echo "[1/7] Local tools and compile output" -requireCommand python3 -requireCommand kubectl -requireCommand curl -requireCommand ssh -requireFile "${HELPER}" -requireFile "${MANIFEST}" -requireFile "${IMAGES_YAML}" -requireFile "${KUBECONFIG_PATH}" -[ -d "${OUTPUT_DIR}" ] || fail "missing output directory: ${OUTPUT_DIR}" -[ -s "${MANIFEST}" ] || fail "empty manifest: ${MANIFEST}" -[ -s "${IMAGES_YAML}" ] || fail "empty images yaml: ${IMAGES_YAML}" - -namespace="$(python3 "${HELPER}" namespace --manifest "${MANIFEST}")" -[ -n "${namespace}" ] || fail "cannot determine namespace from ${MANIFEST}" -image_count="$(python3 "${HELPER}" mapped-images --images-yaml "${IMAGES_YAML}" --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" --registry-prefix "${REGISTRY_PREFIX}" | wc -l)" -[ "${image_count}" -gt 0 ] || fail "no images found in ${IMAGES_YAML}" -echo "namespace=${namespace}" -echo "image_count=${image_count}" - -echo "[2/7] Kubeconfig and API server" -kubectl --kubeconfig "${KUBECONFIG_PATH}" version --client=true >/dev/null -kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide - -echo "[3/7] Node readiness" -not_ready="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ - | awk '$2 != "True" {print $1}' -)" -[ -z "${not_ready}" ] || fail "not-ready nodes: ${not_ready//$'\n'/ }" - -echo "[4/7] kube-system baseline" -kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -o wide -bad_system_pods="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods --no-headers 2>/dev/null \ - | awk '$3 !~ /^(Running|Completed)$/ {print $1 ":" $3}' -)" -[ -z "${bad_system_pods}" ] || fail "kube-system has unhealthy pods: ${bad_system_pods//$'\n'/ }" -if [ "${NETWORK_BACKEND}" = "kube-ovn" ]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s - kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status deployment/kube-ovn-controller --timeout=300s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/kube-ovn-cni --timeout=300s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/ovs-ovn --timeout=300s -fi - -echo "[5/7] Namespace baseline" -if kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; then - fail "namespace ${namespace} already exists; run make clean or wait for previous cleanup before make up" -fi -echo "namespace ${namespace} is absent" - -echo "[6/7] Registry health" -code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" -case "${code}" in - 200|401) echo "local registry_http=${code}" ;; - *) fail "registry ${registry_url}/v2/ is not healthy from local host, http=${code}" ;; -esac - -ssh_target="${SSH_USER}@${registry_host}" -if [ "${MASTER_CONNECTION}" = "local" ]; then - echo "registry-host local registry_http=${code}" -else - requireFile "${SSH_KEY}" - ssh "${ssh_args[@]}" "${ssh_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/'" \ - | awk '{print "registry-host registry_http="$0; if ($0 != "200" && $0 != "401") exit 1}' \ - || fail "registry ${registry_url}/v2/ is not healthy from ${ssh_target}" -fi - -echo "[7/7] Remote build prerequisites and node registry reachability" -if [ "${MASTER_CONNECTION}" = "local" ]; then - sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null \ - || fail "docker/buildx is not ready on local registry host" -else - ssh "${ssh_args[@]}" "${ssh_target}" "sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null" \ - || fail "docker/buildx is not ready on ${ssh_target}" -fi -echo "registry-host docker/buildx ok" - -while IFS=$'\t' read -r node_name node_ip; do - [ -n "${node_name}" ] || continue - node_access="${node_connection[$node_name]:-}" - [ -n "${node_access}" ] || fail "node ${node_name} is not present in configK3s.yaml" - target_ip="${node_ip:-${node_config_ip[$node_name]}}" - [ -n "${target_ip}" ] || fail "cannot determine InternalIP for node ${node_name}" - if [ "${node_access}" = "local" ]; then - code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" - else - requireFile "${node_key[$node_name]}" - node_target="${node_user[$node_name]}@${target_ip}" - node_ssh_args=(-n -i "${node_key[$node_name]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=8) - code="$(ssh "${node_ssh_args[@]}" "${node_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/' || true")" - fi - printf '%s\t%s\tregistry_http=%s\n' "${node_name}" "${node_ip}" "${code}" - case "${code}" in - 200|401) ;; - *) fail "registry ${registry_url}/v2/ is not reachable from ${node_name} (${node_ip}), http=${code}" ;; - esac -done < <( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' -) - -echo "Preflight completed" diff --git a/seedemu/k8sTools/resources/setup/README.md b/seedemu/k8sTools/resources/setup/README.md index 949e716ea..b8cd8cbb4 100644 --- a/seedemu/k8sTools/resources/setup/README.md +++ b/seedemu/k8sTools/resources/setup/README.md @@ -2,12 +2,12 @@ 这个目录是 `seedemu.k8sTools` 的内部资源目录。用户通常不会直接看到它,因为 `K8sTools.build()` 会把这些资源复制到临时目录中执行,完成后只保留用户指定的 -`configK3s.yaml` 和 `kubeconfig.yaml`。 +`configK3s.yaml`、`kubeconfig.yaml`,以及可选的 `inventory.yaml`。 -所有用户侧入口都是 Python 文件;每个迁移中的系统操作入口旁边保留同名 `.sh` -资源,Python 入口只负责调用相邻 shell。这样 shell 命令序列可以正常 review 和 -`bash -n` 检查。它们仍会调用系统工具,例如 `virsh`、`docker`、`kubectl`、 -`ssh`、`ansible-playbook` 和 `helm`;这些命令才是真正的 KVM、K3s、OVN/OVS +所有用户侧和内部操作入口都是 Python 文件。部分复杂系统操作仍保留原 shell +命令序列,但正文内嵌在对应 Python 入口中,不再依赖相邻 `.sh` 资源文件。 +这些入口仍会调用系统工具,例如 `virsh`、`docker`、`kubectl`、`ssh`、 +`ansible-playbook` 和 `helm`;这些命令才是真正的 KVM、K3s、OVN/OVS 和镜像操作接口。 ## 资源分组 @@ -74,7 +74,7 @@ ovn/installKubeOvnFabric.py | `kubeconfig.yaml` | `build` 的持久输出,供 `kubectl` 和 `up/down` 使用。 | | `kvmState.yaml` | 临时 KVM 清理状态,会嵌入最终 `configK3s.yaml` 的 `k8sTools.destroy` 元数据中。 | | `multiHostKvmState.yaml` | 临时多物理机 KVM 清理状态,会嵌入最终 `configK3s.yaml`。 | -| `seedemu-k3s.inventory.yaml` | 临时解释性 inventory;当前 `k8sTools` 默认不把它作为用户主产物。 | +| `inventory.yaml` | 可选持久集群 inventory;通过 `k8sTools.py build --inventory inventory.yaml` 写出,包含节点角色、管理 IP 和 CPU/memory/disk 容量。 | ## 手动调试提示 diff --git a/seedemu/k8sTools/resources/setup/_embeddedShell.py b/seedemu/k8sTools/resources/setup/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml index d7ba828d3..16ffa5f2e 100644 --- a/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml +++ b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml @@ -140,6 +140,12 @@ retries: 30 delay: 5 + - name: Fetch K3s binary for offline worker installs + fetch: + src: /usr/local/bin/k3s + dest: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + flat: yes + - name: Install CNI plugins required by Multus (macvlan/ipvlan/static) shell: | set -euo pipefail @@ -359,7 +365,7 @@ command: - /thin_entrypoint args: - - --multus-conf-file=auto + - --multus-conf-file=/tmp/multus-conf/00-multus.conf - --multus-autoconfig-dir=/host/etc/cni/net.d - --cni-conf-dir=/host/etc/cni/net.d resources: @@ -411,7 +417,7 @@ name: multus-cni-config items: - key: cni-conf.json - path: 70-multus.conf + path: 00-multus.conf - name: Install Multus CNI shell: k3s kubectl apply -f /tmp/seedemu-multus-daemonset.yml @@ -548,9 +554,17 @@ state: restarted when: k3s_agent_dir.stat.exists and k3s_agent_config.changed + - name: Copy K3s binary prepared on master to worker + copy: + src: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + dest: /usr/local/bin/k3s + mode: "0755" + when: not k3s_agent_dir.stat.exists or seed_k3s_force_reinstall | bool + - name: Install K3s agent shell: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_install_version }} \ + curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_DOWNLOAD=true \ + INSTALL_K3S_VERSION={{ k3s_install_version }} \ INSTALL_K3S_ARTIFACT_URL={{ seed_k3s_artifact_url }} \ K3S_URL=https://{{ hostvars[groups['master'][0]]['ansible_host'] }}:6443 \ K3S_TOKEN={{ hostvars[groups['master'][0]]['k3s_server_token'] }} \ diff --git a/seedemu/k8sTools/resources/setup/applyK3sCluster.py b/seedemu/k8sTools/resources/setup/applyK3sCluster.py index 1ef62c211..962fbcd7b 100755 --- a/seedemu/k8sTools/resources/setup/applyK3sCluster.py +++ b/seedemu/k8sTools/resources/setup/applyK3sCluster.py @@ -1,21 +1,655 @@ #!/usr/bin/env python3 -"""Python entrypoint for applyK3sCluster.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for applyK3sCluster with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Build a K3s cluster from configK3s.yaml. The script installs K3s with the +# bundled Ansible playbook, creates the master registry, imports bootstrap +# images, writes kubeconfig/inventory outputs, and validates node readiness. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" +if [ -f "${SCRIPT_DIR}/ansible/k3s-install.yml" ]; then + PLAYBOOK_PATH="${SCRIPT_DIR}/ansible/k3s-install.yml" +else + PLAYBOOK_PATH="${REPO_ROOT}/ansible/k3s-install.yml" +fi +CONFIG_PATH="${1:-${SCRIPT_DIR}/configK3s.yaml}" +ansibleTimeout="3600s" + +# Public bootstrap images are prepared on the host and then copied into VMs. +# Do not make fresh VMs pull these images from the public internet during setup. +HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" +HOST_DOCKER_IO_MIRROR="docker.m.daocloud.io" +dockerPullTimeoutSeconds=180 +REGISTRY_BOOTSTRAP_IMAGE="registry:2" +MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" +K3S_SYSTEM_BOOTSTRAP_IMAGES=( + "rancher/mirrored-coredns-coredns:1.10.1" + "rancher/mirrored-metrics-server:v0.6.3" + "rancher/local-path-provisioner:v0.0.24" +) +seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" +seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" +seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" +seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" +seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" +ubuntuBuildImage="ubuntu:20.04" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveInput() { + if [ "${CONFIG_PATH}" = "-h" ] || [ "${CONFIG_PATH}" = "--help" ]; then + usage + exit 0 + fi + if [ ! -s "${CONFIG_PATH}" ]; then + echo "configK3s.yaml not found or empty: ${CONFIG_PATH}" >&2 + exit 1 + fi +} + +helper() { + local command="$1" + shift + python3 "${HELPER}" --config "${CONFIG_PATH}" "${command}" "$@" +} + +loadConfigVars() { + local vars_output + if ! vars_output="$(helper shell-vars)"; then + return 1 + fi + eval "${vars_output}" + MASTER_IS_LOCAL=false + if [ "${k3sMasterConnection:-ssh}" = "local" ]; then + MASTER_IS_LOCAL=true + fi + SSH_OPTS=( + -i "${k3sSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +loadNodeSshContext() { + # Args: + # $1: node name from configK3s.yaml. + # Reads per-node ssh.user/key and prepares NODE_SSH_OPTS for node-specific + # SSH/scp operations. Master-only operations keep using SSH_OPTS. + local name="$1" + eval "$(helper node-ssh-vars --name "${name}")" + if [ "${nodeConnection:-ssh}" = "local" ]; then + NODE_SSH_OPTS=() + return 0 + fi + [ -f "${nodeSshKey}" ] || { + echo "SSH key not found for ${name}: ${nodeSshKey}" >&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +resolveSeedEmulatorDockerDir() { + # Resolve the host path containing seedemu-base and seedemu-router + # Dockerfiles. Existing YAML value wins; common source-tree locations are + # tried as fallbacks for physical-server workflows. + local cursor="${SCRIPT_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch" \ + "${REPO_ROOT}/../seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +runMasterCommand() { + # Args: + # $1: shell command to run on the K3s master. + # Uses local execution when configK3s.yaml points the master at this host; + # otherwise uses the configured master SSH account. + local command="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + bash -lc "${command}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "${command}" + fi +} + +copyFileToMaster() { + # Args: + # $1: local source path. + # $2: destination path on the master. + local source="$1" + local target="$2" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + cp "${source}" "${target}" + else + scp "${SSH_OPTS[@]}" "${source}" "${k3sUser}@${k3sMasterIp}:${target}" >/dev/null + fi +} + +readMasterFile() { + # Args: + # $1: absolute file path to read from the master with sudo. + local path="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + sudo -n cat "${path}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "sudo -n cat '${path}'" + fi +} + +runNodeCommand() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: shell command to run on that node. + local name="$1" + local ip="$2" + local command="$3" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + bash -lc "${command}" + else + ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "${command}" + fi +} + +copyFileToNode() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: local source path. + # $4: destination path on the node. + local name="$1" + local ip="$2" + local source="$3" + local target="$4" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + cp "${source}" "${target}" + else + scp "${NODE_SSH_OPTS[@]}" "${source}" "${nodeSshUser}@${ip}:${target}" >/dev/null + fi +} + +printPlan() { + echo "config=${CONFIG_PATH}" + echo "K3s node plan:" + helper nodes-tsv | awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' + echo "master=${k3sMasterName} (${k3sMasterIp})" + echo "master_connection=${k3sMasterConnection:-ssh}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + echo "kubeconfig=${outputKubeconfig}" +} + +verifyConnectivity() { + echo "[1/10] Verifying SSH and sudo on all nodes" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + echo " ${name} ${ip}" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + runWithTimeout 12s bash -lc "echo local-ok >/dev/null" + runWithTimeout 12s sudo -n true >/dev/null + else + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ssh-ok" >/dev/null + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n true" >/dev/null + fi + done < <(helper nodes-tsv) +} + +runAnsibleInstall() { + echo "[2/10] Installing K3s via generated Ansible inventory" + local inventory_tmp playbook_tmp node_count + mkdir -p "${setupTmpDir}" + inventory_tmp="$(mktemp "${setupTmpDir}/ansible-inventory.XXXXXX.yml")" + playbook_tmp="$(mktemp "${setupTmpDir}/k3s-install.XXXXXX.yml")" + helper write-ansible-inventory --output "${inventory_tmp}" >/dev/null + node_count="$(helper nodes-tsv | wc -l | tr -d ' ')" + sed "s/ready_nodes.stdout | int >= 3/ready_nodes.stdout | int >= ${node_count}/" \ + "${PLAYBOOK_PATH}" > "${playbook_tmp}" + ANSIBLE_HOST_KEY_CHECKING=False \ + runWithTimeout "${ansibleTimeout}" \ + ansible-playbook \ + -i "${inventory_tmp}" \ + "${playbook_tmp}" \ + --ssh-common-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none" + rm -f "${inventory_tmp}" "${playbook_tmp}" +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + local mirror_ref="${HOST_DOCKER_IO_MIRROR}" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${mirror_ref}" "${image}" + else + printf '%s/library/%s\n' "${mirror_ref}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " host docker pull ${image}" >&2 + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " host docker pull ${mirror_image}" >&2 + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare image on host: ${image}" >&2 + echo "This script intentionally does not ask the VM to pull public images." >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + ensureHostDockerImage "${image}" + if [ -s "${tar_path}" ]; then + printf '%s\n' "${tar_path}" + return 0 + fi + echo " host docker save ${image} -> ${tar_path}" >&2 + docker save -o "${tar_path}" "${image}" + printf '%s\n' "${tar_path}" +} + +loadDockerImageToMaster() { + local image="$1" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + remote_tar="/tmp/$(basename "${tar_path}")" + echo " copy ${image} to ${k3sMasterName}:${remote_tar}" + copyFileToMaster "${tar_path}" "${remote_tar}" + runMasterCommand "sudo -n docker load -i '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +importK3sImageToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${node_name}" "${node_ip}" "${tar_path}" +} + +importK3sImageTarToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path="$4" + local remote_tar + remote_tar="/tmp/$(basename "${tar_path}")" + echo " import ${image} to ${node_name}" + copyFileToNode "${node_name}" "${node_ip}" "${tar_path}" "${remote_tar}" + runNodeCommand "${node_name}" "${node_ip}" \ + "sudo -n k3s ctr -n k8s.io images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +ensureRegistry() { + echo "[3/10] Ensuring private registry on master" + echo " preparing ${REGISTRY_BOOTSTRAP_IMAGE} on host and loading it into master Docker" + loadDockerImageToMaster "${REGISTRY_BOOTSTRAP_IMAGE}" + runMasterCommand " + set -euo pipefail + if ! docker buildx version >/dev/null 2>&1; then + sudo -n apt-get update >/dev/null + sudo -n apt-get install -y docker-buildx >/dev/null + fi + sudo -n docker rm -f registry >/dev/null 2>&1 || true + sudo -n docker image inspect '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + sudo -n docker run -d --network host --restart=always --name registry \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:${registryPort} '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + " +} + +ensureSeedemuHostBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedBaseSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedRouterSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareMasterWorkloadBuildImages() { + echo "[4/10] Preparing workload build base images on master Docker" + [ -d "${seedEmulatorDockerDir}/seedemu-base" ] || { + echo "Missing host seedemu base image directory: ${seedEmulatorDockerDir}/seedemu-base" >&2 + exit 1 + } + [ -d "${seedEmulatorDockerDir}/seedemu-router" ] || { + echo "Missing host seedemu router image directory: ${seedEmulatorDockerDir}/seedemu-router" >&2 + exit 1 + } + + ensureSeedemuHostBuildImages + loadDockerImageToMaster "${ubuntuBuildImage}" + loadDockerImageToMaster "${seedBaseSourceImage}" + loadDockerImageToMaster "${seedRouterSourceImage}" + + echo " ensuring stable compiler hash tags on master Docker" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' '${seedBaseHashImage}' + sudo -n docker tag '${seedRouterSourceImage}' '${seedRouterHashImage}' + " + + echo " pushing seedemu base/router images into master local registry" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' \ + '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker tag '${seedRouterSourceImage}' \ + '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + " +} + +fetchKubeconfig() { + echo "[5/10] Fetching kubeconfig" + mkdir -p "$(dirname "${outputKubeconfig}")" + readMasterFile "/etc/rancher/k3s/k3s.yaml" > "${outputKubeconfig}" + sed -i "s|127.0.0.1|${k3sMasterIp}|g" "${outputKubeconfig}" + echo "kubeconfig=${outputKubeconfig}" +} + +preloadK3sBootstrapImagesAllNodes() { + echo "[6/10] Preloading K3s system and Multus images from host into all K3s containerd nodes" + local preload_node_concurrency="${SEED_K8S_PRELOAD_NODE_CONCURRENCY:-3}" + local preload_failures=0 + if ! [[ "${preload_node_concurrency}" =~ ^[0-9]+$ ]] || [ "${preload_node_concurrency}" -lt 1 ]; then + echo "Invalid SEED_K8S_PRELOAD_NODE_CONCURRENCY: ${preload_node_concurrency}" >&2 + exit 1 + fi + echo " preload node concurrency=${preload_node_concurrency}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + saveHostImageTarball "${image}" >/dev/null + done + activePreloadJobs() { + jobs -rp | wc -l | tr -d ' ' + } + preloadNodeImages() { + local name="$1" + local ip="$2" + local image tar_path + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + tar_path="$(hostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${name}" "${ip}" "${tar_path}" + done + } + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + preloadNodeImages "${name}" "${ip}" & + while [ "$(activePreloadJobs)" -ge "${preload_node_concurrency}" ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + done < <(helper nodes-tsv) + while [ "$(activePreloadJobs)" -gt 0 ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + if [ "${preload_failures}" -ne 0 ]; then + echo "K3s bootstrap image preload failed on ${preload_failures} node(s)" >&2 + exit 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=kube-dns \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=metrics-server \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l app=local-path-provisioner \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l name=multus \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true +} + +installOvnFabricIfConfigured() { + # Install Kube-OVN after K3s and Multus are available. OVN is a Kubernetes + # CNI add-on, so it cannot be prepared in the physical-node preflight stage. + if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + return 0 + fi + echo "[7/10] Installing Kube-OVN non-primary CNI" + python3 "${SCRIPT_DIR}/ovn/installKubeOvnFabric.py" "${CONFIG_PATH}" +} + +applyNodeK3sTuning() { + local name="$1" + local ip="$2" + local remote_runner=() + echo " tuning K3s on ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${k3sMaxPods}" \ + "${kubeletRegistryQps}" \ + "${kubeletRegistryBurst}" \ + "${rebootAfterTuning}" <<'EOF_REMOTE' +set -euo pipefail +MAX_PODS="$1" +REGISTRY_QPS="$2" +REGISTRY_BURST="$3" +ASYNC_RESTART="$4" + +if [ ! -f /etc/sysctl.d/99-seed-vm-limits.conf ]; then + echo "warning: /etc/sysctl.d/99-seed-vm-limits.conf not found; run tuneVmLimits.py before large-scale experiments" >&2 +fi + +mkdir -p /etc/rancher/k3s +touch /etc/rancher/k3s/config.yaml +cfg=/etc/rancher/k3s/config.yaml +tmp=$(mktemp) +awk ' + /^[^[:space:]].*:/ { + if ($0 ~ /^(kubelet-arg|kube-apiserver-arg):/) {skip=1; next} + skip=0 + } + skip == 0 {print} +' "${cfg}" > "${tmp}" +cat >> "${tmp}" <> "${tmp}" <<'EOF_MASTER' +kube-apiserver-arg: + - "max-requests-inflight=1000" + - "max-mutating-requests-inflight=500" +EOF_MASTER +fi +cat "${tmp}" > "${cfg}" +rm -f "${tmp}" + +if [ "${ASYNC_RESTART}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true; systemctl restart containerd 2>/dev/null || true" >/dev/null 2>&1 || true +else + systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true + systemctl restart containerd 2>/dev/null || true +fi +EOF_REMOTE +} + +applyTuningAllNodes() { + echo "[8/10] Applying K3s runtime tuning" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + applyNodeK3sTuning "${name}" "${ip}" + done < <(helper nodes-tsv) +} + +verifyCluster() { + echo "[9/10] Waiting for K3s nodes" + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Ready node --all --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system rollout status daemonset/kube-multus-ds --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o wide + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tPodCIDR: "}{.spec.podCIDR}{"\n"}{end}' + echo +} + +writeOutputs() { + echo "[10/10] Writing inventory output" + helper write-cluster-inventory >/dev/null + echo "inventory=${outputInventory}" + echo "kubeconfig=${outputKubeconfig}" + echo "registry=${registryHost}:${registryPort}" +} + +main() { + requireCommand python3 + requireCommand ansible-playbook + requireCommand docker + requireCommand scp + requireCommand ssh + requireCommand kubectl + requireCommand sed + [ -f "${PLAYBOOK_PATH}" ] || { + echo "K3s Ansible playbook not found: ${PLAYBOOK_PATH}" >&2 + exit 1 + } + + resolveInput + loadConfigVars + resolveSeedEmulatorDockerDir + + if [ "${MASTER_IS_LOCAL}" != "true" ] && [ ! -f "${k3sSshKey}" ]; then + echo "SSH key not found: ${k3sSshKey}" >&2 + exit 1 + fi + + printPlan + verifyConnectivity + runAnsibleInstall + ensureRegistry + prepareMasterWorkloadBuildImages + fetchKubeconfig + preloadK3sBootstrapImagesAllNodes + installOvnFabricIfConfigured + applyTuningAllNodes + verifyCluster + writeOutputs + echo "K3s cluster is ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/applyK3sCluster.sh b/seedemu/k8sTools/resources/setup/applyK3sCluster.sh deleted file mode 100755 index 7b61ac728..000000000 --- a/seedemu/k8sTools/resources/setup/applyK3sCluster.sh +++ /dev/null @@ -1,589 +0,0 @@ -#!/usr/bin/env bash -# Build a K3s cluster from configK3s.yaml. The script installs K3s with the -# bundled Ansible playbook, creates the master registry, imports bootstrap -# images, writes kubeconfig/inventory outputs, and validates node readiness. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" -if [ -f "${SCRIPT_DIR}/ansible/k3s-install.yml" ]; then - PLAYBOOK_PATH="${SCRIPT_DIR}/ansible/k3s-install.yml" -else - PLAYBOOK_PATH="${REPO_ROOT}/ansible/k3s-install.yml" -fi -CONFIG_PATH="${1:-${SCRIPT_DIR}/configK3s.yaml}" -ansibleTimeout="3600s" - -# Public bootstrap images are prepared on the host and then copied into VMs. -# Do not make fresh VMs pull these images from the public internet during setup. -HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" -HOST_DOCKER_IO_MIRROR="docker.m.daocloud.io" -dockerPullTimeoutSeconds=180 -REGISTRY_BOOTSTRAP_IMAGE="registry:2" -MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" -K3S_SYSTEM_BOOTSTRAP_IMAGES=( - "rancher/mirrored-coredns-coredns:1.10.1" - "rancher/mirrored-metrics-server:v0.6.3" - "rancher/local-path-provisioner:v0.0.24" -) -seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" -seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" -seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" -seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" -seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" -ubuntuBuildImage="ubuntu:20.04" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -resolveInput() { - if [ "${CONFIG_PATH}" = "-h" ] || [ "${CONFIG_PATH}" = "--help" ]; then - usage - exit 0 - fi - if [ ! -s "${CONFIG_PATH}" ]; then - echo "configK3s.yaml not found or empty: ${CONFIG_PATH}" >&2 - exit 1 - fi -} - -helper() { - local command="$1" - shift - python3 "${HELPER}" --config "${CONFIG_PATH}" "${command}" "$@" -} - -loadConfigVars() { - local vars_output - if ! vars_output="$(helper shell-vars)"; then - return 1 - fi - eval "${vars_output}" - MASTER_IS_LOCAL=false - if [ "${k3sMasterConnection:-ssh}" = "local" ]; then - MASTER_IS_LOCAL=true - fi - SSH_OPTS=( - -i "${k3sSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -loadNodeSshContext() { - # Args: - # $1: node name from configK3s.yaml. - # Reads per-node ssh.user/key and prepares NODE_SSH_OPTS for node-specific - # SSH/scp operations. Master-only operations keep using SSH_OPTS. - local name="$1" - eval "$(helper node-ssh-vars --name "${name}")" - if [ "${nodeConnection:-ssh}" = "local" ]; then - NODE_SSH_OPTS=() - return 0 - fi - [ -f "${nodeSshKey}" ] || { - echo "SSH key not found for ${name}: ${nodeSshKey}" >&2 - exit 1 - } - NODE_SSH_OPTS=( - -i "${nodeSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -resolveSeedEmulatorDockerDir() { - # Resolve the host path containing seedemu-base and seedemu-router - # Dockerfiles. Existing YAML value wins; common source-tree locations are - # tried as fallbacks for physical-server workflows. - local cursor="${SCRIPT_DIR}" - while [ "${cursor}" != "/" ]; do - if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && - [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then - seedEmulatorDockerDir="${cursor}/docker_images/multiarch" - return 0 - fi - cursor="$(dirname "${cursor}")" - done - - local candidate - for candidate in \ - "${seedEmulatorDockerDir}" \ - "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ - "${HOME}/k8s/seed-emulator/docker_images/multiarch" \ - "${REPO_ROOT}/../seed-emulator/docker_images/multiarch"; do - if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then - seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" - return 0 - fi - done -} - -runMasterCommand() { - # Args: - # $1: shell command to run on the K3s master. - # Uses local execution when configK3s.yaml points the master at this host; - # otherwise uses the configured master SSH account. - local command="$1" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - bash -lc "${command}" - else - ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "${command}" - fi -} - -copyFileToMaster() { - # Args: - # $1: local source path. - # $2: destination path on the master. - local source="$1" - local target="$2" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - cp "${source}" "${target}" - else - scp "${SSH_OPTS[@]}" "${source}" "${k3sUser}@${k3sMasterIp}:${target}" >/dev/null - fi -} - -readMasterFile() { - # Args: - # $1: absolute file path to read from the master with sudo. - local path="$1" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - sudo -n cat "${path}" - else - ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "sudo -n cat '${path}'" - fi -} - -runNodeCommand() { - # Args: - # $1: node name from configK3s.yaml. - # $2: node management IP. - # $3: shell command to run on that node. - local name="$1" - local ip="$2" - local command="$3" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - bash -lc "${command}" - else - ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "${command}" - fi -} - -copyFileToNode() { - # Args: - # $1: node name from configK3s.yaml. - # $2: node management IP. - # $3: local source path. - # $4: destination path on the node. - local name="$1" - local ip="$2" - local source="$3" - local target="$4" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - cp "${source}" "${target}" - else - scp "${NODE_SSH_OPTS[@]}" "${source}" "${nodeSshUser}@${ip}:${target}" >/dev/null - fi -} - -printPlan() { - echo "config=${CONFIG_PATH}" - echo "K3s node plan:" - helper nodes-tsv | awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' - echo "master=${k3sMasterName} (${k3sMasterIp})" - echo "master_connection=${k3sMasterConnection:-ssh}" - echo "seedemu_docker_dir=${seedEmulatorDockerDir}" - echo "kubeconfig=${outputKubeconfig}" -} - -verifyConnectivity() { - echo "[1/10] Verifying SSH and sudo on all nodes" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - echo " ${name} ${ip}" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - runWithTimeout 12s bash -lc "echo local-ok >/dev/null" - runWithTimeout 12s sudo -n true >/dev/null - else - runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ssh-ok" >/dev/null - runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n true" >/dev/null - fi - done < <(helper nodes-tsv) -} - -runAnsibleInstall() { - echo "[2/10] Installing K3s via generated Ansible inventory" - local inventory_tmp playbook_tmp node_count - mkdir -p "${setupTmpDir}" - inventory_tmp="$(mktemp "${setupTmpDir}/ansible-inventory.XXXXXX.yml")" - playbook_tmp="$(mktemp "${setupTmpDir}/k3s-install.XXXXXX.yml")" - helper write-ansible-inventory --output "${inventory_tmp}" >/dev/null - node_count="$(helper nodes-tsv | wc -l | tr -d ' ')" - sed "s/ready_nodes.stdout | int >= 3/ready_nodes.stdout | int >= ${node_count}/" \ - "${PLAYBOOK_PATH}" > "${playbook_tmp}" - ANSIBLE_HOST_KEY_CHECKING=False \ - runWithTimeout "${ansibleTimeout}" \ - ansible-playbook \ - -i "${inventory_tmp}" \ - "${playbook_tmp}" \ - --ssh-common-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none" - rm -f "${inventory_tmp}" "${playbook_tmp}" -} - -imageTarName() { - printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' -} - -dockerIoMirrorRef() { - local image="$1" - local mirror_ref="${HOST_DOCKER_IO_MIRROR}" - - if [[ "${image}" == */*/* ]]; then - return 1 - fi - - if [[ "${image}" == */* ]]; then - printf '%s/%s\n' "${mirror_ref}" "${image}" - else - printf '%s/library/%s\n' "${mirror_ref}" "${image}" - fi -} - -ensureHostDockerImage() { - local image="$1" - local mirror_image="" - - if docker image inspect "${image}" >/dev/null 2>&1; then - return 0 - fi - - echo " host docker pull ${image}" >&2 - if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then - return 0 - fi - - if mirror_image="$(dockerIoMirrorRef "${image}")"; then - echo " host docker pull ${mirror_image}" >&2 - runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null - docker tag "${mirror_image}" "${image}" >/dev/null - return 0 - fi - - echo "Failed to prepare image on host: ${image}" >&2 - echo "This script intentionally does not ask the VM to pull public images." >&2 - return 1 -} - -hostImageTarball() { - local image="$1" - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" -} - -saveHostImageTarball() { - local image="$1" - local tar_path - tar_path="$(hostImageTarball "${image}")" - ensureHostDockerImage "${image}" - echo " host docker save ${image} -> ${tar_path}" >&2 - docker save -o "${tar_path}" "${image}" - printf '%s\n' "${tar_path}" -} - -loadDockerImageToMaster() { - local image="$1" - local tar_path remote_tar - tar_path="$(saveHostImageTarball "${image}")" - remote_tar="/tmp/$(basename "${tar_path}")" - echo " copy ${image} to ${k3sMasterName}:${remote_tar}" - copyFileToMaster "${tar_path}" "${remote_tar}" - runMasterCommand "sudo -n docker load -i '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" -} - -importK3sImageToNode() { - local image="$1" - local node_name="$2" - local node_ip="$3" - local tar_path remote_tar - tar_path="$(saveHostImageTarball "${image}")" - remote_tar="/tmp/$(basename "${tar_path}")" - echo " import ${image} to ${node_name}" - copyFileToNode "${node_name}" "${node_ip}" "${tar_path}" "${remote_tar}" - runNodeCommand "${node_name}" "${node_ip}" \ - "sudo -n k3s ctr -n k8s.io images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" -} - -ensureRegistry() { - echo "[3/10] Ensuring private registry on master" - echo " preparing ${REGISTRY_BOOTSTRAP_IMAGE} on host and loading it into master Docker" - loadDockerImageToMaster "${REGISTRY_BOOTSTRAP_IMAGE}" - runMasterCommand " - set -euo pipefail - if ! docker buildx version >/dev/null 2>&1; then - sudo -n apt-get update >/dev/null - sudo -n apt-get install -y docker-buildx >/dev/null - fi - sudo -n docker rm -f registry >/dev/null 2>&1 || true - sudo -n docker image inspect '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null - sudo -n docker run -d --network host --restart=always --name registry \ - -e REGISTRY_HTTP_ADDR=0.0.0.0:${registryPort} '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null - " -} - -ensureSeedemuHostBuildImages() { - ensureHostDockerImage "${ubuntuBuildImage}" - - if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then - echo " host docker build ${seedBaseSourceImage}" >&2 - DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-base" >/dev/null - fi - - if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then - echo " host docker build ${seedRouterSourceImage}" >&2 - DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-router" >/dev/null - fi - - docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null - docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null -} - -prepareMasterWorkloadBuildImages() { - echo "[4/10] Preparing workload build base images on master Docker" - [ -d "${seedEmulatorDockerDir}/seedemu-base" ] || { - echo "Missing host seedemu base image directory: ${seedEmulatorDockerDir}/seedemu-base" >&2 - exit 1 - } - [ -d "${seedEmulatorDockerDir}/seedemu-router" ] || { - echo "Missing host seedemu router image directory: ${seedEmulatorDockerDir}/seedemu-router" >&2 - exit 1 - } - - ensureSeedemuHostBuildImages - loadDockerImageToMaster "${ubuntuBuildImage}" - loadDockerImageToMaster "${seedBaseSourceImage}" - loadDockerImageToMaster "${seedRouterSourceImage}" - - echo " ensuring stable compiler hash tags on master Docker" - runMasterCommand " - set -euo pipefail - sudo -n docker tag '${seedBaseSourceImage}' '${seedBaseHashImage}' - sudo -n docker tag '${seedRouterSourceImage}' '${seedRouterHashImage}' - " - - echo " pushing seedemu base/router images into master local registry" - runMasterCommand " - set -euo pipefail - sudo -n docker tag '${seedBaseSourceImage}' \ - '127.0.0.1:${registryPort}/${seedBaseSourceImage}' - sudo -n docker push '127.0.0.1:${registryPort}/${seedBaseSourceImage}' - sudo -n docker tag '${seedRouterSourceImage}' \ - '127.0.0.1:${registryPort}/${seedRouterSourceImage}' - sudo -n docker push '127.0.0.1:${registryPort}/${seedRouterSourceImage}' - " -} - -fetchKubeconfig() { - echo "[5/10] Fetching kubeconfig" - mkdir -p "$(dirname "${outputKubeconfig}")" - readMasterFile "/etc/rancher/k3s/k3s.yaml" > "${outputKubeconfig}" - sed -i "s|127.0.0.1|${k3sMasterIp}|g" "${outputKubeconfig}" - echo "kubeconfig=${outputKubeconfig}" -} - -preloadK3sBootstrapImagesAllNodes() { - echo "[6/10] Preloading K3s system and Multus images from host into all K3s containerd nodes" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do - importK3sImageToNode "${image}" "${name}" "${ip}" - done - done < <(helper nodes-tsv) - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=kube-dns \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=metrics-server \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l app=local-path-provisioner \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l name=multus \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true -} - -installOvnFabricIfConfigured() { - # Install Kube-OVN after K3s and Multus are available. OVN is a Kubernetes - # CNI add-on, so it cannot be prepared in the physical-node preflight stage. - if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - return 0 - fi - echo "[7/10] Installing Kube-OVN non-primary CNI" - python3 "${SCRIPT_DIR}/ovn/installKubeOvnFabric.py" "${CONFIG_PATH}" -} - -applyNodeK3sTuning() { - local name="$1" - local ip="$2" - local remote_runner=() - echo " tuning K3s on ${name} (${ip})" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - remote_runner=(sudo -n bash -s --) - else - remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) - fi - "${remote_runner[@]}" \ - "${k3sMaxPods}" \ - "${kubeletRegistryQps}" \ - "${kubeletRegistryBurst}" \ - "${rebootAfterTuning}" <<'EOF_REMOTE' -set -euo pipefail -MAX_PODS="$1" -REGISTRY_QPS="$2" -REGISTRY_BURST="$3" -ASYNC_RESTART="$4" - -if [ ! -f /etc/sysctl.d/99-seed-vm-limits.conf ]; then - echo "warning: /etc/sysctl.d/99-seed-vm-limits.conf not found; run tuneVmLimits.py before large-scale experiments" >&2 -fi - -mkdir -p /etc/rancher/k3s -touch /etc/rancher/k3s/config.yaml -cfg=/etc/rancher/k3s/config.yaml -tmp=$(mktemp) -awk ' - /^[^[:space:]].*:/ { - if ($0 ~ /^(kubelet-arg|kube-apiserver-arg):/) {skip=1; next} - skip=0 - } - skip == 0 {print} -' "${cfg}" > "${tmp}" -cat >> "${tmp}" <> "${tmp}" <<'EOF_MASTER' -kube-apiserver-arg: - - "max-requests-inflight=1000" - - "max-mutating-requests-inflight=500" -EOF_MASTER -fi -cat "${tmp}" > "${cfg}" -rm -f "${tmp}" - -if [ "${ASYNC_RESTART}" = "true" ]; then - systemd-run --on-active=2 /bin/bash -c "systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true; systemctl restart containerd 2>/dev/null || true" >/dev/null 2>&1 || true -else - systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true - systemctl restart containerd 2>/dev/null || true -fi -EOF_REMOTE -} - -applyTuningAllNodes() { - echo "[8/10] Applying K3s runtime tuning" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - applyNodeK3sTuning "${name}" "${ip}" - done < <(helper nodes-tsv) -} - -verifyCluster() { - echo "[9/10] Waiting for K3s nodes" - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Ready node --all --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system rollout status daemonset/kube-multus-ds --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" get nodes -o wide - kubectl --kubeconfig "${outputKubeconfig}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tPodCIDR: "}{.spec.podCIDR}{"\n"}{end}' - echo -} - -writeOutputs() { - echo "[10/10] Writing inventory output" - helper write-cluster-inventory >/dev/null - echo "inventory=${outputInventory}" - echo "kubeconfig=${outputKubeconfig}" - echo "registry=${registryHost}:${registryPort}" -} - -main() { - requireCommand python3 - requireCommand ansible-playbook - requireCommand docker - requireCommand scp - requireCommand ssh - requireCommand kubectl - requireCommand sed - [ -f "${PLAYBOOK_PATH}" ] || { - echo "K3s Ansible playbook not found: ${PLAYBOOK_PATH}" >&2 - exit 1 - } - - resolveInput - loadConfigVars - resolveSeedEmulatorDockerDir - - if [ "${MASTER_IS_LOCAL}" != "true" ] && [ ! -f "${k3sSshKey}" ]; then - echo "SSH key not found: ${k3sSshKey}" >&2 - exit 1 - fi - - printPlan - verifyConnectivity - runAnsibleInstall - ensureRegistry - prepareMasterWorkloadBuildImages - fetchKubeconfig - preloadK3sBootstrapImagesAllNodes - installOvnFabricIfConfigured - applyTuningAllNodes - verifyCluster - writeOutputs - echo "K3s cluster is ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py index 7fe6825d5..c695f6d97 100755 --- a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py +++ b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py @@ -1,21 +1,146 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyPhysicalCluster.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyPhysicalCluster with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove a physical SeedEMU K3s cluster and its optional fabric backend. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Runs K3s uninstall scripts on configured nodes, removes the local registry +# container on the master, deletes generated kubeconfig/inventory files, and +# removes the configured linux-vxlan or Kube-OVN fabric if present. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" shell-vars)" + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[cluster-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s <<<"$script" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"$script" +} + +uninstall_node=' +set -euo pipefail +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then + /usr/local/bin/k3s-agent-uninstall.sh || true +fi +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then + /usr/local/bin/k3s-uninstall.sh || true +fi +' + +remove_registry=' +set -euo pipefail +if command -v docker >/dev/null 2>&1; then + docker rm -f registry >/dev/null 2>&1 || true +fi +' + +echo "Cleaning physical K3s cluster from: $CONFIG_PATH" + +destroy_node_concurrency="${SEED_K8S_DESTROY_NODE_CONCURRENCY:-4}" +if ! [[ "$destroy_node_concurrency" =~ ^[0-9]+$ ]] || [ "$destroy_node_concurrency" -lt 1 ]; then + echo "Invalid SEED_K8S_DESTROY_NODE_CONCURRENCY: $destroy_node_concurrency" >&2 + exit 1 +fi + +activeNodeCleanJobs() { + jobs -rp | wc -l | tr -d ' ' +} + +if [ "${fabricType}" = "ovn" ] || [ "${fabricType}" = "kube-ovn" ]; then + if [ -x "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" ]; then + python3 "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" "$CONFIG_PATH" || true + else + echo "warning: OVN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +node_clean_failures=0 +echo "K3s node uninstall concurrency: ${destroy_node_concurrency}" +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "$HELPER" --config "$CONFIG_PATH" node-ssh-vars --name "$name")" + if [ "$role" = "master" ]; then + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$remove_registry" + fi + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$uninstall_node" & + while [ "$(activeNodeCleanJobs)" -ge "$destroy_node_concurrency" ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi + done +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +while [ "$(activeNodeCleanJobs)" -gt 0 ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi +done + +if [ "$node_clean_failures" -ne 0 ]; then + echo "K3s node uninstall failed on ${node_clean_failures} node(s)" >&2 + exit 1 +fi + +if [ "${fabricType}" = "linux-vxlan" ]; then + if [ -x "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" ]; then + python3 "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + else + echo "warning: VXLAN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +rm -f "$outputKubeconfig" "$outputInventory" +echo "Removed generated kubeconfig/inventory:" +echo " $outputKubeconfig" +echo " $outputInventory" +echo "Physical K3s cluster cleanup finished." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh deleted file mode 100755 index 751d6cb51..000000000 --- a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Remove a physical SeedEMU K3s cluster and its optional fabric backend. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Side effects: -# Runs K3s uninstall scripts on configured nodes, removes the local registry -# container on the master, deletes generated kubeconfig/inventory files, and -# removes the configured linux-vxlan or Kube-OVN fabric if present. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${1:-./configK3s.yaml}" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" shell-vars)" - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - - echo "[cluster-clean] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s <<<"$script" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"$script" -} - -uninstall_node=' -set -euo pipefail -if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then - /usr/local/bin/k3s-agent-uninstall.sh || true -fi -if [ -x /usr/local/bin/k3s-uninstall.sh ]; then - /usr/local/bin/k3s-uninstall.sh || true -fi -' - -remove_registry=' -set -euo pipefail -if command -v docker >/dev/null 2>&1; then - docker rm -f registry >/dev/null 2>&1 || true -fi -' - -echo "Cleaning physical K3s cluster from: $CONFIG_PATH" - -if [ "${fabricType}" = "ovn" ] || [ "${fabricType}" = "kube-ovn" ]; then - if [ -x "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" ]; then - python3 "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" "$CONFIG_PATH" || true - else - echo "warning: OVN cleanup script is not present in this generated setup directory" >&2 - fi -fi - -while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "$HELPER" --config "$CONFIG_PATH" node-ssh-vars --name "$name")" - if [ "$role" = "master" ]; then - runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$remove_registry" - fi - runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$uninstall_node" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) - -if [ "${fabricType}" = "linux-vxlan" ]; then - if [ -x "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" ]; then - python3 "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - else - echo "warning: VXLAN cleanup script is not present in this generated setup directory" >&2 - fi -fi - -rm -f "$outputKubeconfig" "$outputInventory" -echo "Removed generated kubeconfig/inventory:" -echo " $outputKubeconfig" -echo " $outputInventory" -echo "Physical K3s cluster cleanup finished." diff --git a/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py index feee5bcb6..fc6f27a93 100755 --- a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py +++ b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py @@ -1,21 +1,476 @@ #!/usr/bin/env python3 -"""Python entrypoint for createKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for createKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create or start KVM VMs from kvm.yaml. The script resolves non-conflicting +# VM names/IPs/MACs, writes user-facing configK3s.yaml plus internal +# kvmState.yaml, and waits for SSH. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +kvmBaseImageReuseMode="copy" +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 + +eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + +SSH_PUB_KEY="" +EXISTING_VMS_TSV="" +PLANNED_NODES_TSV="" +REUSING_KVM_STATE_PLAN="false" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${EXISTING_VMS_TSV}" ] && rm -f "${EXISTING_VMS_TSV}" || true + [ -n "${PLANNED_NODES_TSV}" ] && rm -f "${PLANNED_NODES_TSV}" || true +} + +domainExists() { + virsh dominfo "$1" >/dev/null 2>&1 +} + +domainRunning() { + virsh domstate "$1" 2>/dev/null | grep -qi "running" +} + +prepareDirs() { + mkdir -p "${kvmStorageDir}/base" + mkdir -p "${kvmDiskDir}" + mkdir -p "${kvmCloudInitDir}" + mkdir -p "$(dirname "${kvmBaseImagePath}")" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +ensureNetwork() { + if ! virsh net-info "${kvmNetwork}" >/dev/null 2>&1; then + echo "Libvirt network not found: ${kvmNetwork}" >&2 + exit 1 + fi + virsh net-start "${kvmNetwork}" >/dev/null 2>&1 || true + virsh net-autostart "${kvmNetwork}" >/dev/null 2>&1 || true +} + +collectExistingVms() { + EXISTING_VMS_TSV="$(mktemp "${kvmStorageDir}/existing-vms.XXXXXX.tsv")" + PLANNED_NODES_TSV="$(mktemp "${kvmStorageDir}/planned-vms.XXXXXX.tsv")" + + if [ -f "${outputKvmState}" ]; then + if ! python3 "${HELPER}" "${CONFIG_PATH}" validate-kvm-state --state "${outputKvmState}"; then + echo "Refusing to reuse stale KVM state: ${outputKvmState}" >&2 + echo "Remove it only if you intentionally want this setup directory to generate a new VM plan." >&2 + exit 1 + fi + REUSING_KVM_STATE_PLAN="true" + python3 "${HELPER}" "${CONFIG_PATH}" state-nodes-tsv --state "${outputKvmState}" > "${PLANNED_NODES_TSV}" + echo "Using existing KVM state: ${outputKvmState}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" + return + fi + + virsh list --all --name 2>/dev/null | awk 'NF {print $1 "\t\t"}' >> "${EXISTING_VMS_TSV}" + + virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + print("\t".join([ + host.get("name") or "", + host.get("ip") or "", + (host.get("mac") or "").lower(), + ])) +' >> "${EXISTING_VMS_TSV}" + + virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk ' + NR <= 2 {next} + NF >= 5 { + name=$6 + if (name == "-" || name == "") name="" + ip=$5 + sub("/.*", "", ip) + print name "\t" ip "\t" tolower($3) + } + ' >> "${EXISTING_VMS_TSV}" || true + + python3 "${HELPER}" "${CONFIG_PATH}" nodes-tsv --existing-tsv "${EXISTING_VMS_TSV}" > "${PLANNED_NODES_TSV}" + echo "Planned KVM nodes:" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" +} + +downloadBaseImage() { + if [ -f "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Using existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Using existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + echo "Downloading Ubuntu cloud image to ${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +loadSshPubkey() { + if [ -f "${sshKey}.pub" ]; then + SSH_PUB_KEY="$(tr -d '\n' < "${sshKey}.pub")" + return + fi + if [ -f "${sshKey}" ]; then + SSH_PUB_KEY="$(ssh-keygen -y -f "${sshKey}" 2>/dev/null | tr -d '\n')" + [ -n "${SSH_PUB_KEY}" ] && return + fi + echo "Cannot read SSH key. Set ssh.key in ${CONFIG_PATH} to a valid private key." >&2 + exit 1 +} + +updateDhcpHost() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + virsh net-update "${kvmNetwork}" add ip-dhcp-host "${host_xml}" --live --config >/dev/null +} + +createVmCloudInit() { + local vm_name="$1" + local vm_dir="${kvmCloudInitDir}/${vm_name}" + mkdir -p "${vm_dir}" + + cat > "${vm_dir}/user-data.yaml" < "${vm_dir}/meta-data.yaml" </dev/null +} + +createOrStartVm() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vcpus="$4" + local memory_mb="$5" + local disk_gb="$6" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + local vm_cloud_dir="${kvmCloudInitDir}/${vm_name}" + + if domainExists "${vm_name}"; then + validateExistingDomainBelongsToSetup "${vm_name}" + if [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "Refusing to reuse existing VM '${vm_name}'. Set kvm.allowExisting: true only if this is intentional." >&2 + exit 1 + fi + echo "VM exists: ${vm_name}" + if ! domainRunning "${vm_name}"; then + virsh start "${vm_name}" >/dev/null + fi + return + fi + + echo "Creating VM: ${vm_name} ip=${vm_ip} vcpus=${vcpus} memory_mb=${memory_mb} disk_gb=${disk_gb}" + virt-install \ + --name "${vm_name}" \ + --memory "${memory_mb}" \ + --vcpus "${vcpus}" \ + --cpu host-passthrough \ + --import \ + --os-variant generic \ + --network "network=${kvmNetwork},model=virtio,mac=${vm_mac}" \ + --disk "path=${vm_disk},format=qcow2,bus=virtio" \ + --graphics none \ + --noautoconsole \ + --cloud-init "user-data=${vm_cloud_dir}/user-data.yaml,meta-data=${vm_cloud_dir}/meta-data.yaml" >/dev/null +} + +validateExistingDomainBelongsToSetup() { + # Args: + # $1: VM domain name that already exists. + # Uses kvmDiskDir from kvm.yaml to make stale kvmState.yaml files fail safe. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for existing VM ${vm_name}; refusing to reuse it." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to reuse existing VM ${vm_name}: disk is outside this setup diskDir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +checkNetworkConflict() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + + virsh net-dumpxml "${kvmNetwork}" | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +name, ip, mac = sys.argv[1], sys.argv[2], sys.argv[3].lower() +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + h_name = host.get("name") or "" + h_ip = host.get("ip") or "" + h_mac = (host.get("mac") or "").lower() + same_host = h_name == name and h_ip == ip and h_mac == mac + if h_ip == ip and not same_host: + raise SystemExit(f"IP {ip} is already reserved by host name={h_name} mac={h_mac}") + if h_mac == mac and not same_host: + raise SystemExit(f"MAC {mac} is already reserved by host name={h_name} ip={h_ip}") +' "${vm_name}" "${vm_ip}" "${vm_mac}" + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk -v ip="${vm_ip}" -v mac="${vm_mac,,}" -v name="${vm_name}" ' + NR <= 2 {next} + { + lease_mac=tolower($3); lease_ip=$5; sub("/.*", "", lease_ip); lease_name=$6 + if ((lease_ip == ip || lease_mac == mac) && !(lease_ip == ip && lease_mac == mac && lease_name == name)) { + print "DHCP lease conflict: name=" lease_name " mac=" lease_mac " ip=" lease_ip > "/dev/stderr" + exit 1 + } + } + '; then + return + fi + exit 1 +} + +checkCreateConflicts() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + + if domainExists "${vm_name}" && [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "VM name conflict: ${vm_name} already exists. Choose a new name or set kvm.allowExisting: true intentionally." >&2 + exit 1 + fi + if [ -e "${vm_disk}" ] && ! domainExists "${vm_name}" ]; then + echo "Disk conflict: ${vm_disk} already exists but VM ${vm_name} is not defined." >&2 + exit 1 + fi + checkNetworkConflict "${vm_name}" "${vm_ip}" "${vm_mac}" +} + +waitForSsh() { + local vm_name="$1" + local vm_ip="$2" + local elapsed=0 + while [ "${elapsed}" -lt "${kvmBootTimeoutSeconds}" ]; do + if ssh -o StrictHostKeyChecking=no \ + -n \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -o ConnectTimeout=5 \ + -i "${sshKey}" \ + "${sshUser}@${vm_ip}" "echo ok" >/dev/null 2>&1; then + echo "SSH ready: ${vm_name} (${vm_ip})" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + echo "Timeout waiting for SSH on ${vm_name} (${vm_ip})" >&2 + return 1 +} + +writeK3sConfig() { + # Args: none. + # Writes the user-facing configK3s.yaml from the transient planned node + # list. It intentionally omits KVM resource details and output paths. + python3 "${HELPER}" "${CONFIG_PATH}" write-k3s-config \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputK3sConfig}" + echo "K3s config: ${outputK3sConfig}" +} + +writeKvmState() { + # Args: none. + # Writes internal kvmState.yaml with VM MAC/resource/disk metadata. Cleanup + # uses this file instead of overloading the user-facing configK3s.yaml. + python3 "${HELPER}" "${CONFIG_PATH}" write-kvm-state \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputKvmState}" + echo "KVM state: ${outputKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand virsh + requireCommand virt-install + requireCommand qemu-img + requireCommand curl + requireCommand ssh + requireCommand ssh-keygen + requireCommand awk + + [ -f "${sshKey}" ] || { + echo "SSH key not found: ${sshKey}" >&2 + exit 1 + } + trap cleanupTmp EXIT + + prepareDirs + ensureNetwork + collectExistingVms + downloadBaseImage + loadSshPubkey + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + checkCreateConflicts "${name}" "${ip}" "${mac}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + updateDhcpHost "${name}" "${ip}" "${mac}" + createVmCloudInit "${name}" + createVmDisk "${name}" "${disk_gb}" + createOrStartVm "${name}" "${ip}" "${mac}" "${vcpus}" "${memory_mb}" "${disk_gb}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < "${PLANNED_NODES_TSV}" + + if [ "${kvmSkipK3sConfig}" = "true" ]; then + echo "Skipping host-local configK3s.yaml generation because kvm.skipK3sConfig=true." + else + writeK3sConfig + fi + writeKvmState + echo "KVM VMs are ready." + echo "Next step: ${SCRIPT_DIR}/tuneVmLimits.py ${outputK3sConfig}" +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh deleted file mode 100755 index 90d2a5bfe..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh +++ /dev/null @@ -1,457 +0,0 @@ -#!/usr/bin/env bash -# Create or start KVM VMs from kvm.yaml. The script resolves non-conflicting -# VM names/IPs/MACs, writes user-facing configK3s.yaml plus internal -# kvmState.yaml, and waits for SSH. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" -kvmBaseImageSearchDirs="${HOME}/k8s/output" -kvmBaseImageReuseMode="copy" -baseImageLowSpeedSeconds=60 -baseImageLowSpeedLimit=1024 - -eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" - -SSH_PUB_KEY="" -EXISTING_VMS_TSV="" -PLANNED_NODES_TSV="" -REUSING_KVM_STATE_PLAN="false" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -cleanupTmp() { - [ -n "${EXISTING_VMS_TSV}" ] && rm -f "${EXISTING_VMS_TSV}" || true - [ -n "${PLANNED_NODES_TSV}" ] && rm -f "${PLANNED_NODES_TSV}" || true -} - -domainExists() { - virsh dominfo "$1" >/dev/null 2>&1 -} - -domainRunning() { - virsh domstate "$1" 2>/dev/null | grep -qi "running" -} - -prepareDirs() { - mkdir -p "${kvmStorageDir}/base" - mkdir -p "${kvmDiskDir}" - mkdir -p "${kvmCloudInitDir}" - mkdir -p "$(dirname "${kvmBaseImagePath}")" -} - -validBaseImage() { - # Args: - # $1: qcow2 image path to validate. - local image_path="$1" - [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 -} - -downloadBaseImageFile() { - # Args: - # $1: URL, $2: destination path. - local image_url="$1" - local image_path="$2" - local partial_path="${image_path}.part" - mkdir -p "$(dirname "${image_path}")" - curl -fL \ - --retry 5 \ - --retry-delay 5 \ - --retry-all-errors \ - --connect-timeout 20 \ - --speed-time "${baseImageLowSpeedSeconds}" \ - --speed-limit "${baseImageLowSpeedLimit}" \ - -C - \ - -o "${partial_path}" \ - "${image_url}" - validBaseImage "${partial_path}" || { - echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 - return 1 - } - mv -f "${partial_path}" "${image_path}" -} - -ensureNetwork() { - if ! virsh net-info "${kvmNetwork}" >/dev/null 2>&1; then - echo "Libvirt network not found: ${kvmNetwork}" >&2 - exit 1 - fi - virsh net-start "${kvmNetwork}" >/dev/null 2>&1 || true - virsh net-autostart "${kvmNetwork}" >/dev/null 2>&1 || true -} - -collectExistingVms() { - EXISTING_VMS_TSV="$(mktemp "${kvmStorageDir}/existing-vms.XXXXXX.tsv")" - PLANNED_NODES_TSV="$(mktemp "${kvmStorageDir}/planned-vms.XXXXXX.tsv")" - - if [ -f "${outputKvmState}" ]; then - if ! python3 "${HELPER}" "${CONFIG_PATH}" validate-kvm-state --state "${outputKvmState}"; then - echo "Refusing to reuse stale KVM state: ${outputKvmState}" >&2 - echo "Remove it only if you intentionally want this setup directory to generate a new VM plan." >&2 - exit 1 - fi - REUSING_KVM_STATE_PLAN="true" - python3 "${HELPER}" "${CONFIG_PATH}" state-nodes-tsv --state "${outputKvmState}" > "${PLANNED_NODES_TSV}" - echo "Using existing KVM state: ${outputKvmState}" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" - return - fi - - virsh list --all --name 2>/dev/null | awk 'NF {print $1 "\t\t"}' >> "${EXISTING_VMS_TSV}" - - virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c ' -import sys -import xml.etree.ElementTree as ET - -root = ET.fromstring(sys.stdin.read()) -for host in root.findall(".//host"): - print("\t".join([ - host.get("name") or "", - host.get("ip") or "", - (host.get("mac") or "").lower(), - ])) -' >> "${EXISTING_VMS_TSV}" - - virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk ' - NR <= 2 {next} - NF >= 5 { - name=$6 - if (name == "-" || name == "") name="" - ip=$5 - sub("/.*", "", ip) - print name "\t" ip "\t" tolower($3) - } - ' >> "${EXISTING_VMS_TSV}" || true - - python3 "${HELPER}" "${CONFIG_PATH}" nodes-tsv --existing-tsv "${EXISTING_VMS_TSV}" > "${PLANNED_NODES_TSV}" - echo "Planned KVM nodes:" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" -} - -downloadBaseImage() { - if [ -f "${kvmBaseImagePath}" ]; then - if validBaseImage "${kvmBaseImagePath}"; then - return - fi - echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" - rm -f "${kvmBaseImagePath}" - fi - if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then - echo "Using existing base image from ${kvmLegacyBaseImagePath}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - fi - return - fi - local output_image="" - local search_dir="" - for search_dir in ${kvmBaseImageSearchDirs}; do - [ -d "${search_dir}" ] || continue - output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" - if [ -n "${output_image}" ]; then - echo "Using existing base image from ${output_image}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${output_image}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" - fi - return - fi - done - echo "Downloading Ubuntu cloud image to ${kvmBaseImagePath}" - downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" -} - -loadSshPubkey() { - if [ -f "${sshKey}.pub" ]; then - SSH_PUB_KEY="$(tr -d '\n' < "${sshKey}.pub")" - return - fi - if [ -f "${sshKey}" ]; then - SSH_PUB_KEY="$(ssh-keygen -y -f "${sshKey}" 2>/dev/null | tr -d '\n')" - [ -n "${SSH_PUB_KEY}" ] && return - fi - echo "Cannot read SSH key. Set ssh.key in ${CONFIG_PATH} to a valid private key." >&2 - exit 1 -} - -updateDhcpHost() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local host_xml="" - virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true - virsh net-update "${kvmNetwork}" add ip-dhcp-host "${host_xml}" --live --config >/dev/null -} - -createVmCloudInit() { - local vm_name="$1" - local vm_dir="${kvmCloudInitDir}/${vm_name}" - mkdir -p "${vm_dir}" - - cat > "${vm_dir}/user-data.yaml" < "${vm_dir}/meta-data.yaml" </dev/null -} - -createOrStartVm() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local vcpus="$4" - local memory_mb="$5" - local disk_gb="$6" - local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" - local vm_cloud_dir="${kvmCloudInitDir}/${vm_name}" - - if domainExists "${vm_name}"; then - validateExistingDomainBelongsToSetup "${vm_name}" - if [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then - echo "Refusing to reuse existing VM '${vm_name}'. Set kvm.allowExisting: true only if this is intentional." >&2 - exit 1 - fi - echo "VM exists: ${vm_name}" - if ! domainRunning "${vm_name}"; then - virsh start "${vm_name}" >/dev/null - fi - return - fi - - echo "Creating VM: ${vm_name} ip=${vm_ip} vcpus=${vcpus} memory_mb=${memory_mb} disk_gb=${disk_gb}" - virt-install \ - --name "${vm_name}" \ - --memory "${memory_mb}" \ - --vcpus "${vcpus}" \ - --cpu host-passthrough \ - --import \ - --os-variant generic \ - --network "network=${kvmNetwork},model=virtio,mac=${vm_mac}" \ - --disk "path=${vm_disk},format=qcow2,bus=virtio" \ - --graphics none \ - --noautoconsole \ - --cloud-init "user-data=${vm_cloud_dir}/user-data.yaml,meta-data=${vm_cloud_dir}/meta-data.yaml" >/dev/null -} - -validateExistingDomainBelongsToSetup() { - # Args: - # $1: VM domain name that already exists. - # Uses kvmDiskDir from kvm.yaml to make stale kvmState.yaml files fail safe. - local vm_name="$1" - local disk_path="" - disk_path="$( - virsh domblklist "${vm_name}" --details 2>/dev/null \ - | awk '$2 == "disk" && $4 != "-" {print $4; exit}' - )" - if [ -z "${disk_path}" ]; then - echo "Cannot determine disk path for existing VM ${vm_name}; refusing to reuse it." >&2 - exit 1 - fi - case "${disk_path}" in - "${kvmDiskDir}"/*) ;; - *) - echo "Refusing to reuse existing VM ${vm_name}: disk is outside this setup diskDir." >&2 - echo " vm_disk=${disk_path}" >&2 - echo " setup_disk_dir=${kvmDiskDir}" >&2 - echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 - exit 1 - ;; - esac -} - -checkNetworkConflict() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - - virsh net-dumpxml "${kvmNetwork}" | python3 -c ' -import sys -import xml.etree.ElementTree as ET - -name, ip, mac = sys.argv[1], sys.argv[2], sys.argv[3].lower() -root = ET.fromstring(sys.stdin.read()) -for host in root.findall(".//host"): - h_name = host.get("name") or "" - h_ip = host.get("ip") or "" - h_mac = (host.get("mac") or "").lower() - same_host = h_name == name and h_ip == ip and h_mac == mac - if h_ip == ip and not same_host: - raise SystemExit(f"IP {ip} is already reserved by host name={h_name} mac={h_mac}") - if h_mac == mac and not same_host: - raise SystemExit(f"MAC {mac} is already reserved by host name={h_name} ip={h_ip}") -' "${vm_name}" "${vm_ip}" "${vm_mac}" - - if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk -v ip="${vm_ip}" -v mac="${vm_mac,,}" -v name="${vm_name}" ' - NR <= 2 {next} - { - lease_mac=tolower($3); lease_ip=$5; sub("/.*", "", lease_ip); lease_name=$6 - if ((lease_ip == ip || lease_mac == mac) && !(lease_ip == ip && lease_mac == mac && lease_name == name)) { - print "DHCP lease conflict: name=" lease_name " mac=" lease_mac " ip=" lease_ip > "/dev/stderr" - exit 1 - } - } - '; then - return - fi - exit 1 -} - -checkCreateConflicts() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" - - if domainExists "${vm_name}" && [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then - echo "VM name conflict: ${vm_name} already exists. Choose a new name or set kvm.allowExisting: true intentionally." >&2 - exit 1 - fi - if [ -e "${vm_disk}" ] && ! domainExists "${vm_name}" ]; then - echo "Disk conflict: ${vm_disk} already exists but VM ${vm_name} is not defined." >&2 - exit 1 - fi - checkNetworkConflict "${vm_name}" "${vm_ip}" "${vm_mac}" -} - -waitForSsh() { - local vm_name="$1" - local vm_ip="$2" - local elapsed=0 - while [ "${elapsed}" -lt "${kvmBootTimeoutSeconds}" ]; do - if ssh -o StrictHostKeyChecking=no \ - -n \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -o ConnectTimeout=5 \ - -i "${sshKey}" \ - "${sshUser}@${vm_ip}" "echo ok" >/dev/null 2>&1; then - echo "SSH ready: ${vm_name} (${vm_ip})" - return 0 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done - echo "Timeout waiting for SSH on ${vm_name} (${vm_ip})" >&2 - return 1 -} - -writeK3sConfig() { - # Args: none. - # Writes the user-facing configK3s.yaml from the transient planned node - # list. It intentionally omits KVM resource details and output paths. - python3 "${HELPER}" "${CONFIG_PATH}" write-k3s-config \ - --nodes-tsv "${PLANNED_NODES_TSV}" \ - --output "${outputK3sConfig}" - echo "K3s config: ${outputK3sConfig}" -} - -writeKvmState() { - # Args: none. - # Writes internal kvmState.yaml with VM MAC/resource/disk metadata. Cleanup - # uses this file instead of overloading the user-facing configK3s.yaml. - python3 "${HELPER}" "${CONFIG_PATH}" write-kvm-state \ - --nodes-tsv "${PLANNED_NODES_TSV}" \ - --output "${outputKvmState}" - echo "KVM state: ${outputKvmState}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand virsh - requireCommand virt-install - requireCommand qemu-img - requireCommand curl - requireCommand ssh - requireCommand ssh-keygen - requireCommand awk - - [ -f "${sshKey}" ] || { - echo "SSH key not found: ${sshKey}" >&2 - exit 1 - } - trap cleanupTmp EXIT - - prepareDirs - ensureNetwork - collectExistingVms - downloadBaseImage - loadSshPubkey - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - checkCreateConflicts "${name}" "${ip}" "${mac}" - done < "${PLANNED_NODES_TSV}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - updateDhcpHost "${name}" "${ip}" "${mac}" - createVmCloudInit "${name}" - createVmDisk "${name}" "${disk_gb}" - createOrStartVm "${name}" "${ip}" "${mac}" "${vcpus}" "${memory_mb}" "${disk_gb}" - done < "${PLANNED_NODES_TSV}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - waitForSsh "${name}" "${ip}" - done < "${PLANNED_NODES_TSV}" - - if [ "${kvmSkipK3sConfig}" = "true" ]; then - echo "Skipping host-local configK3s.yaml generation because kvm.skipK3sConfig=true." - else - writeK3sConfig - fi - writeKvmState - echo "KVM VMs are ready." - echo "Next step: ${SCRIPT_DIR}/tuneVmLimits.py ${outputK3sConfig}" -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py index fa35995c0..45a1cb0f6 100755 --- a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py +++ b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py @@ -1,21 +1,274 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs recorded in kvmState.yaml. +# Inputs: kvmState.yaml generated by createKvmVms.py. +# Outputs/side effects: destroys matching libvirt domains, removes DHCP +# reservations/leases, disks, cloud-init files, generated kubeconfig/inventory, +# and then verifies the cleanup. The disk-dir guard prevents deleting unrelated +# VMs whose names happen to match. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +NODES_TSV="" +VERIFY_NODES_TSV="" +dnsmasqStatus="/var/lib/libvirt/dnsmasq/virbr0.status" +setupOutputGlob="${SETUP_DIR}/seedemu-k3s.*" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${NODES_TSV}" ] && rm -f "${NODES_TSV}" || true + [ -n "${VERIFY_NODES_TSV}" ] && rm -f "${VERIFY_NODES_TSV}" || true +} + +loadConfig() { + if [ ! -s "${CONFIG_PATH}" ]; then + echo "KVM state not found or empty: ${CONFIG_PATH}" >&2 + echo "Run createKvmVms.py first so it generates kvmState.yaml." >&2 + exit 1 + fi + eval "$(python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-vars --state "${CONFIG_PATH}")" + kvmBridgeName="$(virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c 'import sys, xml.etree.ElementTree as ET; data=sys.stdin.read(); print((ET.fromstring(data).find(".//bridge").get("name") if data.strip() else "") or "")' 2>/dev/null || true)" + if [ -n "${kvmBridgeName}" ]; then + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmBridgeName}.status" + else + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmNetwork}.status" + fi + NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-nodes.XXXXXX.tsv")" + VERIFY_NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-verify-nodes.XXXXXX.tsv")" + python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-nodes-tsv --state "${CONFIG_PATH}" > "${NODES_TSV}" + cp "${NODES_TSV}" "${VERIFY_NODES_TSV}" +} + +printPlan() { + echo "Cleaning KVM nodes from: ${CONFIG_PATH}" + echo "KVM network: ${kvmNetwork}" + echo "Disk dir: ${kvmDiskDir}" + echo "Cloud-init dir: ${kvmCloudInitDir}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' "${NODES_TSV}" +} + +cleanupDomainsReservationsAndFiles() { + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + echo "Cleaning VM ${name} (${ip}, ${mac})" + + if virsh dominfo "${name}" >/dev/null 2>&1; then + validateDestroyTargetBelongsToSetup "${name}" + virsh destroy "${name}" >/dev/null 2>&1 || true + virsh undefine "${name}" --nvram --remove-all-storage >/dev/null 2>&1 \ + || virsh undefine "${name}" --nvram >/dev/null 2>&1 \ + || virsh undefine "${name}" >/dev/null + fi + + if [ -n "${ip}" ] && [ -n "${mac}" ]; then + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + fi + + rm -rf "${kvmCloudInitDir}/${name}" + rm -f "${kvmDiskDir}/${name}.qcow2" + done < "${NODES_TSV}" +} + +validateDestroyTargetBelongsToSetup() { + # Args: + # $1: VM domain name to validate before destructive cleanup. + # Uses kvmDiskDir from kvmState.yaml. This prevents a stale state file from + # deleting a pre-existing cluster VM whose disk lives elsewhere. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for ${vm_name}; refusing destructive cleanup." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to destroy ${vm_name}: disk is outside this setup disk dir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +cleanupDnsmasqStaleLeases() { + [ -f "${dnsmasqStatus}" ] || return 0 + + local tmp_nodes + tmp_nodes="$(mktemp)" + awk -F '\t' 'NF >= 4 {print $1 "\t" $3 "\t" tolower($4)}' "${NODES_TSV}" > "${tmp_nodes}" + + sudo cp "${dnsmasqStatus}" "${dnsmasqStatus}.bak.$(date +%Y%m%d_%H%M%S)" + sudo python3 - "${dnsmasqStatus}" "${tmp_nodes}" <<'PY' +import json +import sys +from pathlib import Path + +status_path = Path(sys.argv[1]) +nodes_path = Path(sys.argv[2]) + +remove_names = set() +remove_ips = set() +remove_macs = set() +for line in nodes_path.read_text(encoding="utf-8").splitlines(): + parts = line.split("\t") + if len(parts) < 3: + continue + name, ip, mac = parts[:3] + if name: + remove_names.add(name) + if ip: + remove_ips.add(ip) + if mac: + remove_macs.add(mac.lower()) + +try: + data = json.loads(status_path.read_text(encoding="utf-8") or "[]") +except json.JSONDecodeError: + raise SystemExit(f"Invalid dnsmasq status JSON: {status_path}") + +filtered = [ + item for item in data + if not ( + item.get("hostname") in remove_names + or item.get("ip-address") in remove_ips + or str(item.get("mac-address", "")).lower() in remove_macs + ) +] +status_path.write_text(json.dumps(filtered, indent=2) + "\n", encoding="utf-8") +PY + rm -f "${tmp_nodes}" +} + +cleanupSetupOutputs() { + echo "Cleaning stale setup outputs matching: ${setupOutputGlob}" + rm -f ${setupOutputGlob} + rm -f "${CONFIG_PATH}" "${outputK3sConfig}" "${outputKubeconfig}" "${outputInventory}" + rm -f "${SETUP_DIR}/configRunning.yaml" +} + +verifyClean() { + local failed=0 + + echo "Verifying cleanup..." + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + if virsh dominfo "${name}" >/dev/null 2>&1; then + echo "Residual domain: ${name}" >&2 + failed=1 + fi + if [ -e "${kvmDiskDir}/${name}.qcow2" ]; then + echo "Residual disk: ${kvmDiskDir}/${name}.qcow2" >&2 + failed=1 + fi + if [ -e "${kvmCloudInitDir}/${name}" ]; then + echo "Residual cloud-init dir: ${kvmCloudInitDir}/${name}" >&2 + failed=1 + fi + + if virsh net-dumpxml "${kvmNetwork}" 2>/dev/null \ + | grep -E "]*(mac=['\"]${mac}['\"]|name=['\"]${name}['\"]|ip=['\"]${ip}['\"])" >/dev/null 2>&1; then + echo "Residual DHCP reservation in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null \ + | awk -v name="${name}" -v ip="${ip}/24" -v mac="$(printf '%s' "${mac}" | tr '[:upper:]' '[:lower:]')" ' + NR > 2 { + row = tolower($0) + if (row ~ mac && $5 == ip && $6 == name) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "Residual DHCP lease in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + done < "${VERIFY_NODES_TSV}" + + if [ -e "${CONFIG_PATH}" ]; then + echo "Residual KVM state: ${CONFIG_PATH}" >&2 + failed=1 + fi + + if compgen -G "${setupOutputGlob}" >/dev/null; then + echo "Residual setup output files matching ${setupOutputGlob}" >&2 + compgen -G "${setupOutputGlob}" >&2 || true + failed=1 + fi + + if [ "${failed}" -ne 0 ]; then + echo "Cleanup verification failed." >&2 + exit 1 + fi + echo "Cleanup verification passed." +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand awk + requireCommand grep + requireCommand mktemp + requireCommand python3 + requireCommand sudo + requireCommand virsh + + trap cleanupTmp EXIT + loadConfig + printPlan + cleanupDomainsReservationsAndFiles + cleanupDnsmasqStaleLeases + cleanupSetupOutputs + verifyClean +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh deleted file mode 100755 index f99d3fd2b..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env bash -# Destroy KVM VMs recorded in kvmState.yaml. -# Inputs: kvmState.yaml generated by createKvmVms.py. -# Outputs/side effects: destroys matching libvirt domains, removes DHCP -# reservations/leases, disks, cloud-init files, generated kubeconfig/inventory, -# and then verifies the cleanup. The disk-dir guard prevents deleting unrelated -# VMs whose names happen to match. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvmState.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" -NODES_TSV="" -VERIFY_NODES_TSV="" -dnsmasqStatus="/var/lib/libvirt/dnsmasq/virbr0.status" -setupOutputGlob="${SETUP_DIR}/seedemu-k3s.*" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -cleanupTmp() { - [ -n "${NODES_TSV}" ] && rm -f "${NODES_TSV}" || true - [ -n "${VERIFY_NODES_TSV}" ] && rm -f "${VERIFY_NODES_TSV}" || true -} - -loadConfig() { - if [ ! -s "${CONFIG_PATH}" ]; then - echo "KVM state not found or empty: ${CONFIG_PATH}" >&2 - echo "Run createKvmVms.py first so it generates kvmState.yaml." >&2 - exit 1 - fi - eval "$(python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-vars --state "${CONFIG_PATH}")" - kvmBridgeName="$(virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c 'import sys, xml.etree.ElementTree as ET; data=sys.stdin.read(); print((ET.fromstring(data).find(".//bridge").get("name") if data.strip() else "") or "")' 2>/dev/null || true)" - if [ -n "${kvmBridgeName}" ]; then - dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmBridgeName}.status" - else - dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmNetwork}.status" - fi - NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-nodes.XXXXXX.tsv")" - VERIFY_NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-verify-nodes.XXXXXX.tsv")" - python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-nodes-tsv --state "${CONFIG_PATH}" > "${NODES_TSV}" - cp "${NODES_TSV}" "${VERIFY_NODES_TSV}" -} - -printPlan() { - echo "Cleaning KVM nodes from: ${CONFIG_PATH}" - echo "KVM network: ${kvmNetwork}" - echo "Disk dir: ${kvmDiskDir}" - echo "Cloud-init dir: ${kvmCloudInitDir}" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' "${NODES_TSV}" -} - -cleanupDomainsReservationsAndFiles() { - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - [ -n "${name}" ] || continue - echo "Cleaning VM ${name} (${ip}, ${mac})" - - if virsh dominfo "${name}" >/dev/null 2>&1; then - validateDestroyTargetBelongsToSetup "${name}" - virsh destroy "${name}" >/dev/null 2>&1 || true - virsh undefine "${name}" --nvram --remove-all-storage >/dev/null 2>&1 \ - || virsh undefine "${name}" --nvram >/dev/null 2>&1 \ - || virsh undefine "${name}" >/dev/null - fi - - if [ -n "${ip}" ] && [ -n "${mac}" ]; then - local host_xml="" - virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true - fi - - rm -rf "${kvmCloudInitDir}/${name}" - rm -f "${kvmDiskDir}/${name}.qcow2" - done < "${NODES_TSV}" -} - -validateDestroyTargetBelongsToSetup() { - # Args: - # $1: VM domain name to validate before destructive cleanup. - # Uses kvmDiskDir from kvmState.yaml. This prevents a stale state file from - # deleting a pre-existing cluster VM whose disk lives elsewhere. - local vm_name="$1" - local disk_path="" - disk_path="$( - virsh domblklist "${vm_name}" --details 2>/dev/null \ - | awk '$2 == "disk" && $4 != "-" {print $4; exit}' - )" - if [ -z "${disk_path}" ]; then - echo "Cannot determine disk path for ${vm_name}; refusing destructive cleanup." >&2 - exit 1 - fi - case "${disk_path}" in - "${kvmDiskDir}"/*) ;; - *) - echo "Refusing to destroy ${vm_name}: disk is outside this setup disk dir." >&2 - echo " vm_disk=${disk_path}" >&2 - echo " setup_disk_dir=${kvmDiskDir}" >&2 - echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 - exit 1 - ;; - esac -} - -cleanupDnsmasqStaleLeases() { - [ -f "${dnsmasqStatus}" ] || return 0 - - local tmp_nodes - tmp_nodes="$(mktemp)" - awk -F '\t' 'NF >= 4 {print $1 "\t" $3 "\t" tolower($4)}' "${NODES_TSV}" > "${tmp_nodes}" - - sudo cp "${dnsmasqStatus}" "${dnsmasqStatus}.bak.$(date +%Y%m%d_%H%M%S)" - sudo python3 - "${dnsmasqStatus}" "${tmp_nodes}" <<'PY' -import json -import sys -from pathlib import Path - -status_path = Path(sys.argv[1]) -nodes_path = Path(sys.argv[2]) - -remove_names = set() -remove_ips = set() -remove_macs = set() -for line in nodes_path.read_text(encoding="utf-8").splitlines(): - parts = line.split("\t") - if len(parts) < 3: - continue - name, ip, mac = parts[:3] - if name: - remove_names.add(name) - if ip: - remove_ips.add(ip) - if mac: - remove_macs.add(mac.lower()) - -try: - data = json.loads(status_path.read_text(encoding="utf-8") or "[]") -except json.JSONDecodeError: - raise SystemExit(f"Invalid dnsmasq status JSON: {status_path}") - -filtered = [ - item for item in data - if not ( - item.get("hostname") in remove_names - or item.get("ip-address") in remove_ips - or str(item.get("mac-address", "")).lower() in remove_macs - ) -] -status_path.write_text(json.dumps(filtered, indent=2) + "\n", encoding="utf-8") -PY - rm -f "${tmp_nodes}" -} - -cleanupSetupOutputs() { - echo "Cleaning stale setup outputs matching: ${setupOutputGlob}" - rm -f ${setupOutputGlob} - rm -f "${CONFIG_PATH}" "${outputK3sConfig}" "${outputKubeconfig}" "${outputInventory}" - rm -f "${SETUP_DIR}/configRunning.yaml" -} - -verifyClean() { - local failed=0 - - echo "Verifying cleanup..." - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - [ -n "${name}" ] || continue - if virsh dominfo "${name}" >/dev/null 2>&1; then - echo "Residual domain: ${name}" >&2 - failed=1 - fi - if [ -e "${kvmDiskDir}/${name}.qcow2" ]; then - echo "Residual disk: ${kvmDiskDir}/${name}.qcow2" >&2 - failed=1 - fi - if [ -e "${kvmCloudInitDir}/${name}" ]; then - echo "Residual cloud-init dir: ${kvmCloudInitDir}/${name}" >&2 - failed=1 - fi - - if virsh net-dumpxml "${kvmNetwork}" 2>/dev/null \ - | grep -E "]*(mac=['\"]${mac}['\"]|name=['\"]${name}['\"]|ip=['\"]${ip}['\"])" >/dev/null 2>&1; then - echo "Residual DHCP reservation in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 - failed=1 - fi - - if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null \ - | awk -v name="${name}" -v ip="${ip}/24" -v mac="$(printf '%s' "${mac}" | tr '[:upper:]' '[:lower:]')" ' - NR > 2 { - row = tolower($0) - if (row ~ mac && $5 == ip && $6 == name) { - found = 1 - } - } - END { exit(found ? 0 : 1) } - '; then - echo "Residual DHCP lease in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 - failed=1 - fi - done < "${VERIFY_NODES_TSV}" - - if [ -e "${CONFIG_PATH}" ]; then - echo "Residual KVM state: ${CONFIG_PATH}" >&2 - failed=1 - fi - - if compgen -G "${setupOutputGlob}" >/dev/null; then - echo "Residual setup output files matching ${setupOutputGlob}" >&2 - compgen -G "${setupOutputGlob}" >&2 || true - failed=1 - fi - - if [ "${failed}" -ne 0 ]; then - echo "Cleanup verification failed." >&2 - exit 1 - fi - echo "Cleanup verification passed." -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - - requireCommand awk - requireCommand grep - requireCommand mktemp - requireCommand python3 - requireCommand sudo - requireCommand virsh - - trap cleanupTmp EXIT - loadConfig - printPlan - cleanupDomainsReservationsAndFiles - cleanupDnsmasqStaleLeases - cleanupSetupOutputs - verifyClean -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py index 8b4c50a85..db26816ea 100755 --- a/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py +++ b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py @@ -559,6 +559,9 @@ def to_k3s_node_payload(node: dict[str, Any]) -> dict[str, Any]: "name": node["name"], "role": node["role"], "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), "ssh": { "user": node.get("sshUser", ""), "key": node.get("sshKey", ""), @@ -625,7 +628,7 @@ def write_k3s_config(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.nodes_tsv: Transient node TSV generated by createKvmVms.sh. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. args.output: Destination configK3s.yaml path. """ data = load_config(args.config) @@ -655,7 +658,7 @@ def write_kvm_state(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.nodes_tsv: Transient node TSV generated by createKvmVms.sh. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. args.output: Destination kvmState.yaml path. """ data = load_config(args.config) @@ -734,7 +737,7 @@ def validate_kvm_state(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.state: Existing kvmState.yaml that createKvmVms.sh may reuse. + args.state: Existing kvmState.yaml that createKvmVms.py may reuse. """ data = load_config(args.config) expected = nodes(data) diff --git a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py index 55a600669..73f5e5e27 100755 --- a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py +++ b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py @@ -1,21 +1,317 @@ #!/usr/bin/env python3 -"""Python entrypoint for prepareHostAssets.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for prepareHostAssets with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare host-side assets required before KVM creation: +# Ubuntu cloud image, registry/K3s bootstrap image tarballs, and SeedEMU base +# image tags. All user-configurable paths come from kvm.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" + +HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" +hostDockerIoMirror="docker.m.daocloud.io" +dockerPullTimeoutSeconds=180 +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 +prepareForce="false" +kvmBaseImageReuseMode="copy" +REGISTRY_BOOTSTRAP_IMAGE="registry:2" +MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" +K3S_SYSTEM_BOOTSTRAP_IMAGES=( + "rancher/mirrored-coredns-coredns:1.10.1" + "rancher/mirrored-metrics-server:v0.6.3" + "rancher/local-path-provisioner:v0.0.24" +) +seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" +seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" +seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" +seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" +ubuntuBuildImage="ubuntu:20.04" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveSeedEmulatorDockerDir() { + # Resolve the source-tree path containing seedemu-base and seedemu-router. + # kvm.yaml seedemu.dockerImagesDir wins through manageKvmConfig.py; the + # ancestor scan keeps examples runnable directly from this repository. + local cursor="${SETUP_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${hostDockerIoMirror}" "${image}" + else + printf '%s/library/%s\n' "${hostDockerIoMirror}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " docker pull ${image}" + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " docker pull ${mirror_image}" + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare Docker image on host: ${image}" >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + if [ "${prepareForce}" != "true" ] && [ -s "${tar_path}" ]; then + echo " cache exists: ${tar_path}" + return + fi + ensureHostDockerImage "${image}" + echo " docker save ${image} -> ${tar_path}" + docker save -o "${tar_path}" "${image}" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + # Downloads through a resumable .part file and validates qcow2 metadata + # before publishing the final path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +prepareBaseImage() { + mkdir -p "$(dirname "${kvmBaseImagePath}")" + + if [ -e "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + echo "Base image already exists: ${kvmBaseImagePath}" + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Reusing existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Reusing existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + + echo "Downloading Ubuntu cloud image:" + echo " url=${kvmBaseImageUrl}" + echo " output=${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +prepareSeedemuBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-base" ]; then + echo " docker build ${seedBaseSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + else + ensureHostDockerImage "${seedBaseSourceImage}" + fi + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-router" ]; then + echo " docker build ${seedRouterSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + else + ensureHostDockerImage "${seedRouterSourceImage}" + fi + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareImageCache() { + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + prepareSeedemuBuildImages + + saveHostImageTarball "${REGISTRY_BOOTSTRAP_IMAGE}" + saveHostImageTarball "${MULTUS_BOOTSTRAP_IMAGE}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}"; do + saveHostImageTarball "${image}" + done + saveHostImageTarball "${ubuntuBuildImage}" + saveHostImageTarball "${seedBaseSourceImage}" + saveHostImageTarball "${seedRouterSourceImage}" + + echo " hash tags prepared locally, not saved into image-cache:" + echo " ${seedBaseHashImage}" + echo " ${seedRouterHashImage}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand python3 + requireCommand curl + requireCommand docker + requireCommand find + requireCommand qemu-img + + eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + resolveSeedEmulatorDockerDir + + echo "Preparing setup assets using config: ${CONFIG_PATH}" + echo "base_image_path=${kvmBaseImagePath}" + echo "image_cache_dir=${HOST_IMAGE_CACHE_DIR}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + prepareBaseImage + prepareImageCache + echo "Setup assets are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh deleted file mode 100755 index 65d602426..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env bash -# Prepare host-side assets required before KVM creation: -# Ubuntu cloud image, registry/K3s bootstrap image tarballs, and SeedEMU base -# image tags. All user-configurable paths come from kvm.yaml. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" - -HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" -hostDockerIoMirror="docker.m.daocloud.io" -dockerPullTimeoutSeconds=180 -baseImageLowSpeedSeconds=60 -baseImageLowSpeedLimit=1024 -prepareForce="false" -kvmBaseImageReuseMode="copy" -REGISTRY_BOOTSTRAP_IMAGE="registry:2" -MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" -K3S_SYSTEM_BOOTSTRAP_IMAGES=( - "rancher/mirrored-coredns-coredns:1.10.1" - "rancher/mirrored-metrics-server:v0.6.3" - "rancher/local-path-provisioner:v0.0.24" -) -seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" -kvmBaseImageSearchDirs="${HOME}/k8s/output" -seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" -seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" -seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" -seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" -ubuntuBuildImage="ubuntu:20.04" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -resolveSeedEmulatorDockerDir() { - # Resolve the source-tree path containing seedemu-base and seedemu-router. - # kvm.yaml seedemu.dockerImagesDir wins through manageKvmConfig.py; the - # ancestor scan keeps examples runnable directly from this repository. - local cursor="${SETUP_DIR}" - while [ "${cursor}" != "/" ]; do - if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && - [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then - seedEmulatorDockerDir="${cursor}/docker_images/multiarch" - return 0 - fi - cursor="$(dirname "${cursor}")" - done - local candidate - for candidate in \ - "${seedEmulatorDockerDir}" \ - "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ - "${HOME}/k8s/seed-emulator/docker_images/multiarch"; do - if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then - seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" - return 0 - fi - done -} - -imageTarName() { - printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' -} - -dockerIoMirrorRef() { - local image="$1" - - if [[ "${image}" == */*/* ]]; then - return 1 - fi - - if [[ "${image}" == */* ]]; then - printf '%s/%s\n' "${hostDockerIoMirror}" "${image}" - else - printf '%s/library/%s\n' "${hostDockerIoMirror}" "${image}" - fi -} - -ensureHostDockerImage() { - local image="$1" - local mirror_image="" - - if docker image inspect "${image}" >/dev/null 2>&1; then - return 0 - fi - - echo " docker pull ${image}" - if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then - return 0 - fi - - if mirror_image="$(dockerIoMirrorRef "${image}")"; then - echo " docker pull ${mirror_image}" - runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null - docker tag "${mirror_image}" "${image}" >/dev/null - return 0 - fi - - echo "Failed to prepare Docker image on host: ${image}" >&2 - return 1 -} - -hostImageTarball() { - local image="$1" - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" -} - -saveHostImageTarball() { - local image="$1" - local tar_path - tar_path="$(hostImageTarball "${image}")" - if [ "${prepareForce}" != "true" ] && [ -s "${tar_path}" ]; then - echo " cache exists: ${tar_path}" - return - fi - ensureHostDockerImage "${image}" - echo " docker save ${image} -> ${tar_path}" - docker save -o "${tar_path}" "${image}" -} - -validBaseImage() { - # Args: - # $1: qcow2 image path to validate. - local image_path="$1" - [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 -} - -downloadBaseImageFile() { - # Args: - # $1: URL, $2: destination path. - # Downloads through a resumable .part file and validates qcow2 metadata - # before publishing the final path. - local image_url="$1" - local image_path="$2" - local partial_path="${image_path}.part" - mkdir -p "$(dirname "${image_path}")" - curl -fL \ - --retry 5 \ - --retry-delay 5 \ - --retry-all-errors \ - --connect-timeout 20 \ - --speed-time "${baseImageLowSpeedSeconds}" \ - --speed-limit "${baseImageLowSpeedLimit}" \ - -C - \ - -o "${partial_path}" \ - "${image_url}" - validBaseImage "${partial_path}" || { - echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 - return 1 - } - mv -f "${partial_path}" "${image_path}" -} - -prepareBaseImage() { - mkdir -p "$(dirname "${kvmBaseImagePath}")" - - if [ -e "${kvmBaseImagePath}" ]; then - if validBaseImage "${kvmBaseImagePath}"; then - echo "Base image already exists: ${kvmBaseImagePath}" - return - fi - echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" - rm -f "${kvmBaseImagePath}" - fi - - if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then - echo "Reusing existing base image from ${kvmLegacyBaseImagePath}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - fi - return - fi - - local output_image="" - local search_dir="" - for search_dir in ${kvmBaseImageSearchDirs}; do - [ -d "${search_dir}" ] || continue - output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" - if [ -n "${output_image}" ]; then - echo "Reusing existing base image from ${output_image}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${output_image}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" - fi - return - fi - done - - echo "Downloading Ubuntu cloud image:" - echo " url=${kvmBaseImageUrl}" - echo " output=${kvmBaseImagePath}" - downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" -} - -prepareSeedemuBuildImages() { - ensureHostDockerImage "${ubuntuBuildImage}" - - if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then - if [ -d "${seedEmulatorDockerDir}/seedemu-base" ]; then - echo " docker build ${seedBaseSourceImage}" - DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-base" >/dev/null - else - ensureHostDockerImage "${seedBaseSourceImage}" - fi - fi - - if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then - if [ -d "${seedEmulatorDockerDir}/seedemu-router" ]; then - echo " docker build ${seedRouterSourceImage}" - DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-router" >/dev/null - else - ensureHostDockerImage "${seedRouterSourceImage}" - fi - fi - - docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null - docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null -} - -prepareImageCache() { - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - prepareSeedemuBuildImages - - saveHostImageTarball "${REGISTRY_BOOTSTRAP_IMAGE}" - saveHostImageTarball "${MULTUS_BOOTSTRAP_IMAGE}" - for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}"; do - saveHostImageTarball "${image}" - done - saveHostImageTarball "${ubuntuBuildImage}" - saveHostImageTarball "${seedBaseSourceImage}" - saveHostImageTarball "${seedRouterSourceImage}" - - echo " hash tags prepared locally, not saved into image-cache:" - echo " ${seedBaseHashImage}" - echo " ${seedRouterHashImage}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - - requireCommand python3 - requireCommand curl - requireCommand docker - requireCommand find - requireCommand qemu-img - - eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" - resolveSeedEmulatorDockerDir - - echo "Preparing setup assets using config: ${CONFIG_PATH}" - echo "base_image_path=${kvmBaseImagePath}" - echo "image_cache_dir=${HOST_IMAGE_CACHE_DIR}" - echo "seedemu_docker_dir=${seedEmulatorDockerDir}" - prepareBaseImage - prepareImageCache - echo "Setup assets are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py index a29d87221..3ec843914 100755 --- a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py +++ b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py @@ -1,21 +1,299 @@ #!/usr/bin/env python3 -"""Python entrypoint for tuneVmLimits.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for tuneVmLimits with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Tune OS-level limits on each VM listed in configK3s.yaml. +# Inputs: configK3s.yaml with nodes and SSH settings. +# Outputs/side effects: writes sysctl, limits, systemd drop-ins, and cni0 +# hash_max service on each VM through SSH. No K3s installation is performed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +# Keep these defaults in the script, not in configK3s.yaml. The YAML describes VMs; +# this script describes the current high-density OS limit policy. +vmLimitNofile="10485760" +vmLimitNproc="4194304" +vmLimitMaxNetNamespaces="65536" +vmLimitNeighGcThresh1="1048576" +vmLimitNeighGcThresh2="4194304" +vmLimitNeighGcThresh3="8388608" +vmLimitNetdevMaxBacklog="1000000" +vmLimitOptmemMax="25165824" +vmLimitCni0HashMax="16384" +vmLimitReboot="false" + +NODE_SSH_OPTS=() +nodeSshUser="" +nodeSshKey="" +nodeConnection="" + +usage() { + cat <&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +requireCommand() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +waitForSsh() { + local name="$1" + local ip="$2" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + sudo -n true >/dev/null 2>&1 || { + echo "Local sudo failed for ${name} (${ip})" >&2 + return 1 + } + return 0 + fi + if ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ok" >/dev/null 2>&1; then + return 0 + fi + echo "SSH failed for ${name} (${ip})" >&2 + return 1 +} + +unlockOneVm() { + local name="$1" + local ip="$2" + local remote_runner=() + echo "Unlocking VM limits: ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${vmLimitNofile}" \ + "${vmLimitNproc}" \ + "${vmLimitMaxNetNamespaces}" \ + "${vmLimitNeighGcThresh1}" \ + "${vmLimitNeighGcThresh2}" \ + "${vmLimitNeighGcThresh3}" \ + "${vmLimitNetdevMaxBacklog}" \ + "${vmLimitOptmemMax}" \ + "${vmLimitCni0HashMax}" \ + "${vmLimitReboot}" <<'EOF_REMOTE' +set -euo pipefail + +LIMIT_NOFILE="$1" +LIMIT_NPROC="$2" +MAX_NET_NS="$3" +NEIGH1="$4" +NEIGH2="$5" +NEIGH3="$6" +NETDEV_BACKLOG="$7" +OPTMEM_MAX="$8" +CNI0_HASH_MAX="$9" +DO_REBOOT="${10}" + +appendIfMissing() { + local file="$1" + local pattern="$2" + local line="$3" + touch "${file}" + if ! grep -qF "${pattern}" "${file}"; then + printf '%s\n' "${line}" >> "${file}" + fi +} + +echo ">> limits.conf" +appendIfMissing /etc/security/limits.conf "* soft nofile" "* soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* hard nofile" "* hard nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* soft nproc" "* soft nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "* hard nproc" "* hard nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "root soft nofile" "root soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "root hard nofile" "root hard nofile ${LIMIT_NOFILE}" + +echo ">> sysctl" +cat > /etc/sysctl.d/99-seed-vm-limits.conf </dev/null 2>&1 || true +ip -s -s neigh flush all >/dev/null 2>&1 || true + +echo ">> systemd default limits" +mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d +cat > /etc/systemd/system.conf.d/99-seed-vm-limits.conf < /etc/systemd/user.conf.d/99-seed-vm-limits.conf <> known service drop-ins if present" +for service in k3s k3s-agent containerd docker; do + if systemctl list-unit-files | grep -q "^${service}.service"; then + mkdir -p "/etc/systemd/system/${service}.service.d" + cat > "/etc/systemd/system/${service}.service.d/99-seed-vm-limits.conf" <> cni0 hash_max service for future K3s/flannel bridge" +cat > /usr/local/sbin/seed-vm-cni0-hashmax.sh <<'EOF_TUNE' +#!/usr/bin/env bash +set -euo pipefail +target="${1:-16384}" +for _ in $(seq 1 60); do + if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + printf '%s\n' "${target}" > /sys/class/net/cni0/bridge/hash_max + exit 0 + fi + sleep 2 +done +exit 0 +EOF_TUNE +chmod +x /usr/local/sbin/seed-vm-cni0-hashmax.sh + +cat > /etc/systemd/system/seed-vm-cni0-hashmax.service </dev/null 2>&1 || true +if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + /usr/local/sbin/seed-vm-cni0-hashmax.sh "${CNI0_HASH_MAX}" >/dev/null 2>&1 || true +fi + +if [ "${DO_REBOOT}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "reboot" >/dev/null 2>&1 || true +fi + +echo "VM limit unlock completed on $(hostname)" +EOF_REMOTE +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + echo "Using K3s config: ${CONFIG_PATH}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < <(nodesInput) + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + unlockOneVm "${name}" "${ip}" + done < <(nodesInput) + + echo "VM limit unlock completed for all nodes in ${CONFIG_PATH}" +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh deleted file mode 100755 index 7ce9a4cfa..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env bash -# Tune OS-level limits on each VM listed in configK3s.yaml. -# Inputs: configK3s.yaml with nodes and SSH settings. -# Outputs/side effects: writes sysctl, limits, systemd drop-ins, and cni0 -# hash_max service on each VM through SSH. No K3s installation is performed. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" - -# Keep these defaults in the script, not in configK3s.yaml. The YAML describes VMs; -# this script describes the current high-density OS limit policy. -vmLimitNofile="10485760" -vmLimitNproc="4194304" -vmLimitMaxNetNamespaces="65536" -vmLimitNeighGcThresh1="1048576" -vmLimitNeighGcThresh2="4194304" -vmLimitNeighGcThresh3="8388608" -vmLimitNetdevMaxBacklog="1000000" -vmLimitOptmemMax="25165824" -vmLimitCni0HashMax="16384" -vmLimitReboot="false" - -NODE_SSH_OPTS=() -nodeSshUser="" -nodeSshKey="" -nodeConnection="" - -usage() { - cat <&2 - exit 1 - } - NODE_SSH_OPTS=( - -i "${nodeSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -requireCommand() { - command -v "$1" >/dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -waitForSsh() { - local name="$1" - local ip="$2" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - sudo -n true >/dev/null 2>&1 || { - echo "Local sudo failed for ${name} (${ip})" >&2 - return 1 - } - return 0 - fi - if ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ok" >/dev/null 2>&1; then - return 0 - fi - echo "SSH failed for ${name} (${ip})" >&2 - return 1 -} - -unlockOneVm() { - local name="$1" - local ip="$2" - local remote_runner=() - echo "Unlocking VM limits: ${name} (${ip})" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - remote_runner=(sudo -n bash -s --) - else - remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) - fi - "${remote_runner[@]}" \ - "${vmLimitNofile}" \ - "${vmLimitNproc}" \ - "${vmLimitMaxNetNamespaces}" \ - "${vmLimitNeighGcThresh1}" \ - "${vmLimitNeighGcThresh2}" \ - "${vmLimitNeighGcThresh3}" \ - "${vmLimitNetdevMaxBacklog}" \ - "${vmLimitOptmemMax}" \ - "${vmLimitCni0HashMax}" \ - "${vmLimitReboot}" <<'EOF_REMOTE' -set -euo pipefail - -LIMIT_NOFILE="$1" -LIMIT_NPROC="$2" -MAX_NET_NS="$3" -NEIGH1="$4" -NEIGH2="$5" -NEIGH3="$6" -NETDEV_BACKLOG="$7" -OPTMEM_MAX="$8" -CNI0_HASH_MAX="$9" -DO_REBOOT="${10}" - -appendIfMissing() { - local file="$1" - local pattern="$2" - local line="$3" - touch "${file}" - if ! grep -qF "${pattern}" "${file}"; then - printf '%s\n' "${line}" >> "${file}" - fi -} - -echo ">> limits.conf" -appendIfMissing /etc/security/limits.conf "* soft nofile" "* soft nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "* hard nofile" "* hard nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "* soft nproc" "* soft nproc ${LIMIT_NPROC}" -appendIfMissing /etc/security/limits.conf "* hard nproc" "* hard nproc ${LIMIT_NPROC}" -appendIfMissing /etc/security/limits.conf "root soft nofile" "root soft nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "root hard nofile" "root hard nofile ${LIMIT_NOFILE}" - -echo ">> sysctl" -cat > /etc/sysctl.d/99-seed-vm-limits.conf </dev/null 2>&1 || true -ip -s -s neigh flush all >/dev/null 2>&1 || true - -echo ">> systemd default limits" -mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d -cat > /etc/systemd/system.conf.d/99-seed-vm-limits.conf < /etc/systemd/user.conf.d/99-seed-vm-limits.conf <> known service drop-ins if present" -for service in k3s k3s-agent containerd docker; do - if systemctl list-unit-files | grep -q "^${service}.service"; then - mkdir -p "/etc/systemd/system/${service}.service.d" - cat > "/etc/systemd/system/${service}.service.d/99-seed-vm-limits.conf" <> cni0 hash_max service for future K3s/flannel bridge" -cat > /usr/local/sbin/seed-vm-cni0-hashmax.sh <<'EOF_TUNE' -#!/usr/bin/env bash -set -euo pipefail -target="${1:-16384}" -for _ in $(seq 1 60); do - if [ -w /sys/class/net/cni0/bridge/hash_max ]; then - printf '%s\n' "${target}" > /sys/class/net/cni0/bridge/hash_max - exit 0 - fi - sleep 2 -done -exit 0 -EOF_TUNE -chmod +x /usr/local/sbin/seed-vm-cni0-hashmax.sh - -cat > /etc/systemd/system/seed-vm-cni0-hashmax.service </dev/null 2>&1 || true -if [ -w /sys/class/net/cni0/bridge/hash_max ]; then - /usr/local/sbin/seed-vm-cni0-hashmax.sh "${CNI0_HASH_MAX}" >/dev/null 2>&1 || true -fi - -if [ "${DO_REBOOT}" = "true" ]; then - systemd-run --on-active=2 /bin/bash -c "reboot" >/dev/null 2>&1 || true -fi - -echo "VM limit unlock completed on $(hostname)" -EOF_REMOTE -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - echo "Using K3s config: ${CONFIG_PATH}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - waitForSsh "${name}" "${ip}" - done < <(nodesInput) - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - unlockOneVm "${name}" "${ip}" - done < <(nodesInput) - - echo "VM limit unlock completed for all nodes in ${CONFIG_PATH}" -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/manageK3sConfig.py b/seedemu/k8sTools/resources/setup/manageK3sConfig.py index 3bd63788b..3566b2c08 100644 --- a/seedemu/k8sTools/resources/setup/manageK3sConfig.py +++ b/seedemu/k8sTools/resources/setup/manageK3sConfig.py @@ -233,7 +233,7 @@ def masterNode(nodes: list[dict[str, Any]]) -> dict[str, Any]: def configValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, Any]: - """Build the local shell variable values consumed by applyK3sCluster.sh.""" + """Build the local shell variable values consumed by applyK3sCluster.py.""" cluster_name = str(data.get("clusterName") or data.get("cluster_name") or "seedemu-k3s") master = masterNode(nodes) registry_host = getNested(data, "registry.host", master["ip"]) @@ -285,7 +285,7 @@ def fabricValues(data: dict[str, Any]) -> dict[str, Any]: The first implementation intentionally supports a two-node Linux VXLAN fabric. Larger physical fabrics should use a real L2/VLAN/EVPN/OVS design - instead of an implicit shell-script full mesh. + instead of an implicit command-driven full mesh. """ values = { "fabricType": str(getNested(data, "fabric.type", "none")), @@ -309,7 +309,7 @@ def ovnValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, An data: Parsed configK3s.yaml mapping. nodes: Normalized node list used to discover the master IP. - These values are consumed by ovn/installKubeOvnFabric.sh. Kube-OVN runs as + These values are consumed by ovn/installKubeOvnFabric.py. Kube-OVN runs as a secondary CNI while K3s/flannel remains the primary eth0 network. """ vals = configValues(data, nodes) @@ -432,7 +432,7 @@ def fabricNodeRows(data: dict[str, Any], nodes: list[dict[str, Any]]) -> list[di The user may set fabric.nodes..underlayInterface explicitly. If it is omitted, the helper detects the underlay by asking the node which interface it uses to route to the peer management IP. Test IPs are internal defaults - and are only used by validateLinuxVxlanFabric.sh. + and are only used by validateLinuxVxlanFabric.py. """ values = fabricValues(data) if values["fabricType"] == "none": @@ -582,7 +582,7 @@ def ansibleHostVars(node: dict[str, Any], k3s_role: str, as_group: str) -> dict[ def commandWriteAnsibleInventory(args: argparse.Namespace) -> None: - """Write the temporary Ansible inventory used by applyK3sCluster.sh.""" + """Write the temporary Ansible inventory used by applyK3sCluster.py.""" data = loadYaml(args.config) nodes = yamlNodes(data) vals = configValues(data, nodes) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py index 29fd1da73..c9b5d51e0 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py @@ -1,21 +1,256 @@ #!/usr/bin/env python3 -"""Python entrypoint for createMultiHostKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for createMultiHostKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create KVM VMs across multiple physical hypervisors. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Outputs: +# - Host-local kvm.yaml files under setup/tmp/multiHostKvm/. +# - Remote host-local setup directories under hypervisors[].remoteWorkDir. +# - Global configK3s.yaml for applyK3sCluster.py. +# - Global multiHostKvmState.yaml for destroyMultiHostKvmVms.py. +# +# Execution context: +# Run after prepareKvmHypervisors.py. This script creates VMs on the target +# hypervisors but does not install K3s. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +LOCAL_WORK_DIR="" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +copyVmSshKeyToHost() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key. + # + # Remote hypervisors run kvm/createKvmVms.py locally, so they need a local + # copy of the VM SSH private key to inject its public key into cloud-init + # and wait for SSH readiness. The global configK3s.yaml still keeps the + # original control-host key path for later K3s installation. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local source_key target_key source_pub target_pub + + IFS=$'\t' read -r source_key target_key source_pub target_pub < <( + python3 "${HELPER}" --config "${CONFIG_PATH}" host-vm-ssh-key-tsv --host "${name}" + ) + [ -f "${source_key}" ] || { + echo "VM SSH key not found on control host: ${source_key}" >&2 + exit 1 + } + if [ "${connection}" = "local" ]; then + return + fi + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "$(dirname "${target_key}")")" + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_key}" "${ssh_user}@${ip}:${target_key}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 600 $(printf '%q' "${target_key}")" + if [ -f "${source_pub}" ]; then + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_pub}" "${ssh_user}@${ip}:${target_pub}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 644 $(printf '%q' "${target_pub}")" + fi +} + +prepareLocalWorkDir() { + mkdir -p "${outputTmpDir}" + LOCAL_WORK_DIR="${outputTmpDir}/multiHostKvm" + rm -rf "${LOCAL_WORK_DIR}" + mkdir -p "${LOCAL_WORK_DIR}" +} + +createHostVms() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local remote_work_dir="${10}" + local host_dir="${LOCAL_WORK_DIR}/${name}" + local host_kvm_yaml="${host_dir}/kvm.yaml" + + echo "[multi-host-create] ${name} (${ip})" + mkdir -p "${host_dir}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-local-kvm \ + --host "${name}" \ + --output "${host_kvm_yaml}" >/dev/null + + syncDirToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${SETUP_DIR}/kvm" "${remote_work_dir}/kvm" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${host_kvm_yaml}" "${remote_work_dir}/kvm.yaml" + copyVmSshKeyToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + cd $(printf '%q' "${remote_work_dir}") + python3 ./kvm/prepareHostAssets.py ./kvm.yaml + python3 ./kvm/createKvmVms.py ./kvm.yaml + " +} + +writeGlobalOutputs() { + python3 "${HELPER}" --config "${CONFIG_PATH}" write-global-k3s-config \ + --output "${outputConfigK3s}" >/dev/null + python3 "${HELPER}" --config "${CONFIG_PATH}" write-state \ + --output "${outputMultiHostKvmState}" >/dev/null + echo "K3s config: ${outputConfigK3s}" + echo "Multi-host KVM state: ${outputMultiHostKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand rsync + requireCommand cp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + prepareLocalWorkDir + python3 "${HELPER}" --config "${CONFIG_PATH}" validate + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + createHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + writeGlobalOutputs + echo "Multi-host KVM VMs are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh deleted file mode 100755 index 3a148544a..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash -# Create KVM VMs across multiple physical hypervisors. -# -# Inputs: -# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. -# -# Outputs: -# - Host-local kvm.yaml files under setup/tmp/multiHostKvm/. -# - Remote host-local setup directories under hypervisors[].remoteWorkDir. -# - Global configK3s.yaml for buildK3sCluster.sh. -# - Global multiHostKvmState.yaml for destroyMultiHostKvmVms.py. -# -# Execution context: -# Run after prepareKvmHypervisors.py. This script creates VMs on the target -# hypervisors but does not install K3s. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" -LOCAL_WORK_DIR="" - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null -} - -copyVmSshKeyToHost() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key. - # - # Remote hypervisors run kvm/createKvmVms.py locally, so they need a local - # copy of the VM SSH private key to inject its public key into cloud-init - # and wait for SSH readiness. The global configK3s.yaml still keeps the - # original control-host key path for later K3s installation. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local source_key target_key source_pub target_pub - - IFS=$'\t' read -r source_key target_key source_pub target_pub < <( - python3 "${HELPER}" --config "${CONFIG_PATH}" host-vm-ssh-key-tsv --host "${name}" - ) - [ -f "${source_key}" ] || { - echo "VM SSH key not found on control host: ${source_key}" >&2 - exit 1 - } - if [ "${connection}" = "local" ]; then - return - fi - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "mkdir -p $(printf '%q' "$(dirname "${target_key}")")" - # shellcheck disable=SC2046 - scp $(sshOptions "${ssh_key}") "${source_key}" "${ssh_user}@${ip}:${target_key}" >/dev/null - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "chmod 600 $(printf '%q' "${target_key}")" - if [ -f "${source_pub}" ]; then - # shellcheck disable=SC2046 - scp $(sshOptions "${ssh_key}") "${source_pub}" "${ssh_user}@${ip}:${target_pub}" >/dev/null - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "chmod 644 $(printf '%q' "${target_pub}")" - fi -} - -prepareLocalWorkDir() { - mkdir -p "${outputTmpDir}" - LOCAL_WORK_DIR="${outputTmpDir}/multiHostKvm" - rm -rf "${LOCAL_WORK_DIR}" - mkdir -p "${LOCAL_WORK_DIR}" -} - -createHostVms() { - # Args are read from one hosts-tsv row. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local remote_work_dir="${10}" - local host_dir="${LOCAL_WORK_DIR}/${name}" - local host_kvm_yaml="${host_dir}/kvm.yaml" - - echo "[multi-host-create] ${name} (${ip})" - mkdir -p "${host_dir}" - python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-local-kvm \ - --host "${name}" \ - --output "${host_kvm_yaml}" >/dev/null - - syncDirToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${SETUP_DIR}/kvm" "${remote_work_dir}/kvm" - copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${host_kvm_yaml}" "${remote_work_dir}/kvm.yaml" - copyVmSshKeyToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - cd $(printf '%q' "${remote_work_dir}") - python3 ./kvm/prepareHostAssets.py ./kvm.yaml - python3 ./kvm/createKvmVms.py ./kvm.yaml - " -} - -writeGlobalOutputs() { - python3 "${HELPER}" --config "${CONFIG_PATH}" write-global-k3s-config \ - --output "${outputConfigK3s}" >/dev/null - python3 "${HELPER}" --config "${CONFIG_PATH}" write-state \ - --output "${outputMultiHostKvmState}" >/dev/null - echo "K3s config: ${outputConfigK3s}" - echo "Multi-host KVM state: ${outputMultiHostKvmState}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - requireCommand scp - requireCommand rsync - requireCommand cp - - [ -s "${CONFIG_PATH}" ] || { - echo "Missing config: ${CONFIG_PATH}" >&2 - exit 1 - } - - prepareLocalWorkDir - python3 "${HELPER}" --config "${CONFIG_PATH}" validate - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - createHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) - writeGlobalOutputs - echo "Multi-host KVM VMs are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py index 8d8a21890..9437d7b16 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py @@ -1,21 +1,166 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyMultiHostKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyMultiHostKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs and routed networks created by the multi-host KVM flow. +# +# Inputs: +# $1: multiHostKvmState.yaml. Defaults to ../multiHostKvmState.yaml. +# +# Side effects: +# - Runs each host-local kvm/destroyKvmVms.py on its owning hypervisor. +# - Removes static routes between hypervisor VM subnets. +# - Removes optional point-to-point VXLAN route tunnels. +# - Destroys and undefines the generated libvirt routed networks. +# - Removes generated local configK3s/kubeconfig/inventory/state outputs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +STATE_PATH="${1:-${SETUP_DIR}/multiHostKvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" &2; sleep 2; python3 ./kvm/destroyKvmVms.py ./kvmState.yaml; } + fi + virsh net-destroy $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-undefine $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -t nat -D POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + fi + rm -rf $(printf '%q' "${remote_work_dir}") + " + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + echo " delete route ${remote_cidr} via ${peer_ip} (${peer_name})" + if [ -n "${route_dev}" ]; then + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") dev $(printf '%q' "${route_dev}") >/dev/null 2>&1 || sudo -n ip route del $(printf '%q' "${remote_cidr}") >/dev/null 2>&1 || true" + else + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") >/dev/null 2>&1 || true" + fi + done < <(python3 "${HELPER}" --config /dev/null state-routes-tsv --state "${STATE_PATH}" --host "${name}") + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " delete tunnel ${tunnel_name} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true" + done < <(python3 "${HELPER}" --config /dev/null state-tunnels-tsv --state "${STATE_PATH}" --host "${name}") +} + +cleanupLocalOutputs() { + eval "$(python3 "${HELPER}" --config /dev/null state-output-vars --state "${STATE_PATH}")" + rm -f "${stateConfigK3s}" + rm -f "${stateKubeconfig}" + rm -f "${stateInventory}" + rm -f "${STATE_PATH}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + + [ -s "${STATE_PATH}" ] || { + echo "Missing state file: ${STATE_PATH}" >&2 + exit 1 + } + + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + destroyHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config /dev/null state-hosts-tsv --state "${STATE_PATH}") + cleanupLocalOutputs + echo "Multi-host KVM cleanup completed." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh deleted file mode 100755 index a7051f182..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -# Destroy KVM VMs and routed networks created by the multi-host KVM flow. -# -# Inputs: -# $1: multiHostKvmState.yaml. Defaults to ../multiHostKvmState.yaml. -# -# Side effects: -# - Runs each host-local kvm/destroyKvmVms.py on its owning hypervisor. -# - Removes static routes between hypervisor VM subnets. -# - Removes optional point-to-point VXLAN route tunnels. -# - Destroys and undefines the generated libvirt routed networks. -# - Removes generated local configK3s/kubeconfig/inventory/state outputs. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -STATE_PATH="${1:-${SETUP_DIR}/multiHostKvmState.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" &2; sleep 2; python3 ./kvm/destroyKvmVms.py ./kvmState.yaml; } - fi - virsh net-destroy $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - virsh net-undefine $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - if command -v iptables >/dev/null 2>&1; then - sudo -n iptables -t nat -D POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || true - sudo -n iptables -D FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true - sudo -n iptables -D FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true - fi - rm -rf $(printf '%q' "${remote_work_dir}") - " - - while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do - [ -n "${remote_cidr}" ] || continue - echo " delete route ${remote_cidr} via ${peer_ip} (${peer_name})" - if [ -n "${route_dev}" ]; then - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route del $(printf '%q' "${remote_cidr}") dev $(printf '%q' "${route_dev}") >/dev/null 2>&1 || sudo -n ip route del $(printf '%q' "${remote_cidr}") >/dev/null 2>&1 || true" - else - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route del $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") >/dev/null 2>&1 || true" - fi - done < <(python3 "${HELPER}" --config /dev/null state-routes-tsv --state "${STATE_PATH}" --host "${name}") - - while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do - [ -n "${tunnel_name}" ] || continue - echo " delete tunnel ${tunnel_name} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true" - done < <(python3 "${HELPER}" --config /dev/null state-tunnels-tsv --state "${STATE_PATH}" --host "${name}") -} - -cleanupLocalOutputs() { - eval "$(python3 "${HELPER}" --config /dev/null state-output-vars --state "${STATE_PATH}")" - rm -f "${stateConfigK3s}" - rm -f "${stateKubeconfig}" - rm -f "${stateInventory}" - rm -f "${STATE_PATH}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - - [ -s "${STATE_PATH}" ] || { - echo "Missing state file: ${STATE_PATH}" >&2 - exit 1 - } - - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - destroyHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config /dev/null state-hosts-tsv --state "${STATE_PATH}") - cleanupLocalOutputs - echo "Multi-host KVM cleanup completed." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py index 094f5c0c0..2bd7bb324 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py @@ -576,7 +576,7 @@ def writeHostNetworkXml(args: argparse.Namespace) -> None: def writeHostLocalKvm(args: argparse.Namespace) -> None: - """Write host-local kvm.yaml consumed by kvm/createKvmVms.sh.""" + """Write host-local kvm.yaml consumed by kvm/createKvmVms.py.""" data = loadYaml(args.config) host = requireHost(hostList(data), args.host) vm_ssh = data.get("vmSsh") if isinstance(data.get("vmSsh"), dict) else {} @@ -631,6 +631,9 @@ def writeGlobalK3sConfig(args: argparse.Namespace) -> None: "name": node["name"], "role": node["role"], "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), "ssh": {"user": ssh_user, "key": ssh_key}, } for node in vmPlan(data) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py index cd1d90f07..ac9d86ebb 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py @@ -1,21 +1,217 @@ #!/usr/bin/env python3 -"""Python entrypoint for prepareKvmHypervisors.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for prepareKvmHypervisors with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare physical KVM hypervisors for a multi-host KVM SeedEMU cluster. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Generated/modified state: +# - Creates one routed libvirt network per hypervisor. +# - Enables IPv4 forwarding on each hypervisor. +# - Optionally creates point-to-point VXLAN route tunnels. +# - Installs static routes between the hypervisor VM subnets. +# +# Execution context: +# Run from the generated setup directory before createMultiHostKvmVms.py. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +TMP_DIR="" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +cleanupTmp() { + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" || true +} + +validateConfig() { + python3 "${HELPER}" --config "${CONFIG_PATH}" validate +} + +prepareHost() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local network_name="$6" + local bridge_name="$7" + local cidr="$8" + local remote_work_dir="${10}" + local network_xml="${TMP_DIR}/${name}.xml" + local remote_xml="${remote_work_dir}/network.xml" + + echo "[hypervisor-prepare] ${name} (${ip}) network=${network_name} bridge=${bridge_name}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-network-xml \ + --host "${name}" \ + --output "${network_xml}" >/dev/null + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n true && command -v virsh >/dev/null && command -v virt-install >/dev/null && command -v qemu-img >/dev/null && command -v docker >/dev/null && command -v curl >/dev/null && command -v ssh >/dev/null" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "${remote_work_dir}")" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" "${network_xml}" "${remote_xml}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + virsh net-info $(printf '%q' "${network_name}") >/dev/null 2>&1 || virsh net-define $(printf '%q' "${remote_xml}") >/dev/null + virsh net-start $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-autostart $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + sudo -n sysctl -w net.ipv4.ip_forward=1 >/dev/null + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -C FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -C FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -t nat -C POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || sudo -n iptables -t nat -A POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE + fi + " + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " tunnel ${tunnel_name}: ${local_tunnel_cidr} -> ${peer_tunnel_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + underlay_dev=\$(ip route get $(printf '%q' "${peer_ip}") | awk '{for (i=1;i<=NF;i++) if (\$i==\"dev\") {print \$(i+1); exit}}') + [ -n \"\${underlay_dev}\" ] || { echo 'Cannot determine underlay dev for peer $(printf '%q' "${peer_ip}")' >&2; exit 1; } + sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true + sudo -n ip link add $(printf '%q' "${tunnel_name}") type vxlan id $(printf '%q' "${vni}") local $(printf '%q' "${local_ip}") remote $(printf '%q' "${peer_ip}") dstport $(printf '%q' "${dst_port}") dev \"\${underlay_dev}\" + sudo -n ip addr add $(printf '%q' "${local_tunnel_cidr}") dev $(printf '%q' "${tunnel_name}") + sudo -n ip link set $(printf '%q' "${tunnel_name}") up + " + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-tunnels-tsv --host "${name}") + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + if [ -n "${route_dev}" ]; then + echo " route ${remote_cidr} via ${peer_ip} dev ${route_dev} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") dev $(printf '%q' "${route_dev}")" + else + echo " route ${remote_cidr} via ${peer_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}")" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-routes-tsv --host "${name}") +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand cp + requireCommand mktemp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + TMP_DIR="$(mktemp -d "${SETUP_DIR}/tmp.multi-host-kvm.XXXXXX")" + trap cleanupTmp EXIT + + validateConfig + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + prepareHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + + echo "KVM hypervisors are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh deleted file mode 100755 index 436dcbfb7..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -# Prepare physical KVM hypervisors for a multi-host KVM SeedEMU cluster. -# -# Inputs: -# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. -# -# Generated/modified state: -# - Creates one routed libvirt network per hypervisor. -# - Enables IPv4 forwarding on each hypervisor. -# - Optionally creates point-to-point VXLAN route tunnels. -# - Installs static routes between the hypervisor VM subnets. -# -# Execution context: -# Run from the generated setup directory before createMultiHostKvmVms.py. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" -TMP_DIR="" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null -} - -cleanupTmp() { - [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" || true -} - -validateConfig() { - python3 "${HELPER}" --config "${CONFIG_PATH}" validate -} - -prepareHost() { - # Args are read from one hosts-tsv row. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local network_name="$6" - local bridge_name="$7" - local cidr="$8" - local remote_work_dir="${10}" - local network_xml="${TMP_DIR}/${name}.xml" - local remote_xml="${remote_work_dir}/network.xml" - - echo "[hypervisor-prepare] ${name} (${ip}) network=${network_name} bridge=${bridge_name}" - python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-network-xml \ - --host "${name}" \ - --output "${network_xml}" >/dev/null - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n true && command -v virsh >/dev/null && command -v virt-install >/dev/null && command -v qemu-img >/dev/null && command -v docker >/dev/null && command -v curl >/dev/null && command -v ssh >/dev/null" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "mkdir -p $(printf '%q' "${remote_work_dir}")" - copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" "${network_xml}" "${remote_xml}" - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - virsh net-info $(printf '%q' "${network_name}") >/dev/null 2>&1 || virsh net-define $(printf '%q' "${remote_xml}") >/dev/null - virsh net-start $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - virsh net-autostart $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - sudo -n sysctl -w net.ipv4.ip_forward=1 >/dev/null - if command -v iptables >/dev/null 2>&1; then - sudo -n iptables -C FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT - sudo -n iptables -C FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT - sudo -n iptables -t nat -C POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || sudo -n iptables -t nat -A POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE - fi - " - - while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do - [ -n "${tunnel_name}" ] || continue - echo " tunnel ${tunnel_name}: ${local_tunnel_cidr} -> ${peer_tunnel_ip} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - underlay_dev=\$(ip route get $(printf '%q' "${peer_ip}") | awk '{for (i=1;i<=NF;i++) if (\$i==\"dev\") {print \$(i+1); exit}}') - [ -n \"\${underlay_dev}\" ] || { echo 'Cannot determine underlay dev for peer $(printf '%q' "${peer_ip}")' >&2; exit 1; } - sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true - sudo -n ip link add $(printf '%q' "${tunnel_name}") type vxlan id $(printf '%q' "${vni}") local $(printf '%q' "${local_ip}") remote $(printf '%q' "${peer_ip}") dstport $(printf '%q' "${dst_port}") dev \"\${underlay_dev}\" - sudo -n ip addr add $(printf '%q' "${local_tunnel_cidr}") dev $(printf '%q' "${tunnel_name}") - sudo -n ip link set $(printf '%q' "${tunnel_name}") up - " - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-tunnels-tsv --host "${name}") - - while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do - [ -n "${remote_cidr}" ] || continue - if [ -n "${route_dev}" ]; then - echo " route ${remote_cidr} via ${peer_ip} dev ${route_dev} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") dev $(printf '%q' "${route_dev}")" - else - echo " route ${remote_cidr} via ${peer_ip} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}")" - fi - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-routes-tsv --host "${name}") -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - requireCommand scp - requireCommand cp - requireCommand mktemp - - [ -s "${CONFIG_PATH}" ] || { - echo "Missing config: ${CONFIG_PATH}" >&2 - exit 1 - } - - TMP_DIR="$(mktemp -d "${SETUP_DIR}/tmp.multi-host-kvm.XXXXXX")" - trap cleanupTmp EXIT - - validateConfig - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - prepareHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) - - echo "KVM hypervisors are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py index e05f462d5..ddbc243b2 100755 --- a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py @@ -1,21 +1,137 @@ #!/usr/bin/env python3 -"""Python entrypoint for cleanKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for cleanKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove Kube-OVN non-primary CNI artifacts from a physical K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# Uninstalls the Kube-OVN Helm release when a live cluster is available and +# removes Kube-OVN/OVS/OVN runtime files from configured nodes. It does not +# reboot nodes; users can reboot manually if they need a fully cold network +# state as recommended by Kube-OVN upstream docs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to run as root. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[ovn-clean] ${name} (${ip})" + if [ "${connection}" = "local" ]; then + sudo -n bash -s <<<"${script}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"${script}" +} + +ensureHelm() { + # Print an existing helm binary or the private binary downloaded by + # installKubeOvnFabric.py. + if command -v helm >/dev/null 2>&1; then + command -v helm + return + fi + local helm_bin="${ovnHelmCacheDir}/bin/helm" + if [ -x "${helm_bin}" ]; then + echo "${helm_bin}" + return + fi + echo "" +} + +uninstallHelmRelease() { + # Remove Kubernetes resources while the API server is still available. + [ -s "${outputKubeconfig}" ] || return 0 + local helm_bin + helm_bin="$(ensureHelm)" + if [ -n "${helm_bin}" ]; then + "${helm_bin}" uninstall "${ovnReleaseName}" -n "${ovnNamespace}" --kubeconfig "${outputKubeconfig}" >/dev/null 2>&1 || true + else + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete deploy,ds,sts,svc,cm,sa,role,rolebinding \ + -l app.kubernetes.io/instance="${ovnReleaseName}" --ignore-not-found >/dev/null 2>&1 || true + fi +} + +clean_node_script=' +set -euo pipefail +rm -rf /var/run/openvswitch +rm -rf /var/run/ovn +rm -rf /etc/origin/openvswitch +rm -rf /etc/origin/ovn +rm -rf /etc/cni/net.d/00-kube-ovn.conflist +rm -rf /etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/00-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/log/openvswitch +rm -rf /var/log/ovn +rm -rf /var/log/kube-ovn +ip link del ovn0 2>/dev/null || true +' + +uninstallHelmRelease +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + runNodeRootScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "${clean_node_script}" +done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) + +echo "Kube-OVN fabric cleaned." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh deleted file mode 100755 index a87412506..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# Remove Kube-OVN non-primary CNI artifacts from a physical K3s cluster. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml and the kubeconfig generated by buildK3sCluster.sh. -# -# Side effects: -# Uninstalls the Kube-OVN Helm release when a live cluster is available and -# removes Kube-OVN/OVS/OVN runtime files from configured nodes. It does not -# reboot nodes; users can reboot manually if they need a fully cold network -# state as recommended by Kube-OVN upstream docs. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; nothing to clean." - exit 0 -fi - -sshOptions() { - # Args: - # $1: SSH private key used for the target node. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - # Args: - # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, - # $5: SSH key, $6: shell script to run as root. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - - echo "[ovn-clean] ${name} (${ip})" - if [ "${connection}" = "local" ]; then - sudo -n bash -s <<<"${script}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"${script}" -} - -ensureHelm() { - # Print an existing helm binary or the private binary downloaded by - # installKubeOvnFabric.py. - if command -v helm >/dev/null 2>&1; then - command -v helm - return - fi - local helm_bin="${ovnHelmCacheDir}/bin/helm" - if [ -x "${helm_bin}" ]; then - echo "${helm_bin}" - return - fi - echo "" -} - -uninstallHelmRelease() { - # Remove Kubernetes resources while the API server is still available. - [ -s "${outputKubeconfig}" ] || return 0 - local helm_bin - helm_bin="$(ensureHelm)" - if [ -n "${helm_bin}" ]; then - "${helm_bin}" uninstall "${ovnReleaseName}" -n "${ovnNamespace}" --kubeconfig "${outputKubeconfig}" >/dev/null 2>&1 || true - else - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete deploy,ds,sts,svc,cm,sa,role,rolebinding \ - -l app.kubernetes.io/instance="${ovnReleaseName}" --ignore-not-found >/dev/null 2>&1 || true - fi -} - -clean_node_script=' -set -euo pipefail -rm -rf /var/run/openvswitch -rm -rf /var/run/ovn -rm -rf /etc/origin/openvswitch -rm -rf /etc/origin/ovn -rm -rf /etc/cni/net.d/00-kube-ovn.conflist -rm -rf /etc/cni/net.d/01-kube-ovn.conflist -rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/00-kube-ovn.conflist -rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/01-kube-ovn.conflist -rm -rf /var/log/openvswitch -rm -rf /var/log/ovn -rm -rf /var/log/kube-ovn -ip link del ovn0 2>/dev/null || true -' - -uninstallHelmRelease -while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - runNodeRootScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "${clean_node_script}" -done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) - -echo "Kube-OVN fabric cleaned." diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py index 4d09b25d0..160806a04 100755 --- a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py @@ -1,21 +1,359 @@ #!/usr/bin/env python3 -"""Python entrypoint for installKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for installKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Install Kube-OVN as a non-primary CNI for SeedEMU secondary networks. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml with fabric.type=ovn and optional ovn.* settings. +# +# Outputs/side effects: +# Installs the Kube-OVN Helm release into the configured K3s cluster. K3s +# flannel remains the primary CNI for eth0; Kube-OVN only serves Multus +# secondary interfaces used by SeedEMU simulated networks. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN install." + exit 0 +fi + +requireCommand() { + # Args: + # $1: Command name required by this script. + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + # Args: + # $1: duration accepted by timeout, remaining args: command to run. + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to execute on the node. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + if [ "${connection}" = "local" ]; then + bash -s <<<"${script}" + return + fi + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "bash -s" <<<"${script}" +} + +preloadKubeOvnImage() { + # Pull Kube-OVN once on the setup host, then import the image tar into each + # K3s node's containerd. This avoids repeated node-side Docker Hub pulls. + local image="docker.io/kubeovn/kube-ovn:${ovnChartVersion}" + local mirror_image="docker.m.daocloud.io/kubeovn/kube-ovn:${ovnChartVersion}" + local tar_path="${ovnHelmCacheDir}/kube-ovn_${ovnChartVersion}.tar" + mkdir -p "$(dirname "${tar_path}")" + echo "[ovn] preloading ${image} into K3s containerd nodes" + if ! runWithTimeout 180s sudo -n docker pull "${image}"; then + sudo -n docker pull "${mirror_image}" + sudo -n docker tag "${mirror_image}" "${image}" + fi + sudo -n docker save "${image}" -o "${tar_path}" + sudo -n chmod 0644 "${tar_path}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " import ${image} to ${name}" + if [ "${nodeConnection}" = "local" ]; then + sudo -n k3s ctr images import "${tar_path}" + else + local remote_tar="/tmp/$(basename "${tar_path}")" + # shellcheck disable=SC2046 + scp $(sshOptions "${nodeSshKey}") "${tar_path}" "${nodeSshUser}@${ip}:${remote_tar}" + # shellcheck disable=SC2046 + ssh -n $(sshOptions "${nodeSshKey}") "${nodeSshUser}@${ip}" \ + "sudo -n k3s ctr images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +prepareKubeOvnCniBinDir() { + # Prepare each node's K3s CNI binary directory for Kube-OVN install-cni. + # Args: none. Reads nodes and ovnCniBinDir from configK3s.yaml. + # + # K3s often keeps CNI plugins as symlinks under /var/lib/rancher/k3s/data/cni. + # Kube-OVN's install-cni init container copies these exact binaries into + # that same mounted path and fails if a target is a dangling symlink inside + # the container mount. Remove only the known overwrite targets. + echo "[ovn] preparing K3s CNI binary directory on all nodes: ${ovnCniBinDir}" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " prepare ${name}" + runNodeScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "$(cat </dev/null 2>&1 + } + + if command -v helm >/dev/null 2>&1 && helmUsable "$(command -v helm)"; then + command -v helm + return + fi + + local helm_dir="${ovnHelmCacheDir}/bin" + local helm_bin="${helm_dir}/helm" + if helmUsable "${helm_bin}"; then + echo "${helm_bin}" + return + fi + + mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" + rm -f "${helm_bin}" + local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" + local archive_tmp="${archive}.tmp" + echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 + rm -f "${archive_tmp}" + curl --fail --location --show-error --retry 5 --retry-delay 2 --retry-all-errors \ + https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive_tmp}" + tar -tzf "${archive_tmp}" >/dev/null + mv "${archive_tmp}" "${archive}" + rm -rf "${ovnHelmCacheDir}/download/linux-amd64" + tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" + cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" + chmod +x "${helm_bin}" + if ! helmUsable "${helm_bin}"; then + rm -f "${helm_bin}" + echo "Downloaded helm binary is not usable: ${helm_bin}" >&2 + exit 1 + fi + echo "${helm_bin}" +} + +installKubeOvn() { + # Install Kube-OVN in non-primary mode. Kube-OVN's CNI binary is placed into + # the K3s CNI bin dir so Multus can invoke type=kube-ovn delegates. + local helm_bin="$1" + + "${helm_bin}" repo add "${ovnHelmRepoName}" "${ovnHelmRepoUrl}" >/dev/null 2>&1 || true + "${helm_bin}" repo update >/dev/null + + kubectl --kubeconfig "${outputKubeconfig}" label node "${k3sMasterName}" kube-ovn/role=master --overwrite + + echo "[ovn] installing Kube-OVN ${ovnChartVersion} in non-primary CNI mode" + "${helm_bin}" upgrade --install "${ovnReleaseName}" "${ovnHelmRepoName}/kube-ovn" \ + --kubeconfig "${outputKubeconfig}" \ + --namespace "${ovnNamespace}" \ + --version "${ovnChartVersion}" \ + --set cni_conf.NON_PRIMARY_CNI=true \ + --set cni_conf.CNI_CONF_DIR="${ovnCniConfDir}" \ + --set cni_conf.MOUNT_CNI_CONF_DIR="${ovnMountCniConfDir}" \ + --set cni_conf.CNI_BIN_DIR="${ovnCniBinDir}" \ + --set networking.TUNNEL_TYPE="${ovnTunnelType}" \ + --set networking.IFACE="${ovnIface}" \ + --set ipv4.POD_CIDR="${ovnPodCidr}" \ + --set ipv4.POD_GATEWAY="${ovnPodGateway}" \ + --set ipv4.SVC_CIDR="${ovnServiceCidr}" \ + --set ipv4.JOIN_CIDR="${ovnJoinCidr}" \ + --set MASTER_NODES="${ovnMasterNodes}" \ + --wait \ + --timeout 10m +} + +waitKubeOvnReady() { + # Confirm the controller, OVS/OVN daemonset, and CNI daemonset are rolled + # out before the running stage creates Kube-OVN NAD/Subnet resources. + local resources=( + "deployment/kube-ovn-controller" + "deployment/ovn-central" + "daemonset/ovs-ovn" + "daemonset/kube-ovn-cni" + ) + for resource in "${resources[@]}"; do + if kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get "${resource}" >/dev/null 2>&1; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status "${resource}" --timeout=600s + fi + done + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=300s +} + +repairK3sCniBinDirAfterKubeOvnInstall() { + # Kube-OVN's install-cni init container runs inside a container mount of + # the K3s CNI bin dir. On K3s this can leave the host dir without the + # primary loopback/portmap links or without the kube-ovn delegate binary. + # Use a short-lived hostNetwork pod per node so this repair does not depend + # on the CNI plugin path that it is fixing. + echo "[ovn] verifying K3s CNI binaries after Kube-OVN install" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + local pod_name + pod_name="seedemu-ovn-cni-repair-$(printf '%s' "${name}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')" + pod_name="${pod_name%-}" + echo " repair ${name} with pod/${pod_name}" + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" \ + --ignore-not-found=true --wait=false >/dev/null 2>&1 || true + cat </dev/null || true)" + if [ "${phase}" = "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=80 || true + break + fi + if [ "${phase}" = "Failed" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=120 || true + return 1 + fi + sleep 2 + done + local final_phase + final_phase="$(kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + if [ "${final_phase}" != "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" describe pod "${pod_name}" || true + return 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" --wait=false >/dev/null 2>&1 || true + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +main() { + requireCommand curl + requireCommand kubectl + requireCommand python3 + requireCommand scp + requireCommand ssh + requireCommand docker + requireCommand tar + + if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + echo "Run applyK3sCluster.py or k8sTools.py build before installing Kube-OVN." >&2 + exit 1 + fi + + local helm_bin + helm_bin="$(ensureHelm)" + preloadKubeOvnImage + prepareKubeOvnCniBinDir + installKubeOvn "${helm_bin}" + waitKubeOvnReady + repairK3sCniBinDirAfterKubeOvnInstall + echo "Kube-OVN non-primary CNI is ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh deleted file mode 100755 index 597dfd34f..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash -# Install Kube-OVN as a non-primary CNI for SeedEMU secondary networks. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml with fabric.type=ovn and optional ovn.* settings. -# -# Outputs/side effects: -# Installs the Kube-OVN Helm release into the configured K3s cluster. K3s -# flannel remains the primary CNI for eth0; Kube-OVN only serves Multus -# secondary interfaces used by SeedEMU simulated networks. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; skipping OVN install." - exit 0 -fi - -requireCommand() { - # Args: - # $1: Command name required by this script. - command -v "$1" >/dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - # Args: - # $1: duration accepted by timeout, remaining args: command to run. - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -sshOptions() { - # Args: - # $1: SSH private key used for the target node. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeScript() { - # Args: - # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, - # $5: SSH key, $6: shell script to execute on the node. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - if [ "${connection}" = "local" ]; then - bash -s <<<"${script}" - return - fi - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "bash -s" <<<"${script}" -} - -preloadKubeOvnImage() { - # Pull Kube-OVN once on the setup host, then import the image tar into each - # K3s node's containerd. This avoids repeated node-side Docker Hub pulls. - local image="docker.io/kubeovn/kube-ovn:${ovnChartVersion}" - local mirror_image="docker.m.daocloud.io/kubeovn/kube-ovn:${ovnChartVersion}" - local tar_path="${ovnHelmCacheDir}/kube-ovn_${ovnChartVersion}.tar" - mkdir -p "$(dirname "${tar_path}")" - echo "[ovn] preloading ${image} into K3s containerd nodes" - if ! runWithTimeout 180s sudo -n docker pull "${image}"; then - sudo -n docker pull "${mirror_image}" - sudo -n docker tag "${mirror_image}" "${image}" - fi - sudo -n docker save "${image}" -o "${tar_path}" - sudo -n chmod 0644 "${tar_path}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - echo " import ${image} to ${name}" - if [ "${nodeConnection}" = "local" ]; then - sudo -n k3s ctr images import "${tar_path}" - else - local remote_tar="/tmp/$(basename "${tar_path}")" - # shellcheck disable=SC2046 - scp $(sshOptions "${nodeSshKey}") "${tar_path}" "${nodeSshUser}@${ip}:${remote_tar}" - # shellcheck disable=SC2046 - ssh -n $(sshOptions "${nodeSshKey}") "${nodeSshUser}@${ip}" \ - "sudo -n k3s ctr images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" - fi - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) -} - -prepareKubeOvnCniBinDir() { - # Prepare each node's K3s CNI binary directory for Kube-OVN install-cni. - # Args: none. Reads nodes and ovnCniBinDir from configK3s.yaml. - # - # K3s often keeps CNI plugins as symlinks under /var/lib/rancher/k3s/data/cni. - # Kube-OVN's install-cni init container copies these exact binaries into - # that same mounted path and fails if a target is a dangling symlink inside - # the container mount. Remove only the known overwrite targets. - echo "[ovn] preparing K3s CNI binary directory on all nodes: ${ovnCniBinDir}" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - echo " prepare ${name}" - runNodeScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "$(cat </dev/null 2>&1; then - command -v helm - return - fi - - local helm_dir="${ovnHelmCacheDir}/bin" - local helm_bin="${helm_dir}/helm" - if [ -x "${helm_bin}" ]; then - echo "${helm_bin}" - return - fi - - mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" - local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" - echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 - curl -fsSL https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive}" - tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" - cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" - chmod +x "${helm_bin}" - echo "${helm_bin}" -} - -installKubeOvn() { - # Install Kube-OVN in non-primary mode. Kube-OVN's CNI binary is placed into - # the K3s CNI bin dir so Multus can invoke type=kube-ovn delegates. - local helm_bin="$1" - - "${helm_bin}" repo add "${ovnHelmRepoName}" "${ovnHelmRepoUrl}" >/dev/null 2>&1 || true - "${helm_bin}" repo update >/dev/null - - kubectl --kubeconfig "${outputKubeconfig}" label node "${k3sMasterName}" kube-ovn/role=master --overwrite - - echo "[ovn] installing Kube-OVN ${ovnChartVersion} in non-primary CNI mode" - "${helm_bin}" upgrade --install "${ovnReleaseName}" "${ovnHelmRepoName}/kube-ovn" \ - --kubeconfig "${outputKubeconfig}" \ - --namespace "${ovnNamespace}" \ - --version "${ovnChartVersion}" \ - --set cni_conf.NON_PRIMARY_CNI=true \ - --set cni_conf.CNI_CONF_DIR="${ovnCniConfDir}" \ - --set cni_conf.MOUNT_CNI_CONF_DIR="${ovnMountCniConfDir}" \ - --set cni_conf.CNI_BIN_DIR="${ovnCniBinDir}" \ - --set networking.TUNNEL_TYPE="${ovnTunnelType}" \ - --set networking.IFACE="${ovnIface}" \ - --set ipv4.POD_CIDR="${ovnPodCidr}" \ - --set ipv4.POD_GATEWAY="${ovnPodGateway}" \ - --set ipv4.SVC_CIDR="${ovnServiceCidr}" \ - --set ipv4.JOIN_CIDR="${ovnJoinCidr}" \ - --set MASTER_NODES="${ovnMasterNodes}" \ - --wait \ - --timeout 10m -} - -waitKubeOvnReady() { - # Confirm the controller, OVS/OVN daemonset, and CNI daemonset are rolled - # out before the running stage creates Kube-OVN NAD/Subnet resources. - local resources=( - "deployment/kube-ovn-controller" - "deployment/ovn-central" - "daemonset/ovs-ovn" - "daemonset/kube-ovn-cni" - ) - for resource in "${resources[@]}"; do - if kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get "${resource}" >/dev/null 2>&1; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status "${resource}" --timeout=600s - fi - done - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=300s -} - -repairK3sCniBinDirAfterKubeOvnInstall() { - # Kube-OVN's install-cni init container runs inside a container mount of - # the K3s CNI bin dir. On K3s this can leave the host dir without the - # primary loopback/portmap links or without the kube-ovn delegate binary. - # Use a short-lived hostNetwork pod per node so this repair does not depend - # on the CNI plugin path that it is fixing. - echo "[ovn] verifying K3s CNI binaries after Kube-OVN install" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - local pod_name - pod_name="seedemu-ovn-cni-repair-$(printf '%s' "${name}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')" - pod_name="${pod_name%-}" - echo " repair ${name} with pod/${pod_name}" - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" \ - --ignore-not-found=true --wait=false >/dev/null 2>&1 || true - cat </dev/null || true)" - if [ "${phase}" = "Succeeded" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=80 || true - break - fi - if [ "${phase}" = "Failed" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=120 || true - return 1 - fi - sleep 2 - done - local final_phase - final_phase="$(kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "${final_phase}" != "Succeeded" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" describe pod "${pod_name}" || true - return 1 - fi - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" --wait=false >/dev/null 2>&1 || true - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) -} - -main() { - requireCommand curl - requireCommand kubectl - requireCommand python3 - requireCommand scp - requireCommand ssh - requireCommand docker - requireCommand tar - - if [ ! -s "${outputKubeconfig}" ]; then - echo "Kubeconfig not found: ${outputKubeconfig}" >&2 - echo "Run buildK3sCluster.sh before installing Kube-OVN." >&2 - exit 1 - fi - - local helm_bin - helm_bin="$(ensureHelm)" - preloadKubeOvnImage - prepareKubeOvnCniBinDir - installKubeOvn "${helm_bin}" - waitKubeOvnReady - repairK3sCniBinDirAfterKubeOvnInstall - echo "Kube-OVN non-primary CNI is ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py index 0e7dd68c7..5b6ea0346 100755 --- a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py @@ -1,21 +1,62 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that Kube-OVN non-primary CNI is ready for SeedEMU NADs. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# None. This script only reads cluster state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN validation." + exit 0 +fi + +if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + exit 1 +fi + +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pods -o wide +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status deployment/kube-ovn-controller --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/kube-ovn-cni --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/ovs-ovn --timeout=300s +echo "Kube-OVN fabric validation passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh deleted file mode 100755 index f83175151..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Validate that Kube-OVN non-primary CNI is ready for SeedEMU NADs. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml and the kubeconfig generated by buildK3sCluster.sh. -# -# Side effects: -# None. This script only reads cluster state. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; skipping OVN validation." - exit 0 -fi - -if [ ! -s "${outputKubeconfig}" ]; then - echo "Kubeconfig not found: ${outputKubeconfig}" >&2 - exit 1 -fi - -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pods -o wide -kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s -kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status deployment/kube-ovn-controller --timeout=300s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/kube-ovn-cni --timeout=300s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/ovs-ovn --timeout=300s -echo "Kube-OVN fabric validation passed." diff --git a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py index a1c744731..99981ac62 100755 --- a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py +++ b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py @@ -1,21 +1,86 @@ #!/usr/bin/env python3 -"""Python entrypoint for preparePhysicalNodes.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for preparePhysicalNodes with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate physical nodes before building a SeedEMU K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Checks: +# - Each node can be reached with the configured connection method. +# - sudo -n works for the configured user. +# - iproute2 and curl are available. +# - linux-vxlan fabric nodes expose their configured underlay interface. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + if [ "$connection" = "local" ]; then + bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "$command" /dev/null && command -v curl >/dev/null" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" +if [ "${fabricType}" = "linux-vxlan" ]; then + echo "Validating linux-vxlan underlay interfaces..." + while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + echo " ${name}: underlay=${underlay}" + runNodeCommand "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "ip link show $(printf '%q' "$underlay") >/dev/null" + done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +fi + +echo "Physical node preflight passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh deleted file mode 100755 index 0697de0cf..000000000 --- a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Validate physical nodes before building a SeedEMU K3s cluster. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Checks: -# - Each node can be reached with the configured connection method. -# - sudo -n works for the configured user. -# - iproute2 and curl are available. -# - linux-vxlan fabric nodes expose their configured underlay interface. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${1:-./configK3s.yaml}" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeCommand() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - - if [ "$connection" = "local" ]; then - bash -lc "$command" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "$command" /dev/null && command -v curl >/dev/null" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" -if [ "${fabricType}" = "linux-vxlan" ]; then - echo "Validating linux-vxlan underlay interfaces..." - while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - echo " ${name}: underlay=${underlay}" - runNodeCommand "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "ip link show $(printf '%q' "$underlay") >/dev/null" - done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -fi - -echo "Physical node preflight passed." diff --git a/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py index 053381455..9288effe6 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py @@ -1,21 +1,99 @@ #!/usr/bin/env python3 -"""Python entrypoint for cleanLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for cleanLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove the Linux VXLAN bridge fabric described by configK3s.yaml. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Deletes the configured test macvlan interface, VXLAN interface, and bridge +# on every configured node. It does not uninstall K3s or delete workload pods. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +cleanup_script=' +set -euo pipefail +macvlan_name="$1" +vxlan_name="$2" +bridge_name="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link del "$vxlan_name" 2>/dev/null || true +ip link del "$bridge_name" 2>/dev/null || true +' + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$cleanup_script" \ + "$fabricMacvlanTestName" "$fabricVxlanName" "$fabricBridgeName" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Linux VXLAN fabric cleaned." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh deleted file mode 100755 index c192707bb..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Remove the Linux VXLAN bridge fabric described by configK3s.yaml. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Side effects: -# Deletes the configured test macvlan interface, VXLAN interface, and bridge -# on every configured node. It does not uninstall K3s or delete workload pods. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; nothing to clean." - exit 0 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-clean] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -cleanup_script=' -set -euo pipefail -macvlan_name="$1" -vxlan_name="$2" -bridge_name="$3" - -ip link del "$macvlan_name" 2>/dev/null || true -ip link del "$vxlan_name" 2>/dev/null || true -ip link del "$bridge_name" 2>/dev/null || true -' - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$cleanup_script" \ - "$fabricMacvlanTestName" "$fabricVxlanName" "$fabricBridgeName" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -echo "Linux VXLAN fabric cleaned." diff --git a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py index dd350f622..040f6d024 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py @@ -1,21 +1,144 @@ #!/usr/bin/env python3 -"""Python entrypoint for configureLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for configureLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create a two-node Linux VXLAN bridge fabric for SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Expected config: +# fabric.type: linux-vxlan +# fabric.nodes..underlayInterface: management NIC used as VXLAN underlay. +# +# Side effects: +# Creates a bridge such as br-seedemu and a VXLAN device such as vxseed0 on +# each node. On failure, it calls cleanLinuxVxlanFabric.py to roll back all +# configured fabric interfaces. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric setup." + exit 0 +fi + +rollbackOnError() { + local rc=$? + if [ "$rc" -ne 0 ]; then + echo "[fabric] setup failed; rolling back configured fabric interfaces" >&2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap rollbackOnError EXIT + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-configure] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +configure_script=' +set -euo pipefail +bridge_name="$1" +vxlan_name="$2" +vni="$3" +local_ip="$4" +peer_ip="$5" +underlay="$6" +dst_port="$7" +mtu="$8" + +if [ "${#bridge_name}" -gt 15 ] || [ "${#vxlan_name}" -gt 15 ]; then + echo "Linux interface names must be at most 15 characters" >&2 + exit 1 +fi + +ip link show "$underlay" >/dev/null + +if ip link show "$vxlan_name" >/dev/null 2>&1; then + ip link del "$vxlan_name" +fi + +if ! ip link show "$bridge_name" >/dev/null 2>&1; then + ip link add "$bridge_name" type bridge +fi + +ip link set "$bridge_name" up +ip link add "$vxlan_name" type vxlan id "$vni" local "$local_ip" remote "$peer_ip" dev "$underlay" dstport "$dst_port" +ip link set "$vxlan_name" mtu "$mtu" || true +ip link set "$vxlan_name" master "$bridge_name" +ip link set "$vxlan_name" up +' + +echo "Configuring Linux VXLAN fabric:" +echo " bridge=${fabricBridgeName}" +echo " vxlan=${fabricVxlanName}" +echo " vni=${fabricVni}" +echo " dstPort=${fabricDstPort}" + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$configure_script" \ + "$fabricBridgeName" "$fabricVxlanName" "$fabricVni" "$ip" "$peer_ip" \ + "$underlay" "$fabricDstPort" "$fabricMtu" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +trap - EXIT +echo "Linux VXLAN fabric configured." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh deleted file mode 100755 index add805819..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -# Create a two-node Linux VXLAN bridge fabric for SeedEMU Multus/macvlan. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Expected config: -# fabric.type: linux-vxlan -# fabric.nodes..underlayInterface: management NIC used as VXLAN underlay. -# -# Side effects: -# Creates a bridge such as br-seedemu and a VXLAN device such as vxseed0 on -# each node. On failure, it calls cleanLinuxVxlanFabric.py to roll back all -# configured fabric interfaces. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; skipping fabric setup." - exit 0 -fi - -rollbackOnError() { - local rc=$? - if [ "$rc" -ne 0 ]; then - echo "[fabric] setup failed; rolling back configured fabric interfaces" >&2 - python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - fi - exit "$rc" -} -trap rollbackOnError EXIT - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-configure] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -configure_script=' -set -euo pipefail -bridge_name="$1" -vxlan_name="$2" -vni="$3" -local_ip="$4" -peer_ip="$5" -underlay="$6" -dst_port="$7" -mtu="$8" - -if [ "${#bridge_name}" -gt 15 ] || [ "${#vxlan_name}" -gt 15 ]; then - echo "Linux interface names must be at most 15 characters" >&2 - exit 1 -fi - -ip link show "$underlay" >/dev/null - -if ip link show "$vxlan_name" >/dev/null 2>&1; then - ip link del "$vxlan_name" -fi - -if ! ip link show "$bridge_name" >/dev/null 2>&1; then - ip link add "$bridge_name" type bridge -fi - -ip link set "$bridge_name" up -ip link add "$vxlan_name" type vxlan id "$vni" local "$local_ip" remote "$peer_ip" dev "$underlay" dstport "$dst_port" -ip link set "$vxlan_name" mtu "$mtu" || true -ip link set "$vxlan_name" master "$bridge_name" -ip link set "$vxlan_name" up -' - -echo "Configuring Linux VXLAN fabric:" -echo " bridge=${fabricBridgeName}" -echo " vxlan=${fabricVxlanName}" -echo " vni=${fabricVni}" -echo " dstPort=${fabricDstPort}" - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$configure_script" \ - "$fabricBridgeName" "$fabricVxlanName" "$fabricVni" "$ip" "$peer_ip" \ - "$underlay" "$fabricDstPort" "$fabricMtu" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -trap - EXIT -echo "Linux VXLAN fabric configured." diff --git a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py index 037d32d26..889d212ce 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py @@ -1,21 +1,194 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate the Linux VXLAN bridge fabric used by SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Validation: +# 1. Adds temporary /30 IPs to the configured bridge and pings both ways. +# 2. Creates temporary macvlan interfaces on the bridge and pings both ways. +# +# Side effects: +# Temporary test IPs and macvlan interfaces are always removed. If validation +# fails, the configured VXLAN fabric is also removed to avoid stale state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric validation." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +runNodeRootCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -lc $(printf '%q' "$command")" &2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap cleanupOnExit EXIT + +setup_bridge_ip=' +set -euo pipefail +bridge_name="$1" +bridge_test_ip="$2" + +ip link show "$bridge_name" >/dev/null +ip addr del "$bridge_test_ip" dev "$bridge_name" 2>/dev/null || true +ip addr add "$bridge_test_ip" dev "$bridge_name" +ip link set "$bridge_name" up +' + +setup_macvlan=' +set -euo pipefail +bridge_name="$1" +macvlan_name="$2" +macvlan_test_ip="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link add "$macvlan_name" link "$bridge_name" type macvlan mode bridge +ip addr add "$macvlan_test_ip" dev "$macvlan_name" +ip link set "$macvlan_name" up +' + +echo "Validating Linux VXLAN bridge reachability..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_bridge_ip" \ + "$fabricBridgeName" "$bridge_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 1 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_bridge_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $8; exit}')" + peer_bridge_ip="${peer_bridge_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -c 3 -W 2 $(printf '%q' "$peer_bridge_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Removing bridge test IPs before macvlan validation..." +cleanupTestArtifacts +sleep 1 + +echo "Validating macvlan reachability over Linux VXLAN bridge..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_macvlan" \ + "$fabricBridgeName" "$fabricMacvlanTestName" "$macvlan_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 2 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_macvlan_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $9; exit}')" + peer_macvlan_ip="${peer_macvlan_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -I $(printf '%q' "$fabricMacvlanTestName") -c 3 -W 2 $(printf '%q' "$peer_macvlan_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +cleanupTestArtifacts +trap - EXIT +echo "Linux VXLAN fabric validation passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh deleted file mode 100755 index 5f7321362..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash -# Validate the Linux VXLAN bridge fabric used by SeedEMU Multus/macvlan. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Validation: -# 1. Adds temporary /30 IPs to the configured bridge and pings both ways. -# 2. Creates temporary macvlan interfaces on the bridge and pings both ways. -# -# Side effects: -# Temporary test IPs and macvlan interfaces are always removed. If validation -# fails, the configured VXLAN fabric is also removed to avoid stale state. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; skipping fabric validation." - exit 0 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-validate] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -runNodeRootCommand() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - - echo "[fabric-validate] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -lc "$command" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -lc $(printf '%q' "$command")" &2 - python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - fi - exit "$rc" -} -trap cleanupOnExit EXIT - -setup_bridge_ip=' -set -euo pipefail -bridge_name="$1" -bridge_test_ip="$2" - -ip link show "$bridge_name" >/dev/null -ip addr del "$bridge_test_ip" dev "$bridge_name" 2>/dev/null || true -ip addr add "$bridge_test_ip" dev "$bridge_name" -ip link set "$bridge_name" up -' - -setup_macvlan=' -set -euo pipefail -bridge_name="$1" -macvlan_name="$2" -macvlan_test_ip="$3" - -ip link del "$macvlan_name" 2>/dev/null || true -ip link add "$macvlan_name" link "$bridge_name" type macvlan mode bridge -ip addr add "$macvlan_test_ip" dev "$macvlan_name" -ip link set "$macvlan_name" up -' - -echo "Validating Linux VXLAN bridge reachability..." -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_bridge_ip" \ - "$fabricBridgeName" "$bridge_test_ip" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -sleep 1 - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - peer_bridge_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $8; exit}')" - peer_bridge_ip="${peer_bridge_test_ip%/*}" - runNodeRootCommand \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ - "ping -c 3 -W 2 $(printf '%q' "$peer_bridge_ip")" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -echo "Removing bridge test IPs before macvlan validation..." -cleanupTestArtifacts -sleep 1 - -echo "Validating macvlan reachability over Linux VXLAN bridge..." -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_macvlan" \ - "$fabricBridgeName" "$fabricMacvlanTestName" "$macvlan_test_ip" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -sleep 2 - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - peer_macvlan_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $9; exit}')" - peer_macvlan_ip="${peer_macvlan_test_ip%/*}" - runNodeRootCommand \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ - "ping -I $(printf '%q' "$fabricMacvlanTestName") -c 3 -W 2 $(printf '%q' "$peer_macvlan_ip")" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -cleanupTestArtifacts -trap - EXIT -echo "Linux VXLAN fabric validation passed." diff --git a/seedemu/k8sTools/runner.py b/seedemu/k8sTools/runner.py index e6d6facfa..7f5891b19 100644 --- a/seedemu/k8sTools/runner.py +++ b/seedemu/k8sTools/runner.py @@ -50,22 +50,32 @@ def temporaryWorkDir(prefix: str, keep_temp: bool = False) -> Iterator[Path]: def inferImageRegistryPrefix(output_dir: Path) -> str: - """Infer the compiler image prefix from images.yaml. + """Infer the compiler image prefix from generated image metadata. Args: - output_dir: Compile output directory containing images.yaml. + output_dir: Compile output directory containing images.yaml or images.txt. """ images_path = output_dir / "images.yaml" - if not images_path.exists(): + if images_path.exists(): + data = loadYaml(images_path) + images = data.get("images") if isinstance(data.get("images"), list) else [] + for item in images: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + if "/" in name: + return name.rsplit("/", 1)[0] return "seedemu" - data = loadYaml(images_path) - images = data.get("images") if isinstance(data.get("images"), list) else [] - for item in images: - if not isinstance(item, dict): - continue - name = str(item.get("name") or "").strip() - if "/" in name: - return name.rsplit("/", 1)[0] + + images_txt = output_dir / "images.txt" + if images_txt.exists(): + for line in images_txt.read_text(encoding="utf-8").splitlines(): + name = line.strip() + if not name or name.startswith("#"): + continue + if "/" in name: + return name.rsplit("/", 1)[0] + return "seedemu" return "seedemu" @@ -74,6 +84,7 @@ def buildCluster( config_k3s: str | Path, kubeconfig: str | Path, *, + inventory: str | Path | None = None, keep_temp: bool = False, ) -> None: """Build KVM/physical infrastructure and write final config outputs. @@ -82,6 +93,7 @@ def buildCluster( input_config: User YAML with explicit `kind`. config_k3s: Final configK3s.yaml path requested by the user. kubeconfig: Final kubeconfig path requested by the user. + inventory: Optional final cluster inventory YAML path requested by the user. keep_temp: Keep temporary setup resources for debugging. The first-phase implementation expands bundled setup resources into a @@ -92,6 +104,7 @@ def buildCluster( input_path = resolvePath(input_config) config_path = resolvePath(config_k3s) kubeconfig_path = resolvePath(kubeconfig) + inventory_path = resolvePath(inventory) if inventory is not None else None source = loadYaml(input_path) kind = detectKind(source, input_path) @@ -99,11 +112,11 @@ def buildCluster( setup_dir = copyTree("setup", root / "setup", overwrite=True) chmodScripts(setup_dir) if kind == "kvmOvn": - _buildKvmOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) elif kind == "multiHostKvmOvn": - _buildMultiHostKvmOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildMultiHostKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) elif kind == "physicalOvn": - _buildPhysicalOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildPhysicalOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) else: raise ValueError(f"unsupported kind: {kind}") @@ -253,14 +266,20 @@ def destroyCluster(config_k3s: str | Path, *, keep_temp: bool = False) -> None: raise ValueError(f"unsupported destroy type: {destroy_type}") -def _buildKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a single-hypervisor KVM cluster with Kube-OVN.""" kvm_config = makeKvmConfig( config=input_path, setup_dir=setup_dir, k3s_config_path=setup_dir / "configK3s.yaml", kubeconfig_path=kubeconfig_path, - inventory_path=setup_dir / "cluster.inventory.yaml", + inventory_path=inventory_path or setup_dir / "cluster.inventory.yaml", tmp_dir=setup_dir / "tmp", ) kvm_config.setdefault("fabric", {})["type"] = "ovn" @@ -283,7 +302,13 @@ def _buildKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfi writeYaml(config_path, final_config) -def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildMultiHostKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a multi-hypervisor KVM cluster with routed VM subnets and Kube-OVN.""" source = loadYaml(input_path) source.setdefault("fabric", {})["type"] = "ovn" @@ -291,7 +316,12 @@ def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, outputs = source.setdefault("outputs", {}) outputs["k3sConfig"] = str(setup_dir / "configK3s.yaml") outputs["multiHostKvmState"] = str(setup_dir / "multiHostKvmState.yaml") - setOutputPaths(source, kubeconfig=kubeconfig_path, tmp_dir=setup_dir / "tmp", inventory=setup_dir / "cluster.inventory.yaml") + setOutputPaths( + source, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) temp_input = setup_dir / "multiHostInput.yaml" writeYaml(temp_input, source) kvm_config = makeMultiHostKvmConfig(config=temp_input, setup_dir=setup_dir) @@ -313,13 +343,24 @@ def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, writeYaml(config_path, final_config) -def _buildPhysicalOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildPhysicalOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a K3s + Kube-OVN cluster on existing physical nodes.""" config = loadYaml(input_path) config["kind"] = "physicalOvn" config.setdefault("fabric", {})["type"] = "ovn" _applyOvnK3sDefaults(config) - setOutputPaths(config, kubeconfig=kubeconfig_path, tmp_dir=setup_dir / "tmp", inventory=setup_dir / "cluster.inventory.yaml") + setOutputPaths( + config, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) writeYaml(setup_dir / "configK3s.yaml", config) runCommand(["python3", str(setup_dir / "preparePhysicalNodes.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) runCommand(["python3", str(setup_dir / "applyK3sCluster.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) diff --git a/seedemu/k8sTools/utils.py b/seedemu/k8sTools/utils.py index 09cde405b..6b30f3d2b 100644 --- a/seedemu/k8sTools/utils.py +++ b/seedemu/k8sTools/utils.py @@ -69,13 +69,13 @@ def copyResourceItems( def chmodScripts(path: str | Path) -> None: - """Make every bundled script entrypoint below path executable. + """Make every bundled Python script entrypoint below path executable. Args: path: Root directory copied from package resources. """ root = Path(path).expanduser() - for script in [*root.rglob("*.py"), *root.rglob("*.sh")]: + for script in root.rglob("*.py"): mode = script.stat().st_mode script.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 000000000..35053a67d --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,42 @@ +# Feature-Oriented CI + +The pull-request workflow is intentionally organized around SEED features rather +than broad example buckets. The manifest in `feature_manifest.json` is the source +of truth for which features are covered by unit tests, representative example +compilation, selected Docker builds, and optional runtime integration probes. + +The CI runner writes both human-readable logs and machine-readable artifacts: + +- `ci-summary.json` records every command, return code, duration, and log path. +- `junit.xml` records the same stage in a format GitHub and review tooling can + ingest. +- `feature-coverage.json` records the manifest-derived coverage state, including + covered features and declared gaps that have not landed on this integration + line. + +The static stage compiles importable Python source plus representative examples. +It intentionally excludes embedded payload templates under +`seedemu/services/EthereumService/EthTemplates/`, where some historical `.py` +filenames contain shell script content copied into containers. + +Run stages locally from the repository root: + +```bash +python3 tests/ci/run_ci.py static --artifact-dir ci-artifacts/static +python3 tests/ci/run_ci.py unit --artifact-dir ci-artifacts/unit +python3 tests/ci/run_ci.py example-compile --artifact-dir ci-artifacts/example-compile +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +``` + +Docker image builds and runtime integration are available as explicit entry +points, but they are not default pull-request gates in this PR: + +```bash +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +python3 tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration +``` + +The runtime integration stage is kept as an explicit entry point for future +Docker runtime probes. The current manifest does not enable any runtime group on +this branch, so the stage is a reserved hook unless new groups are added to +`feature_manifest.json`. diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 000000000..2829e92c3 --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +"""Feature-oriented CI helpers for SEED Emulator.""" diff --git a/tests/ci/feature_manifest.json b/tests/ci/feature_manifest.json new file mode 100644 index 000000000..38667cb12 --- /dev/null +++ b/tests/ci/feature_manifest.json @@ -0,0 +1,110 @@ +{ + "schema": 1, + "features": { + "ipv4-default": { + "status": "covered", + "description": "Legacy IPv4 examples continue to compile and build without requiring any new control-plane capability.", + "unit_groups": [], + "compile_examples": [ + "basic-a00-simple-as" + ], + "build_examples": [ + "basic-a00-simple-as" + ], + "runtime_groups": [] + }, + "ipv6-core": { + "status": "declared-gap", + "description": "Repository-level IPv6 readiness is tracked here, but this K8s integration branch does not currently carry a representative IPv6 example or runtime probe.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until native IPv6 topology/API coverage or another representative IPv6 example is added." + }, + "routing-bird-frr": { + "status": "declared-gap", + "description": "BIRD/FRR interoperability coverage from the source CI framework is not currently carried by this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the branch includes a maintained BIRD/FRR example and matching runtime probe." + }, + "exabgp-service": { + "status": "declared-gap", + "description": "ExaBGP coverage exists on the source CI framework branch, but this K8s integration branch does not carry ExaBgpService or its examples.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until ExaBGP service support is present on this branch." + }, + "looking-glass": { + "status": "declared-gap", + "description": "Looking Glass service support is present, but the source CI framework examples are not present on this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the representative Looking Glass example is merged here." + }, + "dns-endpoint": { + "status": "covered", + "description": "DNS endpoint behavior is represented by the mini Internet plus DNS example.", + "unit_groups": [], + "compile_examples": [ + "internet-b02-mini-internet-with-dns" + ], + "build_examples": [ + "internet-b02-mini-internet-with-dns" + ], + "runtime_groups": [] + }, + "legacy-guard": { + "status": "declared-gap", + "description": "Removed legacy control-plane API guards exist on the source CI framework branch, but those tests are not present on this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until control-plane guard tests are merged here." + } + }, + "unit_groups": {}, + "runtime_groups": {}, + "examples": { + "basic-a00-simple-as": { + "description": "Small IPv4 default behavior smoke example.", + "script": "examples/basic/A00_simple_as/simple_as.py", + "args": [ + "amd" + ], + "clean": [ + "output" + ], + "expected": [ + "output/docker-compose.yml" + ], + "compile": true, + "build": true + }, + "internet-b02-mini-internet-with-dns": { + "description": "Mini Internet with DNS component and local DNS endpoints.", + "script": "examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py", + "args": [ + "amd" + ], + "clean": [ + "output", + "base_internet.bin", + "dns_component.bin" + ], + "expected": [ + "output/docker-compose.yml" + ], + "compile": true, + "build": true + } + } +} diff --git a/tests/ci/run_ci.py b/tests/ci/run_ci.py new file mode 100644 index 000000000..0594b0b49 --- /dev/null +++ b/tests/ci/run_ci.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import time +import traceback +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Iterable + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = Path(__file__).with_name("feature_manifest.json") + + +def _rel(path: Path) -> str: + try: + return str(path.resolve().relative_to(REPO_ROOT)) + except ValueError: + return str(path) + + +def _slug(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "check" + + +def _json_dump(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _tail(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[-limit:] + + +def load_manifest() -> dict[str, Any]: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def _validate_manifest(manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if manifest.get("schema") != 1: + errors.append("manifest schema must be 1") + + features = manifest.get("features", {}) + unit_groups = manifest.get("unit_groups", {}) + runtime_groups = manifest.get("runtime_groups", {}) + examples = manifest.get("examples", {}) + required_features = { + "ipv4-default", + "ipv6-core", + "routing-bird-frr", + "exabgp-service", + "looking-glass", + "dns-endpoint", + "legacy-guard", + } + missing = sorted(required_features.difference(features)) + if missing: + errors.append(f"missing required feature declarations: {', '.join(missing)}") + + for feature_id, feature in features.items(): + for group_id in feature.get("unit_groups", []): + if group_id not in unit_groups: + errors.append(f"feature {feature_id} references missing unit group {group_id}") + for group_id in feature.get("runtime_groups", []): + if group_id not in runtime_groups: + errors.append(f"feature {feature_id} references missing runtime group {group_id}") + for example_id in feature.get("compile_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing compile example {example_id}") + for example_id in feature.get("build_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing build example {example_id}") + + for example_id, example in examples.items(): + script = REPO_ROOT / example.get("script", "") + if not script.is_file(): + errors.append(f"example {example_id} script does not exist: {_rel(script)}") + for expected in example.get("expected", []): + if Path(expected).is_absolute(): + errors.append(f"example {example_id} expected path must be relative: {expected}") + for clean in example.get("clean", []): + if Path(clean).is_absolute(): + errors.append(f"example {example_id} clean path must be relative: {clean}") + return errors + + +def feature_coverage(manifest: dict[str, Any]) -> dict[str, Any]: + features = {} + for feature_id, feature in sorted(manifest["features"].items()): + features[feature_id] = { + "status": feature["status"], + "description": feature.get("description", ""), + "unit_groups": feature.get("unit_groups", []), + "compile_examples": feature.get("compile_examples", []), + "build_examples": feature.get("build_examples", []), + "runtime_groups": feature.get("runtime_groups", []), + "notes": feature.get("notes", ""), + } + return { + "schema": manifest["schema"], + "generated_by": "tests/ci/run_ci.py", + "features": features, + } + + +def _write_junit(stage: str, checks: list[dict[str, Any]], path: Path) -> None: + tests = len(checks) + failures = sum(1 for check in checks if check["status"] == "failed") + skipped = sum(1 for check in checks if check["status"] == "skipped") + suite = ET.Element( + "testsuite", + { + "name": f"seedemu-ci-{stage}", + "tests": str(tests), + "failures": str(failures), + "errors": "0", + "skipped": str(skipped), + "time": f"{sum(float(check.get('duration_s', 0.0)) for check in checks):.3f}", + }, + ) + for check in checks: + case = ET.SubElement( + suite, + "testcase", + { + "classname": f"seedemu.ci.{stage}", + "name": check["name"], + "time": f"{float(check.get('duration_s', 0.0)):.3f}", + }, + ) + if check["status"] == "failed": + failure = ET.SubElement( + case, + "failure", + { + "message": check.get("message", "check failed"), + "type": "CommandFailure", + }, + ) + failure.text = check.get("details", "") + elif check["status"] == "skipped": + skipped_node = ET.SubElement(case, "skipped", {"message": check.get("message", "skipped")}) + skipped_node.text = check.get("details", "") + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def _stage_result(stage: str, checks: list[dict[str, Any]], artifact_dir: Path) -> int: + summary = { + "stage": stage, + "status": "failed" if any(check["status"] == "failed" for check in checks) else "passed", + "checks": checks, + } + _json_dump(artifact_dir / "ci-summary.json", summary) + _write_junit(stage, checks, artifact_dir / "junit.xml") + + print(f"[seed-ci] stage={stage} status={summary['status']} artifact_dir={_rel(artifact_dir)}") + for check in checks: + print( + "[seed-ci] " + f"{check['status']}: {check['name']} " + f"log={check.get('log_path', '')}" + ) + if check["status"] == "failed": + command = check.get("command") + if command: + print(f"[seed-ci] failed-command: {' '.join(command)}") + details = (check.get("stderr_tail") or check.get("details") or "").strip() + if details: + print("[seed-ci] failure-tail:") + print(_tail(details, limit=1200)) + return 1 if summary["status"] == "failed" else 0 + + +def _run_command( + name: str, + cmd: list[str], + artifact_dir: Path, + *, + cwd: Path = REPO_ROOT, + env: dict[str, str] | None = None, + timeout: int | None = None, +) -> dict[str, Any]: + start = time.monotonic() + log_path = artifact_dir / "logs" / f"{_slug(name)}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + try: + result = subprocess.run( + cmd, + cwd=str(cwd), + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + duration = time.monotonic() - start + log_path.write_text( + "COMMAND: {}\nCWD: {}\nEXIT: {}\n\nSTDOUT:\n{}\n\nSTDERR:\n{}\n".format( + " ".join(cmd), + _rel(cwd), + result.returncode, + result.stdout, + result.stderr, + ), + encoding="utf-8", + ) + status = "passed" if result.returncode == 0 else "failed" + return { + "name": name, + "status": status, + "command": cmd, + "cwd": _rel(cwd), + "returncode": result.returncode, + "duration_s": duration, + "log_path": _rel(log_path), + "stdout_tail": _tail(result.stdout), + "stderr_tail": _tail(result.stderr), + "message": "command failed" if status == "failed" else "command passed", + "details": _tail(result.stdout + "\n" + result.stderr), + } + except Exception as exc: # pragma: no cover - defensive diagnostics for CI infrastructure failures. + duration = time.monotonic() - start + details = traceback.format_exc() + log_path.write_text(details, encoding="utf-8") + return { + "name": name, + "status": "failed", + "command": cmd, + "cwd": _rel(cwd), + "returncode": None, + "duration_s": duration, + "log_path": _rel(log_path), + "message": str(exc), + "details": details, + } + + +def _git_diff_check_command() -> list[str]: + base_ref = os.environ.get("GITHUB_BASE_REF") + if base_ref: + candidates = [f"origin/{base_ref}", base_ref] + for candidate in candidates: + probe = subprocess.run( + ["git", "rev-parse", "--verify", candidate], + cwd=str(REPO_ROOT), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if probe.returncode == 0: + return ["git", "diff", "--check", f"{candidate}...HEAD"] + return ["git", "diff", "--check"] + + +def _example_ids(manifest: dict[str, Any], key: str) -> list[str]: + ids: list[str] = [] + for example_id, example in manifest["examples"].items(): + if example.get(key): + ids.append(example_id) + return ids + + +def _pythonpath_env(extra: dict[str, str] | None = None) -> dict[str, str]: + env = dict(os.environ) + existing = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = str(REPO_ROOT) if not existing else f"{REPO_ROOT}:{existing}" + env.setdefault("DOCKER_BUILDKIT", "0") + env.setdefault("COMPOSE_BAKE", "false") + env.setdefault("COMPOSE_PARALLEL_LIMIT", "1") + if extra: + env.update(extra) + return env + + +def _pytest_env() -> dict[str, str]: + return _pythonpath_env({"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}) + + +def _safe_clean(example_dir: Path, relative_paths: Iterable[str]) -> None: + base = example_dir.resolve() + for item in relative_paths: + target = (example_dir / item).resolve() + if target == base or base not in target.parents: + raise ValueError(f"refusing to clean path outside example directory: {target}") + if target.is_dir(): + shutil.rmtree(target) + elif target.exists(): + target.unlink() + + +def _compile_example( + example_id: str, + example: dict[str, Any], + artifact_dir: Path, + *, + clean: bool, + name_prefix: str = "compile", +) -> dict[str, Any]: + script = REPO_ROOT / example["script"] + example_dir = script.parent + if clean: + _safe_clean(example_dir, example.get("clean", [])) + cmd = [sys.executable, script.name] + list(example.get("args", [])) + check = _run_command( + f"{name_prefix}:{example_id}", + cmd, + artifact_dir, + cwd=example_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("timeout", 900)), + ) + if check["status"] != "passed": + return check + + missing = [item for item in example.get("expected", []) if not (example_dir / item).exists()] + if missing: + check["status"] = "failed" + check["message"] = "expected compile outputs are missing" + check["details"] = "Missing outputs: " + ", ".join(missing) + return check + + +def run_static(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + + errors = _validate_manifest(manifest) + coverage = feature_coverage(manifest) + _json_dump(artifact_dir / "feature-coverage.json", coverage) + checks.append( + { + "name": "manifest", + "status": "failed" if errors else "passed", + "duration_s": 0.0, + "message": "manifest validation failed" if errors else "manifest validation passed", + "details": "\n".join(errors), + "log_path": _rel(artifact_dir / "feature-coverage.json"), + } + ) + + checks.append(_run_command("whitespace", _git_diff_check_command(), artifact_dir)) + + compile_targets = [ + target + for target in [ + "seedemu", + "tests/control_plane", + "tests/ci", + ] + if (REPO_ROOT / target).exists() + ] + example_dirs = sorted( + { + str(Path(manifest["examples"][example_id]["script"]).parent) + for example_id in _example_ids(manifest, "compile") + } + ) + compile_targets.extend(example_dirs) + checks.append( + _run_command( + "compileall", + [ + sys.executable, + "-m", + "compileall", + "-q", + "-x", + "seedemu/services/EthereumService/EthTemplates/", + ] + + compile_targets, + artifact_dir, + ) + ) + + smoke = ( + "import seedemu; " + "from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf; " + "from seedemu.services import BgpLookingGlassService; " + "from seedemu.compiler import Docker; " + "import tests.ci.run_ci" + ) + checks.append(_run_command("import-smoke", [sys.executable, "-c", smoke], artifact_dir, env=_pythonpath_env())) + return _stage_result("static", checks, artifact_dir) + + +def run_unit(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["unit_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"pytest:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=900)) + return _stage_result("unit", checks, artifact_dir) + + +def run_example_compile(artifact_dir: Path) -> int: + manifest = load_manifest() + checks = [ + _compile_example(example_id, manifest["examples"][example_id], artifact_dir, clean=True) + for example_id in _example_ids(manifest, "compile") + ] + return _stage_result("example-compile", checks, artifact_dir) + + +def run_example_build(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for example_id in _example_ids(manifest, "build"): + example = manifest["examples"][example_id] + compile_check = _compile_example( + example_id, + example, + artifact_dir, + clean=True, + name_prefix="compile-before-build", + ) + checks.append(compile_check) + if compile_check["status"] != "passed": + continue + + script = REPO_ROOT / example["script"] + compose_file = script.parent / "output" / "docker-compose.yml" + checks.append( + _run_command( + f"docker-build:{example_id}", + ["docker", "compose", "-f", str(compose_file), "build"], + artifact_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("build_timeout", 1800)), + ) + ) + return _stage_result("example-build", checks, artifact_dir) + + +def run_runtime_integration(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["runtime_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"runtime:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=7200)) + return _stage_result("runtime-integration", checks, artifact_dir) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run feature-oriented SEED Emulator CI stages.") + parser.add_argument( + "stage", + choices=["static", "unit", "example-compile", "example-build", "runtime-integration"], + ) + parser.add_argument("--artifact-dir", default="ci-artifacts", help="Directory for logs, JSON, and JUnit output.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + artifact_dir = (REPO_ROOT / args.artifact_dir).resolve() + artifact_dir.mkdir(parents=True, exist_ok=True) + if args.stage == "static": + return run_static(artifact_dir) + if args.stage == "unit": + return run_unit(artifact_dir) + if args.stage == "example-compile": + return run_example_compile(artifact_dir) + if args.stage == "example-build": + return run_example_build(artifact_dir) + if args.stage == "runtime-integration": + return run_runtime_integration(artifact_dir) + raise AssertionError(f"unknown stage: {args.stage}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/k8s/PHASE2_KVM_E2E.md b/tests/k8s/PHASE2_KVM_E2E.md deleted file mode 100644 index 5f1cfc7b5..000000000 --- a/tests/k8s/PHASE2_KVM_E2E.md +++ /dev/null @@ -1,154 +0,0 @@ -# Phase 2: KVM + OVN + OVS E2E PR Check - -This document records the intended second-stage native Kubernetes PR check. It -is not implemented yet. The first-stage workflow only compiles manifests and -builds workload images on GitHub-hosted runners. - -## Goal - -Phase 2 should prove that the generated B61 native Kubernetes example can run -end to end on a KVM-backed K3s cluster using Kube-OVN and OVS. - -The check should cover: - -- KVM VM creation from `kvm.yaml`. -- K3s installation on the generated master and worker VMs. -- Kube-OVN/OVS fabric installation and validation. -- Native SeedEMU image build and push to the cluster registry. -- Manifest deployment with `make up`. -- Runtime validation that deployments and pods become ready. -- Cleanup with `make clean` and VM destruction. - -Real physical-machine E2E is intentionally out of scope because it is expensive -and needs lab-specific hosts and secrets. - -## Runner Requirements - -The job must run on a dedicated self-hosted GitHub Actions runner, not on -`ubuntu-latest`. - -Required runner labels: - -```yaml -runs-on: [self-hosted, linux, x64, kvm, seedemu-k8s] -``` - -Required host capabilities: - -- `/dev/kvm` is available. -- libvirt and `virsh` are installed and usable by the runner. -- the default libvirt network or configured bridge is available. -- Docker with buildx is available. -- enough CPU, memory, and disk exist for the B61 KVM topology. -- outbound network access is available for image and package downloads. - -The runner should be isolated from long-running user clusters. If the same host -must be shared, VM names, libvirt storage pools, namespace names, registry ports, -and working directories must be unique to the CI run. - -## Trigger Conditions - -The workflow should appear on PRs but only run automatically for PRs from this -repository, not from forks. - -Recommended trigger: - -```yaml -on: - pull_request: - branches: - - master - - development - paths: - - "seedemu/compiler/kubernetes.py" - - "seedemu/compiler/__init__.py" - - "seedemu/k8sTools/**" - - "examples/internet/b61_k8s_compile/**" - - "tests/k8s/**" - - ".github/workflows/k8s-*.yaml" - - "setup.py" - - "MANIFEST.in" - workflow_dispatch: -``` - -Recommended job guard: - -```yaml -if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository -``` - -Recommended concurrency: - -```yaml -concurrency: - group: seedemu-k8s-kvm-e2e - cancel-in-progress: false -``` - -The KVM E2E check should start as optional. After repeated stable runs, the -repository can decide whether to mark it as required for K8s-related PRs. - -## Expected Flow - -The job should use a fresh working directory and generate all stage artifacts -from committed files: - -```bash -source development.env -python examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir examples/internet/b61_k8s_compile/output - -cd examples/internet/b61_k8s_compile -python ./k8sTools.py build \ - --input configKvmOvn.yaml \ - --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml -python ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml -python ./k8sTools.py down -f ./output -k kubeconfig.yaml -python ./k8sTools.py destroy -d configK3s.yaml -``` - -The final implementation should use CI-specific names and directories so that -parallel or failed runs do not collide with developer experiments. - -## Cleanup Contract - -Cleanup must run with `if: always()` or shell traps. - -Required cleanup steps: - -- `make clean || true` for the SeedEMU namespace. -- `destroyKvmVms.sh || true` for all VMs created by the job. -- remove any temporary local registry container used by the job. -- collect diagnostics before destruction when a failure occurs. - -Do not run broad destructive commands such as `docker system prune` on a shared -self-hosted runner unless the runner is dedicated to this workflow. - -## Failure Artifacts - -Upload these artifacts on failure: - -- compiled `k8s.kube-ovn.yaml` and `images.yaml`. -- generated `setup/*.yaml` and `running/configRunning.yaml`. -- K3s kubeconfig and inventory if they do not contain secrets. -- `kubectl get pods -A -o wide`. -- `kubectl describe pods -n seedemu-k8s-b61`. -- recent Kubernetes events. -- Kube-OVN pod status and relevant logs. -- `virsh list --all`. -- generated `kvmState.yaml`. - -Secrets such as private SSH keys must never be uploaded. - -## Success Criteria - -The Phase 2 job should pass only when: - -- all VMs are created and reachable; -- K3s nodes are Ready; -- Kube-OVN validation passes; -- all images from `images.yaml` build and push to the registry; -- `make up` completes and all SeedEMU workloads become Ready; -- `make clean` succeeds; -- `destroyKvmVms.sh` succeeds or confirms no CI-owned VMs remain. diff --git a/tests/k8s/README.md b/tests/k8s/README.md deleted file mode 100644 index 956c53431..000000000 --- a/tests/k8s/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Native Kubernetes PR Checks - -This directory contains the first-stage GitHub PR checks for the -`seedemu.k8sTools` native Kubernetes workflow. - -## PR Workflow - -`.github/workflows/k8s-check.yaml` runs automatically on PRs that touch native -K8s compiler, k8sTools, B61 example, packaging, Docker image sources, or these -test helpers. - -The workflow has three jobs: - -- `K8s Static`: compiles Python sources, runs `bash -n` on shell resources, - parses YAML files, and rejects checked-in host-local absolute paths. -- `K8s Compile`: compiles the B61 native K8s example and validates generated - Kube-OVN manifests plus `images.yaml`. -- `K8s Build Images`: builds every image listed in `images.yaml`, pushes to a - temporary local registry, then pulls the pushed images back for verification. - This job recompiles the B61 output in its own runner so generated - `base_images/` contexts are present before Docker builds start. - -These jobs do not create KVM VMs, install K3s, or apply Kubernetes resources. -The destructive KVM/K3s E2E plan is documented separately in -`PHASE2_KVM_E2E.md`. - -## Local Reproduction - -Run the non-destructive checks: - -```bash -bash tests/k8s/runNativeK8sPrCheck.sh -``` - -Also build and push workload images to a temporary local registry: - -```bash -bash tests/k8s/runNativeK8sPrCheck.sh --build-images -``` diff --git a/tests/k8s/buildNativeK8sImages.sh b/tests/k8s/buildNativeK8sImages.sh deleted file mode 100755 index 32310bdfb..000000000 --- a/tests/k8s/buildNativeK8sImages.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Build native SeedEMU Kubernetes workload images for PR checks. -# -# Inputs: -# - --output-dir: compile output containing k8s.kube-ovn.yaml, images.yaml, -# base_images/, and per-node Docker build contexts. -# - --registry-prefix: local registry host:port used for image pushes. -# - --image-registry-prefix: logical compiler image prefix to rewrite. -# -# Generated outputs: -# - Docker images tagged under the selected registry prefix. -# -# Side effects: -# - starts a temporary local Docker registry when the registry host is -# 127.0.0.1 or localhost; -# - builds images with docker buildx; -# - pushes workload images to the local registry; -# - removes the temporary registry before exit. -# -# Expected context: -# - GitHub-hosted Ubuntu runner or local developer machine with Docker, -# buildx, Python 3, and PyYAML. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -OUTPUT_DIR="" -REGISTRY_PREFIX="127.0.0.1:5000" -IMAGE_REGISTRY_PREFIX="seedemu" -REGISTRY_NAME="seedemu-native-k8s-ci-registry" -STARTED_REGISTRY="" -ORIGINAL_BUILDER="" -if command -v docker >/dev/null 2>&1; then - ORIGINAL_BUILDER="$( - docker buildx ls 2>/dev/null | - awk 'NR > 1 && $1 ~ /\*$/ {gsub(/\*/, "", $1); builder=$1} END {print builder}' || - true - )" -fi - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - echo "--output-dir is required" >&2 - usage >&2 - exit 1 -fi - -OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" -RUNNING_DIR="${REPO_ROOT}/seedemu/k8sTools/resources/running" -MANIFEST_HELPER="${RUNNING_DIR}/manageK8sManifest.py" -BUILD_SCRIPT="${RUNNING_DIR}/buildRegistryImages.py" -IMAGES_YAML="${OUTPUT_DIR}/images.yaml" - -cleanup() { - if [ -n "${STARTED_REGISTRY}" ]; then - docker rm -f "${STARTED_REGISTRY}" >/dev/null 2>&1 || true - fi - if [ -n "${ORIGINAL_BUILDER}" ]; then - docker buildx use "${ORIGINAL_BUILDER}" >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -requireFile() { - # Args: - # $1: file path that must exist. - local path="$1" - if [ ! -f "${path}" ]; then - echo "Required file not found: ${path}" >&2 - exit 1 - fi -} - -startLocalRegistryIfNeeded() { - # Args: - # $1: registry prefix in host:port form. - local prefix="$1" - local host="${prefix%%:*}" - local port="${prefix##*:}" - if [ "${host}" != "127.0.0.1" ] && [ "${host}" != "localhost" ]; then - return 0 - fi - if [ "${port}" = "${prefix}" ]; then - echo "Local registry prefix must include a port: ${prefix}" >&2 - exit 1 - fi - docker rm -f "${REGISTRY_NAME}" >/dev/null 2>&1 || true - docker run -d --name "${REGISTRY_NAME}" -p "${port}:5000" registry:2 >/dev/null - STARTED_REGISTRY="${REGISTRY_NAME}" -} - -selectDockerDriverBuilder() { - # Use a docker driver builder so --load images are visible as local FROM - # dependencies to later builds in the same check. - local default_driver - default_driver="$(docker buildx inspect default 2>/dev/null | awk -F: '$1 == "Driver" {driver=$2} END {gsub(/^[ \t]+|[ \t]+$/, "", driver); print driver}')" - if [ "${default_driver}" != "docker" ]; then - echo "Docker buildx default builder must use the docker driver for native K8s image checks." >&2 - exit 1 - fi - docker buildx use default >/dev/null -} - -verifyPushedImages() { - # Args: - # $1: images.yaml path. - # $2: logical compiler image prefix. - # $3: pushed registry prefix. - local images_yaml="$1" - local image_registry_prefix="$2" - local registry_prefix="$3" - python3 "${MANIFEST_HELPER}" mapped-images \ - --images-yaml "${images_yaml}" \ - --image-registry-prefix "${image_registry_prefix}" \ - --registry-prefix "${registry_prefix}" | - while IFS=$'\t' read -r image _context; do - [ -n "${image}" ] || continue - echo "+ docker pull ${image}" - docker pull "${image}" >/dev/null - done -} - -requireFile "${IMAGES_YAML}" -requireFile "${MANIFEST_HELPER}" -requireFile "${BUILD_SCRIPT}" - -docker info >/dev/null -docker buildx version >/dev/null - -selectDockerDriverBuilder -startLocalRegistryIfNeeded "${REGISTRY_PREFIX}" - -python3 "${BUILD_SCRIPT}" \ - --output-dir "${OUTPUT_DIR}" \ - --registry-prefix "${REGISTRY_PREFIX}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" - -verifyPushedImages "${IMAGES_YAML}" "${IMAGE_REGISTRY_PREFIX}" "${REGISTRY_PREFIX}" - -echo "Native K8s image build check completed for ${OUTPUT_DIR}." diff --git a/tests/k8s/runNativeK8sPrCheck.sh b/tests/k8s/runNativeK8sPrCheck.sh deleted file mode 100755 index 5c5a39527..000000000 --- a/tests/k8s/runNativeK8sPrCheck.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# Run the non-destructive native Kubernetes PR checks locally. -# -# Inputs: -# - optional --output-dir path for compiler output. -# - optional --build-images to also build/push images to a local registry. -# - optional --registry-prefix host:port for the image build check. -# -# Generated outputs: -# - compiler output under the selected output directory. -# -# Side effects: -# - without --build-images: none outside the output directory. -# - with --build-images: starts a temporary local Docker registry when the -# registry prefix points at localhost, builds images, pushes them, and removes -# the registry container before exit. -# -# Expected context: -# - repository checkout with Python 3.10+, PyYAML, and compile dependencies. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -OUTPUT_DIR="" -BUILD_IMAGES="false" -REGISTRY_PREFIX="127.0.0.1:5000" -IMAGE_REGISTRY_PREFIX="seedemu" -NAMESPACE="seedemu-k8s-b61" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - OUTPUT_DIR="$(mktemp -d)/seedemu-k8s-output" -fi - -cd "${REPO_ROOT}" -source development.env - -python3 tests/k8s/validateK8sStatic.py -python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${OUTPUT_DIR}" -python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${OUTPUT_DIR}" \ - --expected-namespace "${NAMESPACE}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" - -if [ "${BUILD_IMAGES}" = "true" ]; then - bash tests/k8s/buildNativeK8sImages.sh \ - --output-dir "${OUTPUT_DIR}" \ - --registry-prefix "${REGISTRY_PREFIX}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" -fi - -echo "Native K8s PR checks completed for ${OUTPUT_DIR}." diff --git a/tests/k8s/validateK8sStatic.py b/tests/k8s/validateK8sStatic.py deleted file mode 100755 index 8db8f81f6..000000000 --- a/tests/k8s/validateK8sStatic.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python3 -"""Validate SeedEMU native Kubernetes source files before PR build checks. - -Inputs: -- repository source tree, defaulting to the current checkout. - -Outputs: -- console diagnostics only. - -Side effects: -- none; Python files are compiled to syntax trees in memory and shell scripts - are checked with bash -n. - -Expected context: -- GitHub-hosted or local developer machine with Python 3.10+ and bash. -""" -from __future__ import annotations - -import argparse -import py_compile -import subprocess -import sys -from pathlib import Path - -import yaml - - -SOURCE_ROOTS = ( - Path(".github/workflows/k8s-check.yaml"), - Path("MANIFEST.in"), - Path("setup.py"), - Path("seedemu/compiler/kubernetes.py"), - Path("seedemu/compiler/__init__.py"), - Path("seedemu/k8sTools"), - Path("examples/internet/b61_k8s_compile"), - Path("tests/k8s"), -) -GENERATED_EXAMPLE_PREFIXES = ( - "examples/internet/b61_k8s_compile/output/", - "examples/internet/b61_k8s_compile/configK3s.yaml", - "examples/internet/b61_k8s_compile/configK3sTemplate.yaml", - "examples/internet/b61_k8s_compile/kubeconfig.yaml", -) -TEXT_SUFFIXES = {".py", ".sh", ".yaml", ".yml", ".md", ".in"} -FORBIDDEN_TEXT = (str(Path("/home") / "lxl") + "/", str(Path("/home") / "lxl")) - - -def parseArgs() -> argparse.Namespace: - """Parse command-line options.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--repo-root", - type=Path, - default=Path(__file__).resolve().parents[2], - help="SeedEMU repository root.", - ) - return parser.parse_args() - - -def relativePath(path: Path, repo_root: Path) -> str: - """Return a POSIX relative path used for matching and diagnostics. - - Args: - path: Absolute or repository-relative path. - repo_root: Repository root directory. - """ - return path.resolve().relative_to(repo_root.resolve()).as_posix() - - -def isGeneratedExamplePath(path: Path, repo_root: Path) -> bool: - """Return true for generated B61 setup/running/output artifacts. - - Args: - path: Source file candidate. - repo_root: Repository root directory. - """ - rel = relativePath(path, repo_root) - return any(rel.startswith(prefix) for prefix in GENERATED_EXAMPLE_PREFIXES) - - -def isIgnoredPath(path: Path, repo_root: Path) -> bool: - """Return true for cache/build files that should not be source-checked. - - Args: - path: Source file candidate. - repo_root: Repository root directory. - """ - parts = path.relative_to(repo_root).parts - if "__pycache__" in parts: - return True - if path.name == "codex_worklog.md": - return True - return isGeneratedExamplePath(path, repo_root) - - -def iterSourceFiles(repo_root: Path) -> list[Path]: - """Return source files under K8s-related paths. - - Args: - repo_root: Repository root directory. - """ - files: list[Path] = [] - for root in SOURCE_ROOTS: - candidate = repo_root / root - if not candidate.exists(): - continue - if candidate.is_file(): - files.append(candidate) - continue - files.extend(path for path in candidate.rglob("*") if path.is_file()) - return sorted(path for path in files if not isIgnoredPath(path, repo_root)) - - -def validatePythonSyntax(paths: list[Path]) -> None: - """Compile Python source files without importing them. - - Args: - paths: Python files to validate. - """ - for path in paths: - py_compile.compile(str(path), doraise=True) - - -def validateShellSyntax(paths: list[Path]) -> None: - """Run bash -n on shell scripts. - - Args: - paths: Shell script paths to validate. - """ - for path in paths: - subprocess.run(["bash", "-n", str(path)], check=True) - - -def validateYamlSyntax(paths: list[Path]) -> None: - """Parse YAML files without applying YAML 1.1 scalar coercions. - - Args: - paths: YAML files to validate. - """ - for path in paths: - list(yaml.compose_all(path.read_text(encoding="utf-8"))) - - -def validateNoHostLocalArtifacts(paths: list[Path], repo_root: Path) -> None: - """Reject known host-local paths and checked-in image tarballs. - - Args: - paths: K8s-related source files. - repo_root: Repository root directory. - """ - failures: list[str] = [] - for path in paths: - rel = relativePath(path, repo_root) - if path.suffix == ".tar": - failures.append(f"{rel}: image tarball must not be checked in") - continue - if path.suffix not in TEXT_SUFFIXES and path.name not in {"Makefile"}: - continue - try: - text = path.read_text(encoding="utf-8") - except UnicodeDecodeError: - continue - for token in FORBIDDEN_TEXT: - if token in text: - failures.append(f"{rel}: contains host-local path {token}") - if failures: - raise SystemExit("\n".join(failures)) - - -def main() -> int: - """Run static validation for native K8s sources.""" - args = parseArgs() - repo_root = args.repo_root.expanduser().resolve() - files = iterSourceFiles(repo_root) - python_files = [path for path in files if path.suffix == ".py"] - shell_files = [path for path in files if path.suffix == ".sh"] - yaml_files = [path for path in files if path.suffix in {".yaml", ".yml"}] - - validatePythonSyntax(python_files) - validateShellSyntax(shell_files) - validateYamlSyntax(yaml_files) - validateNoHostLocalArtifacts(files, repo_root) - - print( - "Validated " - f"{len(python_files)} Python files, " - f"{len(shell_files)} shell scripts, " - f"and {len(yaml_files)} YAML files." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/k8s/validateNativeK8sOutput.py b/tests/k8s/validateNativeK8sOutput.py deleted file mode 100755 index 2ec955d0c..000000000 --- a/tests/k8s/validateNativeK8sOutput.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env python3 -"""Validate native Kubernetes compiler output used by PR checks. - -Inputs: -- an output directory generated by mini_internet_k8s.py. - -Outputs: -- console diagnostics only. - -Side effects: -- none; this script only parses YAML, JSON, and Docker build contexts. - -Expected context: -- GitHub-hosted or local developer machine with Python 3.10+ and PyYAML. -""" -from __future__ import annotations - -import argparse -import ipaddress -import json -import re -import sys -from pathlib import Path -from typing import Any - -import yaml - - -MANIFEST_NAME = "k8s.kube-ovn.yaml" -IMAGES_NAME = "images.yaml" -LEGACY_MANIFEST_NAME = "k8s.yaml" -FORBIDDEN_TEXT = (str(Path("/home") / "lxl") + "/", str(Path("/home") / "lxl")) -COMPILER_BASE_TAG_RE = re.compile(r"^[0-9a-f]{32}(?::latest)?$") - - -def parseArgs() -> argparse.Namespace: - """Parse command-line options.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output-dir", type=Path, required=True, help="Compiler output directory.") - parser.add_argument("--expected-namespace", required=True, help="Expected Kubernetes namespace name.") - parser.add_argument( - "--image-registry-prefix", - default="seedemu", - help="Logical image prefix written by NativeKubernetesCompiler.", - ) - return parser.parse_args() - - -def fail(message: str) -> None: - """Exit with a validation failure message. - - Args: - message: Human-readable validation error. - """ - raise SystemExit(f"native k8s output validation failed: {message}") - - -def loadYamlDocuments(path: Path) -> list[dict[str, Any]]: - """Load a Kubernetes multi-document YAML file. - - Args: - path: Manifest path. - """ - docs = [doc for doc in yaml.safe_load_all(path.read_text(encoding="utf-8")) if doc is not None] - invalid = [doc for doc in docs if not isinstance(doc, dict)] - if invalid: - fail(f"{path.name} contains non-mapping YAML documents") - return docs - - -def loadImages(path: Path) -> list[dict[str, str]]: - """Load images.yaml and return image entries. - - Args: - path: images.yaml path. - """ - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - if not isinstance(data, dict): - fail("images.yaml root must be a mapping") - images = data.get("images") - if not isinstance(images, list) or not images: - fail("images.yaml must contain a non-empty images list") - for item in images: - if not isinstance(item, dict): - fail("images.yaml entries must be mappings") - if not isinstance(item.get("name"), str) or not item["name"].strip(): - fail("images.yaml entry requires non-empty name") - if not isinstance(item.get("context"), str) or not item["context"].strip(): - fail(f"image {item.get('name')} requires non-empty context") - return images - - -def indexKinds(docs: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: - """Group Kubernetes documents by kind. - - Args: - docs: Parsed Kubernetes manifest documents. - """ - by_kind: dict[str, list[dict[str, Any]]] = {} - for doc in docs: - kind = str(doc.get("kind") or "") - if not kind: - fail(f"manifest document lacks kind: {doc}") - by_kind.setdefault(kind, []).append(doc) - return by_kind - - -def validateNamespace(by_kind: dict[str, list[dict[str, Any]]], expected_namespace: str) -> None: - """Validate the compiler-created namespace. - - Args: - by_kind: Manifest documents grouped by kind. - expected_namespace: Namespace expected from the example compiler. - """ - namespaces = by_kind.get("Namespace", []) - if len(namespaces) != 1: - fail(f"expected exactly one Namespace, got {len(namespaces)}") - namespace = str((namespaces[0].get("metadata") or {}).get("name") or "") - if namespace != expected_namespace: - fail(f"expected namespace {expected_namespace}, got {namespace}") - - -def validateRequiredKinds(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate that Kube-OVN output contains all required resource kinds. - - Args: - by_kind: Manifest documents grouped by kind. - """ - required = ("Namespace", "Vpc", "Subnet", "NetworkAttachmentDefinition", "Deployment") - missing = [kind for kind in required if not by_kind.get(kind)] - if missing: - fail(f"manifest missing required kinds: {', '.join(missing)}") - - -def validateVpcAndSubnets(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate Kube-OVN Vpc and Subnet resource shape. - - Args: - by_kind: Manifest documents grouped by kind. - """ - vpcs = by_kind.get("Vpc", []) - if len(vpcs) != 1: - fail(f"expected exactly one Kube-OVN Vpc, got {len(vpcs)}") - vpc_name = str((vpcs[0].get("metadata") or {}).get("name") or "") - if not vpc_name: - fail("Kube-OVN Vpc requires metadata.name") - for subnet in by_kind.get("Subnet", []): - spec = subnet.get("spec") or {} - if spec.get("vpc") != vpc_name: - fail(f"Subnet {(subnet.get('metadata') or {}).get('name')} does not reference Vpc {vpc_name}") - provider = str(spec.get("provider") or "") - if not provider.endswith(".ovn"): - fail(f"Subnet provider must use Kube-OVN provider suffix: {provider}") - cidr = str(spec.get("cidrBlock") or "") - try: - ipaddress.ip_network(cidr, strict=False) - except ValueError as exc: - fail(f"invalid Subnet cidrBlock {cidr}: {exc}") - if not spec.get("gateway"): - fail(f"Subnet {(subnet.get('metadata') or {}).get('name')} requires gateway") - - -def validateNetworkAttachments(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate that Multus NADs are rendered for Kube-OVN, not macvlan. - - Args: - by_kind: Manifest documents grouped by kind. - """ - for nad in by_kind.get("NetworkAttachmentDefinition", []): - metadata = nad.get("metadata") or {} - name = str(metadata.get("name") or "") - spec = nad.get("spec") or {} - raw_config = spec.get("config") - if not isinstance(raw_config, str): - fail(f"NAD {name} requires string spec.config") - try: - config = json.loads(raw_config) - except json.JSONDecodeError as exc: - fail(f"NAD {name} spec.config is not JSON: {exc}") - if config.get("type") != "kube-ovn": - fail(f"NAD {name} must use type kube-ovn, got {config.get('type')}") - provider = str(config.get("provider") or "") - if not provider.endswith(".ovn"): - fail(f"NAD {name} provider must end with .ovn") - if "master" in config: - fail(f"NAD {name} still contains macvlan master field") - - -def podTemplateAnnotations(workload: dict[str, Any]) -> dict[str, Any]: - """Return workload pod-template annotations. - - Args: - workload: Deployment-like Kubernetes resource. - """ - return ( - ((workload.get("spec") or {}).get("template") or {}) - .get("metadata", {}) - .get("annotations", {}) - or {} - ) - - -def validateDeployments( - by_kind: dict[str, list[dict[str, Any]]], - expected_namespace: str, - image_names: set[str], -) -> None: - """Validate Deployment image and Kube-OVN annotation contracts. - - Args: - by_kind: Manifest documents grouped by kind. - expected_namespace: Namespace expected from the example compiler. - image_names: Logical image names loaded from images.yaml. - """ - for deployment in by_kind.get("Deployment", []): - metadata = deployment.get("metadata") or {} - name = str(metadata.get("name") or "") - if metadata.get("namespace") != expected_namespace: - fail(f"Deployment {name} must be in namespace {expected_namespace}") - containers = ( - ((deployment.get("spec") or {}).get("template") or {}) - .get("spec", {}) - .get("containers", []) - ) - if len(containers) != 1: - fail(f"Deployment {name} expected one main container, got {len(containers)}") - image = str(containers[0].get("image") or "") - if image not in image_names: - fail(f"Deployment {name} image {image} missing from images.yaml") - annotations = podTemplateAnnotations(deployment) - raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") - if not isinstance(raw_networks, str): - fail(f"Deployment {name} lacks Multus network annotation") - try: - networks = json.loads(raw_networks) - except json.JSONDecodeError as exc: - fail(f"Deployment {name} network annotation is not JSON: {exc}") - if not isinstance(networks, list) or not networks: - fail(f"Deployment {name} network annotation must be a non-empty list") - for network in networks: - if not isinstance(network, dict): - fail(f"Deployment {name} contains non-mapping network selection") - if "ips" in network: - fail(f"Deployment {name} still contains macvlan-style ips field") - nad_name = str(network.get("name") or "") - nad_namespace = str(network.get("namespace") or expected_namespace) - key = f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address" - if key not in annotations: - fail(f"Deployment {name} lacks Kube-OVN IP annotation {key}") - - -def validateImages(images: list[dict[str, str]], output_dir: Path, image_registry_prefix: str) -> set[str]: - """Validate image entries and Docker build contexts. - - Args: - images: Parsed images.yaml entries. - output_dir: Compiler output directory. - image_registry_prefix: Expected logical image prefix. - """ - image_names: set[str] = set() - logical_prefix = image_registry_prefix.rstrip("/") + "/" - for item in images: - image = item["name"].strip() - context = item["context"].strip() - if not image.startswith(logical_prefix): - fail(f"image {image} must start with {logical_prefix}") - if image in image_names: - fail(f"duplicate image entry {image}") - image_names.add(image) - context_path = Path(context) - if context_path.is_absolute(): - fail(f"image {image} context must be relative, got {context}") - resolved_context = (output_dir / context_path).resolve() - try: - resolved_context.relative_to(output_dir.resolve()) - except ValueError: - fail(f"image {image} context escapes output directory: {context}") - if not (resolved_context / "Dockerfile").is_file(): - fail(f"image {image} context lacks Dockerfile: {context}") - return image_names - - -def firstFromImage(dockerfile: Path) -> str: - """Return the first FROM image reference in a Dockerfile. - - Args: - dockerfile: Dockerfile path. - """ - for line in dockerfile.read_text(encoding="utf-8").splitlines(): - parts = line.strip().split() - if not parts or parts[0].upper() != "FROM": - continue - if len(parts) >= 3 and parts[1].startswith("--platform="): - return parts[2] - if len(parts) >= 2: - return parts[1] - fail(f"{dockerfile.relative_to(dockerfile.parent.parent)} has malformed FROM line") - fail(f"{dockerfile.relative_to(dockerfile.parent.parent)} lacks FROM line") - - -def validateCompilerBaseImages(output_dir: Path) -> None: - """Validate that hash-tagged node Dockerfiles have staged base contexts. - - Args: - output_dir: Compiler output directory. - """ - required: set[str] = set() - for dockerfile in output_dir.rglob("Dockerfile"): - try: - dockerfile.relative_to(output_dir / "base_images") - continue - except ValueError: - pass - from_image = firstFromImage(dockerfile) - if COMPILER_BASE_TAG_RE.match(from_image): - required.add(from_image.removesuffix(":latest")) - - for base_tag in sorted(required): - context = output_dir / "base_images" / base_tag / "Dockerfile" - if not context.is_file(): - fail(f"missing base_images/{base_tag}/Dockerfile for generated FROM {base_tag}") - - -def validateNoHostLocalText(output_dir: Path) -> None: - """Reject host-local absolute paths in generated text artifacts. - - Args: - output_dir: Compiler output directory. - """ - for path in output_dir.rglob("*"): - if not path.is_file() or path.suffix == ".tar": - continue - try: - text = path.read_text(encoding="utf-8") - except UnicodeDecodeError: - continue - for token in FORBIDDEN_TEXT: - if token in text: - fail(f"{path.relative_to(output_dir)} contains host-local path {token}") - - -def main() -> int: - """Run native K8s compile-output validation.""" - args = parseArgs() - output_dir = args.output_dir.expanduser().resolve() - manifest_path = output_dir / MANIFEST_NAME - images_path = output_dir / IMAGES_NAME - legacy_manifest_path = output_dir / LEGACY_MANIFEST_NAME - - if not manifest_path.is_file(): - fail(f"missing {MANIFEST_NAME}") - if not images_path.is_file(): - fail(f"missing {IMAGES_NAME}") - if legacy_manifest_path.exists(): - fail(f"{LEGACY_MANIFEST_NAME} should not be generated for Kube-OVN output") - - docs = loadYamlDocuments(manifest_path) - images = loadImages(images_path) - by_kind = indexKinds(docs) - image_names = validateImages(images, output_dir, args.image_registry_prefix) - - validateRequiredKinds(by_kind) - validateNamespace(by_kind, args.expected_namespace) - validateVpcAndSubnets(by_kind) - validateNetworkAttachments(by_kind) - validateDeployments(by_kind, args.expected_namespace, image_names) - validateCompilerBaseImages(output_dir) - validateNoHostLocalText(output_dir) - - print( - "Validated native K8s output: " - f"{len(docs)} manifest documents, {len(images)} image contexts." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main())