From a67c2560a648ac1edbbef37ec583c1ad87496233 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Fri, 5 Jun 2026 08:11:51 +0800 Subject: [PATCH 1/6] Import development2 PR CI framework --- .github/workflows/k8s-check.yaml | 136 ----- .github/workflows/on-pull-request.yaml | 253 ++++----- tests/ci/README.md | 36 ++ tests/ci/__init__.py | 1 + tests/ci/feature_manifest.json | 145 +++++ tests/ci/run_ci.py | 492 +++++++++++++++++ .../test_bgp_control_plane_extensions.py | 496 ++++++++++++++++++ .../control_plane/test_bgp_runtime_matrix.py | 413 +++++++++++++++ tests/k8s/PHASE2_KVM_E2E.md | 154 ------ tests/k8s/README.md | 39 -- tests/k8s/buildNativeK8sImages.sh | 178 ------- tests/k8s/runNativeK8sPrCheck.sh | 91 ---- tests/k8s/validateK8sStatic.py | 194 ------- tests/k8s/validateNativeK8sOutput.py | 378 ------------- 14 files changed, 1685 insertions(+), 1321 deletions(-) delete mode 100644 .github/workflows/k8s-check.yaml create mode 100644 tests/ci/README.md create mode 100644 tests/ci/__init__.py create mode 100644 tests/ci/feature_manifest.json create mode 100644 tests/ci/run_ci.py create mode 100644 tests/control_plane/test_bgp_control_plane_extensions.py create mode 100644 tests/control_plane/test_bgp_runtime_matrix.py delete mode 100644 tests/k8s/PHASE2_KVM_E2E.md delete mode 100644 tests/k8s/README.md delete mode 100755 tests/k8s/buildNativeK8sImages.sh delete mode 100755 tests/k8s/runNativeK8sPrCheck.sh delete mode 100755 tests/k8s/validateK8sStatic.py delete mode 100755 tests/k8s/validateNativeK8sOutput.py diff --git a/.github/workflows/k8s-check.yaml b/.github/workflows/k8s-check.yaml deleted file mode 100644 index a52c51330..000000000 --- a/.github/workflows/k8s-check.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: Check Native Kubernetes - -on: - pull_request: - branches: - - master - - development - - k8s-new - paths: - - ".github/workflows/k8s-check.yaml" - - "MANIFEST.in" - - "development.env" - - "requirements.txt" - - "setup.py" - - "docker_images/multiarch/**" - - "seedemu/compiler/Docker.py" - - "seedemu/compiler/kubernetes.py" - - "seedemu/compiler/__init__.py" - - "seedemu/k8sTools/**" - - "examples/internet/b61_k8s_compile/**" - - "tests/k8s/**" - workflow_dispatch: - inputs: - build_images: - description: "Build and push native K8s workload images to a local registry" - type: boolean - required: false - default: true - -permissions: - contents: read - -env: - PYTHON_VERSION: "3.10" - K8S_NAMESPACE: seedemu-k8s-b61 - K8S_IMAGE_REGISTRY_PREFIX: seedemu - K8S_OUTPUT_DIR: /tmp/seedemu-k8s-output - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - k8s-static: - name: K8s Static - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out the source repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install static-check dependencies - run: python -m pip install PyYAML - - name: Validate K8s source files - run: python3 tests/k8s/validateK8sStatic.py - - k8s-compile: - name: K8s Compile - runs-on: ubuntu-latest - needs: k8s-static - timeout-minutes: 20 - steps: - - name: Check out the source repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install compile dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Compile native K8s example - run: | - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Archive native K8s compile output - uses: actions/upload-artifact@v4 - with: - name: native-k8s-output - path: ${{ env.K8S_OUTPUT_DIR }} - if-no-files-found: error - - k8s-build-images: - name: K8s Build Images - runs-on: ubuntu-latest - needs: k8s-compile - timeout-minutes: 45 - if: ${{ github.event_name == 'pull_request' || github.event.inputs.build_images == 'true' }} - steps: - - name: Check out the source repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Check Docker buildx - run: | - docker version - docker buildx version - - name: Compile native K8s example for image build - run: | - rm -rf "${K8S_OUTPUT_DIR}" - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s build output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Build and push images to local registry - run: | - bash tests/k8s/buildNativeK8sImages.sh \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --registry-prefix 127.0.0.1:5000 \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" diff --git a/.github/workflows/on-pull-request.yaml b/.github/workflows/on-pull-request.yaml index ca4885b50..ac6bc177e 100644 --- a/.github/workflows/on-pull-request.yaml +++ b/.github/workflows/on-pull-request.yaml @@ -5,47 +5,57 @@ on: branches: - master - development + - development2 + - development-ipv6 workflow_dispatch: inputs: - run-quick-tests: - description: "Run quick tests | ignores other options for testing (they still apply for compile/build)" + run-example-build: + description: "Build selected representative examples" type: boolean required: false - default: "false" - run-basic-tests: - description: "Run basic tests | Compile and builds all the basic examples - Runs basic tests" + default: false + run-runtime-integration: + description: "Run Docker runtime integration probes" type: boolean required: false - default: "true" - run-internet-tests: - description: "Run internet tests | Compile and builds all the internet examples - Runs internet tests" - type: boolean - required: false - default: "true" - run-blockchain-tests: - description: "Run blockchain tests | Compile and builds all the blockchain examples - Runs blockchain tests" - type: boolean - required: false - default: "false" - run-scion-tests: - description: "Run SCION tests | Compile and builds all the SCION examples - Runs SCION tests" - type: boolean - required: false - default: "false" + default: false + +permissions: + contents: read +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: - compile-examples: - name: Compile Examples + static: + name: Static + runs-on: ubuntu-latest + steps: + - name: Check out the source repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: "**/*requirements.txt" + - name: Install Python dependencies + run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + - name: Run static checks + run: python tests/ci/run_ci.py static --artifact-dir ci-artifacts/static + - name: Upload static artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: static-ci-artifacts + path: ci-artifacts/static + + unit: + name: Unit + needs: static runs-on: ubuntu-latest - permissions: - contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} steps: - name: Check out the source repository uses: actions/checkout@v4 @@ -53,46 +63,48 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.10" - cache: 'pip' + cache: "pip" cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ env.SCION_TESTS == 'true' }} - run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Compile Examples - run: | - source development.env - export PATH=$PATH:$PWD/bin - CMD="tests/compile-and-build-test/compile-test.py" - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" - fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD + - name: Install Python dependencies + run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + - name: Run unit checks + run: python tests/ci/run_ci.py unit --artifact-dir ci-artifacts/unit + - name: Upload unit artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: unit-ci-artifacts + path: ci-artifacts/unit + + example-compile: + name: Example Compile + needs: static + runs-on: ubuntu-latest + steps: + - name: Check out the source repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: "**/*requirements.txt" + - name: Install Python dependencies + run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + - name: Compile representative examples + run: python tests/ci/run_ci.py example-compile --artifact-dir ci-artifacts/example-compile + - name: Upload example compile artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: example-compile-ci-artifacts + path: ci-artifacts/example-compile - run-tests: - name: Run Tests - needs: compile-examples # Only run heavy tests if quick compile test succeeds + example-build: + name: Example Build + needs: example-compile + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs['run-example-build'] == 'true' }} runs-on: ubuntu-latest - permissions: - contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} steps: - name: Check out the source repository uses: actions/checkout@v4 @@ -100,59 +112,24 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.10" - cache: 'pip' + cache: "pip" cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ env.SCION_TESTS == 'true' }} - run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Run tests - run: | - source development.env - export PATH=$PATH:$PWD/bin - CMD="tests/run-tests.py" - if [ "$QUICK_TESTS" == "true" ]; then - CMD+=" --ci" - fi - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" - fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD - - name: Archive test results + - name: Install Python dependencies + run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + - name: Build selected examples + run: python tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build + - name: Upload example build artifacts + if: always() uses: actions/upload-artifact@v4 with: - name: test_result - path: | - tests/test_result.txt - tests/**/test_log - - name: Check for errors - shell: bash - run: set +e; grep -E "^score" tests/test_result.txt | grep -Eq '[1-9][0-9]* errors|[1-9][0-9]* failures'; test $? -eq 1 + name: example-build-ci-artifacts + path: ci-artifacts/example-build - build-examples: - name: Build Examples - needs: compile-examples # Only run heavy tests if quick compile test succeeds + runtime-integration: + name: Runtime Integration + needs: example-compile + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs['run-runtime-integration'] == 'true' }} runs-on: ubuntu-latest - permissions: - contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} steps: - name: Check out the source repository uses: actions/checkout@v4 @@ -160,41 +137,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.10" - cache: 'pip' + cache: "pip" cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ github.event.inputs.run-scion-tests == 'true' }} - run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Build Examples - run: | - source development.env - export PATH=$PATH:$PWD/bin - cd tests/compile-and-build-test - CMD="compile-and-build-test.py" - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" - fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD - - name: Archive test results + - name: Install Python dependencies + run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + - name: Run runtime integration probes + run: python tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration + - name: Upload runtime integration artifacts + if: always() uses: actions/upload-artifact@v4 with: - name: test_log - path: | - tests/compile-and-build-test/test_log/build_log.txt - tests/compile-and-build-test/test_log/log.txt - - name: Check for errors - shell: bash - run: set +e; grep -E "^score" tests/compile-and-build-test/test_log/log.txt | grep -Eq '[1-9][0-9]* errors|[1-9][0-9]* failures'; test $? -eq 1 + name: runtime-integration-ci-artifacts + path: ci-artifacts/runtime-integration diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 000000000..cfcdfd551 --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,36 @@ +# Feature-Oriented CI + +The pull-request workflow is intentionally organized around SEED features rather +than broad example buckets. The manifest in `feature_manifest.json` is the source +of truth for which features are covered by unit tests, representative example +compilation, selected Docker builds, and optional runtime integration probes. + +The CI runner writes both human-readable logs and machine-readable artifacts: + +- `ci-summary.json` records every command, return code, duration, and log path. +- `junit.xml` records the same stage in a format GitHub and review tooling can + ingest. +- `feature-coverage.json` records the manifest-derived coverage state, including + declared gaps such as IPv6 work that has not landed on this integration line. + +The static stage compiles importable Python source plus representative examples. +It intentionally excludes embedded payload templates under +`seedemu/services/EthereumService/EthTemplates/`, where some historical `.py` +filenames contain shell script content copied into containers. + +Run stages locally from the repository root: + +```bash +python3 tests/ci/run_ci.py static --artifact-dir ci-artifacts/static +python3 tests/ci/run_ci.py unit --artifact-dir ci-artifacts/unit +python3 tests/ci/run_ci.py example-compile --artifact-dir ci-artifacts/example-compile +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +``` + +Docker image builds and runtime integration are available as explicit entry +points, but they are not default pull-request gates in this PR: + +```bash +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +python3 tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration +``` diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 000000000..2829e92c3 --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +"""Feature-oriented CI helpers for SEED Emulator.""" diff --git a/tests/ci/feature_manifest.json b/tests/ci/feature_manifest.json new file mode 100644 index 000000000..ad33eed55 --- /dev/null +++ b/tests/ci/feature_manifest.json @@ -0,0 +1,145 @@ +{ + "schema": 1, + "features": { + "ipv4-default": { + "status": "covered", + "description": "Legacy IPv4 examples continue to compile and build without requiring any new control-plane capability.", + "unit_groups": [], + "compile_examples": ["basic-a00-simple-as"], + "build_examples": ["basic-a00-simple-as"], + "runtime_groups": [] + }, + "ipv6-core": { + "status": "declared-gap", + "description": "Repository-level IPv6 readiness is tracked here, but this integration branch does not yet carry the IPv6 readiness examples or runtime probes.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Do not mark IPv6 as covered on this branch until the IPv6 readiness work is merged into this integration line." + }, + "routing-bird-frr": { + "status": "covered", + "description": "Router-selected BIRD/FRR backends are rendered by Routing from protocol-layer intent.", + "unit_groups": ["control-plane-unit"], + "compile_examples": ["basic-a12-bgp-mixed-backend"], + "build_examples": ["basic-a12-bgp-mixed-backend"], + "runtime_groups": ["control-plane-runtime"] + }, + "exabgp-service": { + "status": "covered", + "description": "ExaBGP is exercised only as ExaBgpService plus Binding, including the IX speaker example.", + "unit_groups": ["control-plane-unit"], + "compile_examples": [ + "basic-a13-exabgp-control-plane", + "internet-b30-mini-internet-exabgp-ix" + ], + "build_examples": ["basic-a13-exabgp-control-plane"], + "runtime_groups": ["control-plane-runtime"] + }, + "looking-glass": { + "status": "covered", + "description": "Looking Glass remains a route-state observer and is tested separately from ExaBGP event streams.", + "unit_groups": ["control-plane-unit"], + "compile_examples": ["basic-a14-bgp-event-looking-glass"], + "build_examples": ["basic-a14-bgp-event-looking-glass"], + "runtime_groups": ["control-plane-runtime"] + }, + "dns-endpoint": { + "status": "covered", + "description": "DNS endpoint behavior is represented by the mini Internet plus DNS example.", + "unit_groups": [], + "compile_examples": ["internet-b02-mini-internet-with-dns"], + "build_examples": ["internet-b02-mini-internet-with-dns"], + "runtime_groups": [] + }, + "legacy-guard": { + "status": "covered", + "description": "Removed legacy control-plane APIs stay removed through explicit guard tests.", + "unit_groups": ["control-plane-unit"], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [] + } + }, + "unit_groups": { + "control-plane-unit": { + "description": "Control-plane API, rendering, service, and legacy guard checks.", + "pytest_args": ["tests/control_plane/test_bgp_control_plane_extensions.py"] + } + }, + "runtime_groups": { + "control-plane-runtime": { + "description": "Docker runtime probes for selected BIRD/FRR, ExaBGP, and Looking Glass examples.", + "pytest_args": ["-m", "integration", "tests/control_plane/test_bgp_runtime_matrix.py"], + "default": false, + "notes": "Manual workflow-dispatch entry for this PR. PR3 will make runtime evidence richer and artifact-indexed." + } + }, + "examples": { + "basic-a00-simple-as": { + "description": "Small IPv4 default behavior smoke example.", + "script": "examples/basic/A00_simple_as/simple_as.py", + "args": ["amd"], + "clean": ["output"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": true + }, + "basic-a12-bgp-mixed-backend": { + "description": "Mixed BIRD/FRR routing backend example.", + "script": "examples/basic/A12_bgp_mixed_backend/bgp_mixed_backend.py", + "args": ["amd"], + "clean": ["output"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": true + }, + "basic-a13-exabgp-control-plane": { + "description": "ExaBGP service plus Binding control-plane speaker example.", + "script": "examples/basic/A13_exabgp_control_plane/exabgp_control_plane.py", + "args": ["amd"], + "env": { + "SEED_A13_EXABGP_PORT": "5101" + }, + "clean": ["output"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": true + }, + "basic-a14-bgp-event-looking-glass": { + "description": "Looking Glass route-state observer plus separate ExaBGP event dashboard example.", + "script": "examples/basic/A14_bgp_event_looking_glass/bgp_event_looking_glass.py", + "args": ["amd"], + "env": { + "SEED_A14_LG_PORT": "5002", + "SEED_A14_EVENT_PORT": "5003" + }, + "clean": ["output"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": true + }, + "internet-b02-mini-internet-with-dns": { + "description": "Mini Internet with DNS component and local DNS endpoints.", + "script": "examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py", + "args": ["amd"], + "clean": ["output", "base_internet.bin", "dns_component.bin"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": true + }, + "internet-b30-mini-internet-exabgp-ix": { + "description": "Internet-scale ExaBGP IX speaker compile coverage.", + "script": "examples/internet/B30_mini_internet_exabgp_ix/mini_internet_exabgp_ix.py", + "args": ["amd"], + "env": { + "SEED_B30_EXABGP_PORT": "5130" + }, + "clean": ["output", "base_internet.bin"], + "expected": ["output/docker-compose.yml"], + "compile": true, + "build": false + } + } +} diff --git a/tests/ci/run_ci.py b/tests/ci/run_ci.py new file mode 100644 index 000000000..9d74d4ade --- /dev/null +++ b/tests/ci/run_ci.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import time +import traceback +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Iterable + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = Path(__file__).with_name("feature_manifest.json") + + +def _rel(path: Path) -> str: + try: + return str(path.resolve().relative_to(REPO_ROOT)) + except ValueError: + return str(path) + + +def _slug(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "check" + + +def _json_dump(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _tail(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[-limit:] + + +def load_manifest() -> dict[str, Any]: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def _validate_manifest(manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if manifest.get("schema") != 1: + errors.append("manifest schema must be 1") + + features = manifest.get("features", {}) + unit_groups = manifest.get("unit_groups", {}) + runtime_groups = manifest.get("runtime_groups", {}) + examples = manifest.get("examples", {}) + required_features = { + "ipv4-default", + "ipv6-core", + "routing-bird-frr", + "exabgp-service", + "looking-glass", + "dns-endpoint", + "legacy-guard", + } + missing = sorted(required_features.difference(features)) + if missing: + errors.append(f"missing required feature declarations: {', '.join(missing)}") + + for feature_id, feature in features.items(): + for group_id in feature.get("unit_groups", []): + if group_id not in unit_groups: + errors.append(f"feature {feature_id} references missing unit group {group_id}") + for group_id in feature.get("runtime_groups", []): + if group_id not in runtime_groups: + errors.append(f"feature {feature_id} references missing runtime group {group_id}") + for example_id in feature.get("compile_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing compile example {example_id}") + for example_id in feature.get("build_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing build example {example_id}") + + for example_id, example in examples.items(): + script = REPO_ROOT / example.get("script", "") + if not script.is_file(): + errors.append(f"example {example_id} script does not exist: {_rel(script)}") + for expected in example.get("expected", []): + if Path(expected).is_absolute(): + errors.append(f"example {example_id} expected path must be relative: {expected}") + for clean in example.get("clean", []): + if Path(clean).is_absolute(): + errors.append(f"example {example_id} clean path must be relative: {clean}") + return errors + + +def feature_coverage(manifest: dict[str, Any]) -> dict[str, Any]: + features = {} + for feature_id, feature in sorted(manifest["features"].items()): + features[feature_id] = { + "status": feature["status"], + "description": feature.get("description", ""), + "unit_groups": feature.get("unit_groups", []), + "compile_examples": feature.get("compile_examples", []), + "build_examples": feature.get("build_examples", []), + "runtime_groups": feature.get("runtime_groups", []), + "notes": feature.get("notes", ""), + } + return { + "schema": manifest["schema"], + "generated_by": "tests/ci/run_ci.py", + "features": features, + } + + +def _write_junit(stage: str, checks: list[dict[str, Any]], path: Path) -> None: + tests = len(checks) + failures = sum(1 for check in checks if check["status"] == "failed") + skipped = sum(1 for check in checks if check["status"] == "skipped") + suite = ET.Element( + "testsuite", + { + "name": f"seedemu-ci-{stage}", + "tests": str(tests), + "failures": str(failures), + "errors": "0", + "skipped": str(skipped), + "time": f"{sum(float(check.get('duration_s', 0.0)) for check in checks):.3f}", + }, + ) + for check in checks: + case = ET.SubElement( + suite, + "testcase", + { + "classname": f"seedemu.ci.{stage}", + "name": check["name"], + "time": f"{float(check.get('duration_s', 0.0)):.3f}", + }, + ) + if check["status"] == "failed": + failure = ET.SubElement( + case, + "failure", + { + "message": check.get("message", "check failed"), + "type": "CommandFailure", + }, + ) + failure.text = check.get("details", "") + elif check["status"] == "skipped": + skipped_node = ET.SubElement(case, "skipped", {"message": check.get("message", "skipped")}) + skipped_node.text = check.get("details", "") + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def _stage_result(stage: str, checks: list[dict[str, Any]], artifact_dir: Path) -> int: + summary = { + "stage": stage, + "status": "failed" if any(check["status"] == "failed" for check in checks) else "passed", + "checks": checks, + } + _json_dump(artifact_dir / "ci-summary.json", summary) + _write_junit(stage, checks, artifact_dir / "junit.xml") + + print(f"[seed-ci] stage={stage} status={summary['status']} artifact_dir={_rel(artifact_dir)}") + for check in checks: + print( + "[seed-ci] " + f"{check['status']}: {check['name']} " + f"log={check.get('log_path', '')}" + ) + if check["status"] == "failed": + command = check.get("command") + if command: + print(f"[seed-ci] failed-command: {' '.join(command)}") + details = (check.get("stderr_tail") or check.get("details") or "").strip() + if details: + print("[seed-ci] failure-tail:") + print(_tail(details, limit=1200)) + return 1 if summary["status"] == "failed" else 0 + + +def _run_command( + name: str, + cmd: list[str], + artifact_dir: Path, + *, + cwd: Path = REPO_ROOT, + env: dict[str, str] | None = None, + timeout: int | None = None, +) -> dict[str, Any]: + start = time.monotonic() + log_path = artifact_dir / "logs" / f"{_slug(name)}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + try: + result = subprocess.run( + cmd, + cwd=str(cwd), + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + duration = time.monotonic() - start + log_path.write_text( + "COMMAND: {}\nCWD: {}\nEXIT: {}\n\nSTDOUT:\n{}\n\nSTDERR:\n{}\n".format( + " ".join(cmd), + _rel(cwd), + result.returncode, + result.stdout, + result.stderr, + ), + encoding="utf-8", + ) + status = "passed" if result.returncode == 0 else "failed" + return { + "name": name, + "status": status, + "command": cmd, + "cwd": _rel(cwd), + "returncode": result.returncode, + "duration_s": duration, + "log_path": _rel(log_path), + "stdout_tail": _tail(result.stdout), + "stderr_tail": _tail(result.stderr), + "message": "command failed" if status == "failed" else "command passed", + "details": _tail(result.stdout + "\n" + result.stderr), + } + except Exception as exc: # pragma: no cover - defensive diagnostics for CI infrastructure failures. + duration = time.monotonic() - start + details = traceback.format_exc() + log_path.write_text(details, encoding="utf-8") + return { + "name": name, + "status": "failed", + "command": cmd, + "cwd": _rel(cwd), + "returncode": None, + "duration_s": duration, + "log_path": _rel(log_path), + "message": str(exc), + "details": details, + } + + +def _git_diff_check_command() -> list[str]: + base_ref = os.environ.get("GITHUB_BASE_REF") + if base_ref: + candidates = [f"origin/{base_ref}", base_ref] + for candidate in candidates: + probe = subprocess.run( + ["git", "rev-parse", "--verify", candidate], + cwd=str(REPO_ROOT), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if probe.returncode == 0: + return ["git", "diff", "--check", f"{candidate}...HEAD"] + return ["git", "diff", "--check"] + + +def _example_ids(manifest: dict[str, Any], key: str) -> list[str]: + ids: list[str] = [] + for example_id, example in manifest["examples"].items(): + if example.get(key): + ids.append(example_id) + return ids + + +def _pythonpath_env(extra: dict[str, str] | None = None) -> dict[str, str]: + env = dict(os.environ) + existing = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = str(REPO_ROOT) if not existing else f"{REPO_ROOT}:{existing}" + env.setdefault("DOCKER_BUILDKIT", "0") + env.setdefault("COMPOSE_BAKE", "false") + env.setdefault("COMPOSE_PARALLEL_LIMIT", "1") + if extra: + env.update(extra) + return env + + +def _pytest_env() -> dict[str, str]: + return _pythonpath_env({"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}) + + +def _safe_clean(example_dir: Path, relative_paths: Iterable[str]) -> None: + base = example_dir.resolve() + for item in relative_paths: + target = (example_dir / item).resolve() + if target == base or base not in target.parents: + raise ValueError(f"refusing to clean path outside example directory: {target}") + if target.is_dir(): + shutil.rmtree(target) + elif target.exists(): + target.unlink() + + +def _compile_example( + example_id: str, + example: dict[str, Any], + artifact_dir: Path, + *, + clean: bool, + name_prefix: str = "compile", +) -> dict[str, Any]: + script = REPO_ROOT / example["script"] + example_dir = script.parent + if clean: + _safe_clean(example_dir, example.get("clean", [])) + cmd = [sys.executable, script.name] + list(example.get("args", [])) + check = _run_command( + f"{name_prefix}:{example_id}", + cmd, + artifact_dir, + cwd=example_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("timeout", 900)), + ) + if check["status"] != "passed": + return check + + missing = [item for item in example.get("expected", []) if not (example_dir / item).exists()] + if missing: + check["status"] = "failed" + check["message"] = "expected compile outputs are missing" + check["details"] = "Missing outputs: " + ", ".join(missing) + return check + + +def run_static(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + + errors = _validate_manifest(manifest) + coverage = feature_coverage(manifest) + _json_dump(artifact_dir / "feature-coverage.json", coverage) + checks.append( + { + "name": "manifest", + "status": "failed" if errors else "passed", + "duration_s": 0.0, + "message": "manifest validation failed" if errors else "manifest validation passed", + "details": "\n".join(errors), + "log_path": _rel(artifact_dir / "feature-coverage.json"), + } + ) + + checks.append(_run_command("whitespace", _git_diff_check_command(), artifact_dir)) + + compile_targets = [ + "seedemu", + "tests/control_plane", + "tests/ci", + ] + example_dirs = sorted( + { + str(Path(manifest["examples"][example_id]["script"]).parent) + for example_id in _example_ids(manifest, "compile") + } + ) + compile_targets.extend(example_dirs) + checks.append( + _run_command( + "compileall", + [ + sys.executable, + "-m", + "compileall", + "-q", + "-x", + "seedemu/services/EthereumService/EthTemplates/", + ] + + compile_targets, + artifact_dir, + ) + ) + + smoke = ( + "import seedemu; " + "from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf; " + "from seedemu.services import ExaBgpService, BgpLookingGlassService; " + "from seedemu.compiler import Docker; " + "import tests.ci.run_ci" + ) + checks.append(_run_command("import-smoke", [sys.executable, "-c", smoke], artifact_dir, env=_pythonpath_env())) + return _stage_result("static", checks, artifact_dir) + + +def run_unit(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["unit_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"pytest:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=900)) + return _stage_result("unit", checks, artifact_dir) + + +def run_example_compile(artifact_dir: Path) -> int: + manifest = load_manifest() + checks = [ + _compile_example(example_id, manifest["examples"][example_id], artifact_dir, clean=True) + for example_id in _example_ids(manifest, "compile") + ] + return _stage_result("example-compile", checks, artifact_dir) + + +def run_example_build(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for example_id in _example_ids(manifest, "build"): + example = manifest["examples"][example_id] + compile_check = _compile_example( + example_id, + example, + artifact_dir, + clean=True, + name_prefix="compile-before-build", + ) + checks.append(compile_check) + if compile_check["status"] != "passed": + continue + + script = REPO_ROOT / example["script"] + compose_file = script.parent / "output" / "docker-compose.yml" + checks.append( + _run_command( + f"docker-build:{example_id}", + ["docker", "compose", "-f", str(compose_file), "build"], + artifact_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("build_timeout", 1800)), + ) + ) + return _stage_result("example-build", checks, artifact_dir) + + +def run_runtime_integration(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["runtime_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"runtime:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=7200)) + return _stage_result("runtime-integration", checks, artifact_dir) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run feature-oriented SEED Emulator CI stages.") + parser.add_argument( + "stage", + choices=["static", "unit", "example-compile", "example-build", "runtime-integration"], + ) + parser.add_argument("--artifact-dir", default="ci-artifacts", help="Directory for logs, JSON, and JUnit output.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + artifact_dir = (REPO_ROOT / args.artifact_dir).resolve() + artifact_dir.mkdir(parents=True, exist_ok=True) + if args.stage == "static": + return run_static(artifact_dir) + if args.stage == "unit": + return run_unit(artifact_dir) + if args.stage == "example-compile": + return run_example_compile(artifact_dir) + if args.stage == "example-build": + return run_example_build(artifact_dir) + if args.stage == "runtime-integration": + return run_runtime_integration(artifact_dir) + raise AssertionError(f"unknown stage: {args.stage}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/control_plane/test_bgp_control_plane_extensions.py b/tests/control_plane/test_bgp_control_plane_extensions.py new file mode 100644 index 000000000..b271c2cd0 --- /dev/null +++ b/tests/control_plane/test_bgp_control_plane_extensions.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import importlib +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.core.Node import DEFAULT_SOFTWARE +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.layers._bgp_metadata import set_bgp_backend +from seedemu.services import BgpLookingGlassService, ExaBgpService + + +def _file_content(node, path: str) -> str: + for file in node.getFiles(): + file_path, content = file.get() + if file_path == path: + return content + return "" + + +def _attach_exabgp_peer(server, router_name: str, *, router_asn: int): + for method_name in ("addPeer", "addRouterPeer", "attachToRouter"): + if not hasattr(server, method_name): + continue + method = getattr(server, method_name) + try: + return method(router_name, router_asn=router_asn) + except TypeError: + return method(router_name, router_asn) + raise AssertionError("ExaBGP server does not expose a router peer attachment API") + + +def _compiled_output_text(output_dir: Path) -> str: + chunks = [] + for path in output_dir.rglob("*"): + if path.is_file(): + chunks.append(path.read_text(encoding="utf-8", errors="replace")) + return "\n".join(chunks) + + +def test_default_node_software_uses_installable_netcat_package(): + assert "netcat-openbsd" in DEFAULT_SOFTWARE + assert "netcat" not in DEFAULT_SOFTWARE + + +def test_legacy_frr_bgp_layer_is_removed(): + with pytest.raises(ModuleNotFoundError): + importlib.import_module("seedemu.layers.FrrBgp") + assert not hasattr(importlib.import_module("seedemu.layers"), "FrrBgp") + + +def test_router_rejects_legacy_exabgp_backends(): + as2 = Base().createAutonomousSystem(2) + with pytest.raises(AssertionError, match="unsupported routing backend"): + as2.createRouter("exabgp", routingBackend="exabgp") + with pytest.raises(AssertionError, match="unsupported routing backend"): + as2.createRouter("external", routingBackend="external") + + +def test_route_server_peering_renders_from_intent_in_routing_layer(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + as150 = base.createAutonomousSystem(150) + as150.createNetwork("net0") + as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp.addRsPeer(100, 150) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + reg = emu.getRegistry() + route_server = reg.get("ix", "rs", "ix100") + router = reg.get("150", "rnode", "router0") + + rs_conf = _file_content(route_server, "/etc/bird/bird.conf") + router_conf = _file_content(router, "/etc/bird/bird.conf") + + assert "protocol bgp p_as150" in rs_conf + assert "rs client" in rs_conf + assert "neighbor 10.100.0.150 as 150" in rs_conf + assert "protocol bgp p_rs100" in router_conf + assert "neighbor 10.100.0.100 as 100" in router_conf + assert "bgp_large_community.add(PEER_COMM)" in router_conf + + +def test_frr_route_server_backend_is_explicitly_rejected(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + ix = base.createInternetExchange(100) + set_bgp_backend(ix.getRouteServerNode(), "frr") + as150 = base.createAutonomousSystem(150) + as150.createNetwork("net0") + as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp.addRsPeer(100, 150) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + + with pytest.raises(NotImplementedError, match="FRR route-server nodes are not supported yet"): + emu.render() + + +def test_frr_backend_renders_frr_config_for_selected_router(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + ebgp = Ebgp() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") + as2.createRouter("r2", routingBackend="frr").joinNetwork("net0").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(ebgp) + emu.render() + + reg = emu.getRegistry() + r1 = reg.get("2", "rnode", "r1") + r2 = reg.get("2", "rnode", "r2") + + assert any(cmd == "bird -d" for cmd, _ in r1.getStartCommands()) + assert not any(cmd == "bird -d" for cmd, _ in r2.getStartCommands()) + assert _file_content(r2, "/etc/bird/bird.conf") == "" + + frr_conf = _file_content(r2, "/etc/frr/frr.conf") + assert "router bgp 2" in frr_conf + assert "neighbor" in frr_conf + assert "RM_CONNECTED_TO_BGP" in frr_conf + assert "LC_LOCAL_OR_CUSTOMER" in frr_conf + assert "router ospf" in frr_conf + assert "interface net0" in frr_conf + + +def test_exabgp_service_renders_dashboard_and_router_peer(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as151.createHost("observer").joinNetwork("net0") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + + exabgp.install("observer_tool").attachToRouter("router0").setLocalAsn(65010).addAnnouncement("198.51.100.0/24") + emu.addBinding(Binding("observer_tool", filter=Filter(nodeName="observer", asn=151))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(exabgp) + emu.render() + + reg = emu.getRegistry() + observer = reg.get("151", "hnode", "observer") + router = reg.get("151", "rnode", "router0") + + exabgp_conf = _file_content(observer, "/etc/exabgp/exabgp.conf") + assert "198.51.100.0/24" in exabgp_conf + assert "process exabgp_json_sink" in exabgp_conf + assert "peer-as 151" in exabgp_conf + + dashboard = _file_content(observer, "/opt/exabgp/dashboard.py") + assert "/api/events" in dashboard + + bird_conf = _file_content(router, "/etc/bird/bird.conf") + assert "exabgp_65010" in bird_conf + assert "peer table t_bgp" in bird_conf + assert "bgp_large_community.add(CUSTOMER_COMM)" in bird_conf + assert "bgp_local_pref = 30" in bird_conf + + +def test_exabgp_service_renders_frr_peer_without_bird_config(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + ebgp = Ebgp() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix100") + as2.createHost("observer").joinNetwork("net0") + + exabgp.install("observer_tool").attachToRouter("router0").setLocalAsn(65010).addAnnouncement("198.51.100.0/24") + emu.addBinding(Binding("observer_tool", filter=Filter(nodeName="observer", asn=2))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(ebgp) + emu.addLayer(exabgp) + emu.render() + + reg = emu.getRegistry() + router = reg.get("2", "rnode", "router0") + observer = reg.get("2", "hnode", "observer") + + assert _file_content(router, "/etc/bird/bird.conf") == "" + frr_conf = _file_content(router, "/etc/frr/frr.conf") + assert "neighbor 10.2.0.71 remote-as 65010" in frr_conf + assert "description exabgp_65010" in frr_conf + assert "address-family ipv4 unicast" in frr_conf + + exabgp_conf = _file_content(observer, "/etc/exabgp/exabgp.conf") + assert "198.51.100.0/24" in exabgp_conf + assert "peer-as 2" in exabgp_conf + + +def test_exabgp_service_renders_multi_peer_ix_speaker_config(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + as2 = base.createAutonomousSystem(2) + as2.createRouter("r100").joinNetwork("ix100") + as3 = base.createAutonomousSystem(3) + as3.createRouter("r100").joinNetwork("ix100") + speaker_as = base.createAutonomousSystem(65030) + speaker_as.createHost("route_speaker").joinNetwork("ix100", address="10.100.0.230") + + speaker = exabgp.install("external_route_speaker") + speaker.setLocalAsn(65030).addAnnouncement("203.0.113.0/24").enableDashboard(5000) + _attach_exabgp_peer(speaker, "r100", router_asn=2) + _attach_exabgp_peer(speaker, "r100", router_asn=3) + emu.addBinding(Binding("external_route_speaker", filter=Filter(nodeName="route_speaker", asn=65030))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(exabgp) + emu.render() + + reg = emu.getRegistry() + route_speaker = reg.get("65030", "hnode", "route_speaker") + as2_router = reg.get("2", "rnode", "r100") + as3_router = reg.get("3", "rnode", "r100") + + exabgp_conf = _file_content(route_speaker, "/etc/exabgp/exabgp.conf") + assert exabgp_conf.count("neighbor ") == 2 + assert "peer-as 2" in exabgp_conf + assert "peer-as 3" in exabgp_conf + assert "203.0.113.0/24" in exabgp_conf + assert "process exabgp_json_sink" in exabgp_conf + assert _file_content(route_speaker, "/etc/bird/bird.conf") == "" + assert not any(cmd == "bird -d" for cmd, _ in route_speaker.getStartCommands()) + + as2_conf = _file_content(as2_router, "/etc/bird/bird.conf") + as3_conf = _file_content(as3_router, "/etc/bird/bird.conf") + assert "exabgp_65030" in as2_conf + assert "exabgp_65030" in as3_conf + assert "neighbor 10.100.0.230 as 65030" in as2_conf + assert "neighbor 10.100.0.230 as 65030" in as3_conf + assert "bgp_large_community.add(CUSTOMER_COMM)" in as2_conf + assert "bgp_large_community.add(CUSTOMER_COMM)" in as3_conf + assert "bgp_local_pref = 30" in as2_conf + assert "bgp_local_pref = 30" in as3_conf + + +def test_exabgp_service_renders_ix_speaker_peer(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + as2 = base.createAutonomousSystem(2) + as2.createRouter("r100").joinNetwork("ix100") + speaker_as = base.createAutonomousSystem(65030) + speaker_as.createHost("external_speaker").joinNetwork("ix100", address="10.100.0.230") + + speaker = exabgp.install("external_route_speaker") + speaker.setLocalAsn(65030).addAnnouncement("203.0.113.0/24") + _attach_exabgp_peer(speaker, "r100", router_asn=2) + emu.addBinding(Binding("external_route_speaker", filter=Filter(nodeName="external_speaker", asn=65030))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(exabgp) + emu.render() + + reg = emu.getRegistry() + external_speaker = reg.get("65030", "hnode", "external_speaker") + as2_router = reg.get("2", "rnode", "r100") + + exabgp_conf = _file_content(external_speaker, "/etc/exabgp/exabgp.conf") + assert "local-as 65030" in exabgp_conf + assert "peer-as 2" in exabgp_conf + assert "203.0.113.0/24" in exabgp_conf + assert _file_content(external_speaker, "/etc/bird/bird.conf") == "" + assert not any(cmd == "bird -d" for cmd, _ in external_speaker.getStartCommands()) + + bird_conf = _file_content(as2_router, "/etc/bird/bird.conf") + assert "exabgp_65030" in bird_conf + assert "neighbor 10.100.0.230 as 65030" in bird_conf + + +def test_frr_bgp_respects_ospf_stub_intent(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createRouter("r1", routingBackend="frr").joinNetwork("net0").joinNetwork("net1") + + ospf.markAsStub(2, "net1") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.render() + + reg = emu.getRegistry() + r1 = reg.get("2", "rnode", "r1") + frr_conf = _file_content(r1, "/etc/frr/frr.conf") + assert "interface net0" in frr_conf + assert "interface net1" in frr_conf + assert "ip ospf passive" in frr_conf + + +def test_bgp_looking_glass_supports_frr_router_route_state(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + looking_glass = BgpLookingGlassService() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0", routingBackend="frr").joinNetwork("net0") + as2.createHost("lg").joinNetwork("net0") + + looking_glass.install("bgp_lg").attach("router0") + emu.addBinding(Binding("bgp_lg", filter=Filter(nodeName="lg", asn=2))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(looking_glass) + + emu.render() + + reg = emu.getRegistry() + lg = reg.get("2", "hnode", "lg") + router = reg.get("2", "rnode", "router0") + + proxy = _file_content(router, "/opt/seed-lg/proxy.py") + frontend = _file_content(lg, "/opt/seed-lg/frontend.py") + proxy_cmds = [cmd for cmd, _ in router.getStartCommands()] + + assert "show bgp summary" in proxy + assert "show ip ospf neighbor" in proxy + assert "/api/state" in frontend + assert any("SEED_LG_BACKEND=\"frr\"" in cmd for cmd in proxy_cmds) + assert any("waiting for frr" in cmd for cmd in proxy_cmds) + assert not any("waiting for bird" in cmd for cmd in proxy_cmds) + + +def test_new_bgp_examples_compile_outputs_exist(): + repo_root = Path(__file__).resolve().parents[2] + examples = [ + repo_root / "examples" / "basic" / "A12_bgp_mixed_backend" / "bgp_mixed_backend.py", + repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "exabgp_control_plane.py", + repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "bgp_event_looking_glass.py", + ] + for script in examples: + env = dict(**os.environ) + env["PYTHONPATH"] = str(repo_root) + result = subprocess.run( + [sys.executable, str(script), "amd"], + cwd=script.parent, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + for output_dir in [ + repo_root / "examples" / "basic" / "A12_bgp_mixed_backend" / "output", + repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "output", + repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output", + ]: + assert output_dir.exists() + a13_compose = (repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "output" / "docker-compose.yml").read_text(encoding="utf-8") + a14_compose = (repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output" / "docker-compose.yml").read_text(encoding="utf-8") + assert "5001:5000/tcp" in a13_compose + assert "5002:5000/tcp" in a14_compose + assert "5003:5000/tcp" in a14_compose + + +def test_b30_mini_internet_exabgp_ix_compile_assertions(): + repo_root = Path(__file__).resolve().parents[2] + script = repo_root / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "mini_internet_exabgp_ix.py" + output_dir = script.parent / "output" + + env = dict(**os.environ) + env["PYTHONPATH"] = str(repo_root) + env["SEED_B30_EXABGP_PORT"] = "5130" + result = subprocess.run( + [sys.executable, str(script), "amd"], + cwd=script.parent, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert output_dir.exists() + + compose = (output_dir / "docker-compose.yml").read_text(encoding="utf-8") + output_text = compose + "\n" + _compiled_output_text(output_dir) + assert "5130:5000/tcp" in compose + assert "203.0.113.0/24" in output_text + assert "peer-as 2" in output_text + assert "peer-as 3" in output_text + assert "neighbor 10.100.0.180 as 180" in output_text + assert "bgp_large_community.add(CUSTOMER_COMM)" in output_text + assert "bgp_local_pref = 30" in output_text + assert "process exabgp_json_sink" in output_text + assert "dashboard.py" in output_text + + +def test_b30_mini_internet_exabgp_ix_emulator_compiles_from_builder(): + from examples.internet.B30_mini_internet_exabgp_ix import mini_internet_exabgp_ix + + emu = mini_internet_exabgp_ix.build_emulator() + emu.render() + output_dir = Path("/tmp/seedemu-b30-static-compile") + emu.compile(Docker(selfManagedNetwork=True, platform=Platform.AMD64), str(output_dir), override=True) + + output_text = _compiled_output_text(output_dir) + assert "203.0.113.0/24" in output_text + assert "peer-as 2" in output_text + assert "peer-as 3" in output_text diff --git a/tests/control_plane/test_bgp_runtime_matrix.py b/tests/control_plane/test_bgp_runtime_matrix.py new file mode 100644 index 000000000..ec8def2a7 --- /dev/null +++ b/tests/control_plane/test_bgp_runtime_matrix.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +import re + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PYTHON = Path(os.environ.get("PYTHON", sys.executable)) + + +def _run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=str(cwd) if cwd is not None else str(REPO_ROOT), + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + + +def _must_run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]: + result = _run(cmd, cwd=cwd, env=env, timeout=timeout) + assert result.returncode == 0, ( + f"command failed: {' '.join(cmd)}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + return result + + +def _classic_docker_build_env() -> dict[str, str]: + env = dict(os.environ) + env["COMPOSE_BAKE"] = "false" + env["DOCKER_BUILDKIT"] = "0" + env["COMPOSE_PARALLEL_LIMIT"] = "1" + return env + + +def _docker_exec(container: str, shell_cmd: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]: + return _must_run(["docker", "exec", container, "sh", "-lc", shell_cmd], timeout=timeout) + + +def _docker_exec_maybe(container: str, shell_cmd: str, *, timeout: int = 120) -> str | None: + result = _run(["docker", "exec", container, "sh", "-lc", shell_cmd], timeout=timeout) + if result.returncode != 0: + return None + return result.stdout + + +def _docker_ps_names() -> list[str]: + result = _must_run(["docker", "ps", "--format", "{{.Names}}"]) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def _wait_container_name(predicate, *, timeout_s: int = 90, error_message: str) -> str: + def _probe(): + for name in _docker_ps_names(): + if predicate(name): + return name + return None + + return _wait_until(_probe, timeout_s=timeout_s, interval_s=2, error_message=error_message) + + +def _wait_http_ok(url: str, *, timeout_s: int = 60) -> None: + deadline = time.time() + timeout_s + while time.time() < deadline: + result = _run(["curl", "--noproxy", "*", "-fsS", "-o", "/dev/null", "-w", "%{http_code}", url], timeout=5) + if result.returncode == 0 and result.stdout.strip() == "200": + return + time.sleep(1) + raise AssertionError(f"HTTP endpoint did not become ready: {url}") + + +def _wait_until(predicate, *, timeout_s: int = 90, interval_s: int = 2, error_message: str = "condition not met"): + deadline = time.time() + timeout_s + last_value = None + while time.time() < deadline: + last_value = predicate() + if last_value: + return last_value + time.sleep(interval_s) + raise AssertionError(f"{error_message}\nlast_value={last_value}") + + +def _wait_docker_output( + container: str, + shell_cmd: str, + *, + contains: list[str], + timeout_s: int = 90, + interval_s: int = 2, + error_message: str, +) -> str: + def _probe(): + text = _docker_exec_maybe(container, shell_cmd) + if text is None: + return None + return text if all(needle in text for needle in contains) else None + + return _wait_until(_probe, timeout_s=timeout_s, interval_s=interval_s, error_message=error_message) + + +def _wait_command_output( + cmd: list[str], + *, + contains: list[str], + timeout_s: int = 90, + interval_s: int = 2, + error_message: str, +) -> str: + def _probe(): + result = _run(cmd) + if result.returncode != 0: + return None + return result.stdout if all(needle in result.stdout for needle in contains) else None + + return _wait_until(_probe, timeout_s=timeout_s, interval_s=interval_s, error_message=error_message) + + +def _assert_runtime_clear() -> None: + result = _must_run(["docker", "ps", "--format", "{{.Names}}"]) + offenders = [ + line.strip() + for line in result.stdout.splitlines() + if line.strip().startswith("as") or line.strip() == "seedemu_internet_map" + ] + assert not offenders, f"runtime validation requires a clean emulator environment, found: {offenders}" + + +def _assert_bridge_nf_disabled() -> None: + result = _must_run( + ["sysctl", "net.bridge.bridge-nf-call-iptables", "net.bridge.bridge-nf-call-ip6tables", "net.bridge.bridge-nf-call-arptables"] + ) + values = {} + for line in result.stdout.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + values[key.strip()] = value.strip() + blocking = {key: value for key, value in values.items() if value != "0"} + assert not blocking, f"bridge netfilter must be disabled before runtime validation:\n{result.stdout}" + + +def _compose_down(compose_file: Path) -> None: + _run(["docker", "compose", "-f", str(compose_file), "down", "--remove-orphans"], timeout=300) + + +def _map_url_from_compose(compose_file: Path) -> str: + text = compose_file.read_text(encoding="utf-8") + match = re.search(r"-\s*(?:\$\{SEED_DEMO_MAP_PORT:-)?([0-9]+)(?:\})?:8080/tcp", text) + host_port = match.group(1) if match else "8080" + return f"http://127.0.0.1:{host_port}/pro/home" + + +@pytest.mark.integration +def test_runtime_a12_mixed_backend_fresh_build(): + _assert_runtime_clear() + _assert_bridge_nf_disabled() + + script = REPO_ROOT / "examples" / "basic" / "A12_bgp_mixed_backend" / "bgp_mixed_backend.py" + compose = REPO_ROOT / "examples" / "basic" / "A12_bgp_mixed_backend" / "output" / "docker-compose.yml" + + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO_ROOT) + env["SEED_DEMO_MAP_PORT"] = "18080" + + try: + _must_run([str(PYTHON), str(script), "amd"], env=env) + _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) + _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) + + ospf = _wait_docker_output( + "as2brd-r2-10.2.0.253", + 'vtysh -c "show ip ospf neighbor"', + contains=["Full"], + error_message="A12 OSPF neighbor did not become visible", + ) + bgp = _wait_docker_output( + "as2brd-r2-10.2.0.253", + 'vtysh -c "show ip bgp summary"', + contains=["10.101.0.152", " 1"], + error_message="A12 BGP summary did not become available", + ) + host_ping = _wait_command_output( + ["docker", "exec", "as151h-web-10.151.0.71", "sh", "-lc", "ping -c 2 -W 1 10.152.0.71"], + contains=["0% packet loss"], + error_message="A12 host-to-host reachability did not converge", + ) + router_ping = _wait_command_output( + ["docker", "exec", "as151brd-router0-10.151.0.254", "sh", "-lc", "ping -c 2 -W 1 10.152.0.254"], + contains=["0% packet loss"], + error_message="A12 router-to-router reachability did not converge", + ) + route = _wait_docker_output( + "as152brd-router0-10.152.0.254", + "birdc show route 10.152.0.0/24 all", + contains=["local_nets", "BGP.large_community: (152, 0, 0)"], + error_message="A12 local route export did not appear", + ) + _wait_http_ok(_map_url_from_compose(compose)) + finally: + _compose_down(compose) + + +@pytest.mark.integration +def test_runtime_a13_exabgp_fresh_build(): + _assert_runtime_clear() + _assert_bridge_nf_disabled() + + script = REPO_ROOT / "examples" / "basic" / "A13_exabgp_control_plane" / "exabgp_control_plane.py" + compose = REPO_ROOT / "examples" / "basic" / "A13_exabgp_control_plane" / "output" / "docker-compose.yml" + + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO_ROOT) + env["SEED_DEMO_MAP_PORT"] = "18080" + env["SEED_A13_EXABGP_PORT"] = "5101" + + try: + _must_run([str(PYTHON), str(script), "amd"], env=env) + _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) + _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) + + _wait_http_ok(_map_url_from_compose(compose)) + _wait_http_ok("http://127.0.0.1:5101/") + procs = _docker_exec( + "as151h-ExaBGP_Control_Plane_Tool-10.151.0.71", + 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep', + ).stdout + assert "dashboard.py" in procs and "exabgp" in procs and "event_sink.py" in procs + route = _wait_docker_output( + "as151brd-router0-10.151.0.254", + "birdc show route 198.51.100.0/24 all", + contains=["198.51.100.0/24", "AS65010"], + error_message="A13 ExaBGP route did not appear on Bird peer", + ) + finally: + _compose_down(compose) + + +@pytest.mark.integration +def test_runtime_a14_observability_fresh_build(): + _assert_runtime_clear() + _assert_bridge_nf_disabled() + + script = REPO_ROOT / "examples" / "basic" / "A14_bgp_event_looking_glass" / "bgp_event_looking_glass.py" + compose = REPO_ROOT / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output" / "docker-compose.yml" + + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO_ROOT) + env["SEED_DEMO_MAP_PORT"] = "18080" + env["SEED_A14_LG_PORT"] = "5002" + env["SEED_A14_EVENT_PORT"] = "5003" + + try: + _must_run([str(PYTHON), str(script), "amd"], env=env) + _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) + _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) + + _wait_http_ok(_map_url_from_compose(compose)) + _wait_http_ok("http://127.0.0.1:5002/summary/router0") + _wait_http_ok("http://127.0.0.1:5003/") + lg = _docker_exec("as2h-looking-glass-10.2.0.71", 'ps -ef | egrep "frontend|proxy" | grep -v grep').stdout + assert "frontend" in lg + exa = _docker_exec("as151h-ExaBGP_Control_Plane_Tool-10.151.0.71", 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep').stdout + assert "dashboard.py" in exa and "exabgp" in exa and "event_sink.py" in exa + lg_ping = _docker_exec("as2h-looking-glass-10.2.0.71", "ping -c 2 -W 1 10.2.0.254").stdout + assert "0% packet loss" in lg_ping + finally: + _compose_down(compose) + + +@pytest.mark.integration +def test_runtime_exabgp_to_frr_peer(): + _assert_runtime_clear() + _assert_bridge_nf_disabled() + + out_dir = Path(tempfile.mkdtemp(prefix="frr-exabgp-runtime-", dir="/tmp")) + compose = out_dir / "docker-compose.yml" + + source = """ +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Routing +from seedemu.services import ExaBgpService +from seedemu.compiler import Docker, Platform + +emu = Emulator() +base = Base() +routing = Routing() +ebgp = Ebgp() +exabgp = ExaBgpService() + +base.createInternetExchange(100) +as151 = base.createAutonomousSystem(151) +as151.createNetwork('net0') +as151.createRouter('router0', routingBackend='frr').joinNetwork('net0').joinNetwork('ix100') +as151.createHost('observer').joinNetwork('net0').addPortForwarding(5105, 5000) + +exabgp.install('observer_tool').attachToRouter('router0').setLocalAsn(65010).addAnnouncement('198.51.100.0/24').enableDashboard(5000) +emu.addBinding(Binding('observer_tool', filter=Filter(nodeName='observer', asn=151))) + +emu.addLayer(base) +emu.addLayer(routing) +emu.addLayer(ebgp) +emu.addLayer(exabgp) +emu.render() +emu.compile(Docker(platform=Platform.AMD64, internetMapEnabled=False), OUTPUT_DIR, override=True) +""" + script = out_dir / "generate.py" + script.write_text(source.replace("OUTPUT_DIR", repr(str(out_dir))), encoding="utf-8") + + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO_ROOT) + + try: + _must_run([str(PYTHON), str(script)], env=env) + _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) + _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) + + _wait_http_ok("http://127.0.0.1:5105/") + summary = _wait_docker_output( + "as151brd-router0-10.151.0.254", + 'vtysh -c "show ip bgp summary"', + contains=["65010", " 1"], + error_message="FRR peer summary did not converge", + ) + route = _wait_docker_output( + "as151brd-router0-10.151.0.254", + 'vtysh -c "show ip bgp 198.51.100.0/24"', + contains=["198.51.100.0/24", "valid"], + error_message="FRR peer route did not appear", + ) + finally: + _compose_down(compose) + + +@pytest.mark.integration +def test_runtime_b30_mini_internet_exabgp_ix_fresh_build(): + _assert_runtime_clear() + _assert_bridge_nf_disabled() + + script = REPO_ROOT / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "mini_internet_exabgp_ix.py" + compose = REPO_ROOT / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "output" / "docker-compose.yml" + + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO_ROOT) + env["SEED_DEMO_MAP_PORT"] = "18080" + env["SEED_B30_EXABGP_PORT"] = "5130" + + try: + _must_run([str(PYTHON), str(script), "amd"], env=env, timeout=900) + _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) + _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) + + _wait_http_ok(_map_url_from_compose(compose), timeout_s=120) + _wait_http_ok("http://127.0.0.1:5130/", timeout_s=120) + + exabgp_container = _wait_container_name( + lambda name: "ExaBGP_Control_Plane_Tool" in name or "external" in name.lower() and "speaker" in name.lower(), + error_message="B30 ExaBGP external speaker container did not start", + ) + as2_peer = _wait_container_name( + lambda name: name.startswith("as2brd-r100-"), + error_message="B30 AS2 r100 peer container did not start", + ) + as3_peer = _wait_container_name( + lambda name: name.startswith("as3brd-r100-"), + error_message="B30 AS3 r100 peer container did not start", + ) + + procs = _docker_exec( + exabgp_container, + 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep', + ).stdout + assert "dashboard.py" in procs and "exabgp" in procs and "event_sink.py" in procs + + version = _docker_exec(exabgp_container, "exabgp --version 2>&1").stdout + assert "ExaBGP" in version or "exabgp" in version.lower() + + config = _docker_exec(exabgp_container, "cat /etc/exabgp/exabgp.conf").stdout + assert "peer-as 2" in config + assert "peer-as 3" in config + assert "203.0.113.0/24" in config + assert config.count("neighbor ") >= 2 + + _wait_docker_output( + as2_peer, + "birdc show route 203.0.113.0/24 all", + contains=["203.0.113.0/24"], + timeout_s=180, + error_message="B30 AS2 peer did not learn the ExaBGP IX tool route", + ) + _wait_docker_output( + as3_peer, + "birdc show route 203.0.113.0/24 all", + contains=["203.0.113.0/24"], + timeout_s=180, + error_message="B30 AS3 peer did not learn the ExaBGP IX tool route", + ) + + event_log = _docker_exec(exabgp_container, "test -r /var/log/exabgp/events.jsonl && wc -l /var/log/exabgp/events.jsonl").stdout + assert "/var/log/exabgp/events.jsonl" in event_log + finally: + _compose_down(compose) diff --git a/tests/k8s/PHASE2_KVM_E2E.md b/tests/k8s/PHASE2_KVM_E2E.md deleted file mode 100644 index 5f1cfc7b5..000000000 --- a/tests/k8s/PHASE2_KVM_E2E.md +++ /dev/null @@ -1,154 +0,0 @@ -# Phase 2: KVM + OVN + OVS E2E PR Check - -This document records the intended second-stage native Kubernetes PR check. It -is not implemented yet. The first-stage workflow only compiles manifests and -builds workload images on GitHub-hosted runners. - -## Goal - -Phase 2 should prove that the generated B61 native Kubernetes example can run -end to end on a KVM-backed K3s cluster using Kube-OVN and OVS. - -The check should cover: - -- KVM VM creation from `kvm.yaml`. -- K3s installation on the generated master and worker VMs. -- Kube-OVN/OVS fabric installation and validation. -- Native SeedEMU image build and push to the cluster registry. -- Manifest deployment with `make up`. -- Runtime validation that deployments and pods become ready. -- Cleanup with `make clean` and VM destruction. - -Real physical-machine E2E is intentionally out of scope because it is expensive -and needs lab-specific hosts and secrets. - -## Runner Requirements - -The job must run on a dedicated self-hosted GitHub Actions runner, not on -`ubuntu-latest`. - -Required runner labels: - -```yaml -runs-on: [self-hosted, linux, x64, kvm, seedemu-k8s] -``` - -Required host capabilities: - -- `/dev/kvm` is available. -- libvirt and `virsh` are installed and usable by the runner. -- the default libvirt network or configured bridge is available. -- Docker with buildx is available. -- enough CPU, memory, and disk exist for the B61 KVM topology. -- outbound network access is available for image and package downloads. - -The runner should be isolated from long-running user clusters. If the same host -must be shared, VM names, libvirt storage pools, namespace names, registry ports, -and working directories must be unique to the CI run. - -## Trigger Conditions - -The workflow should appear on PRs but only run automatically for PRs from this -repository, not from forks. - -Recommended trigger: - -```yaml -on: - pull_request: - branches: - - master - - development - paths: - - "seedemu/compiler/kubernetes.py" - - "seedemu/compiler/__init__.py" - - "seedemu/k8sTools/**" - - "examples/internet/b61_k8s_compile/**" - - "tests/k8s/**" - - ".github/workflows/k8s-*.yaml" - - "setup.py" - - "MANIFEST.in" - workflow_dispatch: -``` - -Recommended job guard: - -```yaml -if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository -``` - -Recommended concurrency: - -```yaml -concurrency: - group: seedemu-k8s-kvm-e2e - cancel-in-progress: false -``` - -The KVM E2E check should start as optional. After repeated stable runs, the -repository can decide whether to mark it as required for K8s-related PRs. - -## Expected Flow - -The job should use a fresh working directory and generate all stage artifacts -from committed files: - -```bash -source development.env -python examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir examples/internet/b61_k8s_compile/output - -cd examples/internet/b61_k8s_compile -python ./k8sTools.py build \ - --input configKvmOvn.yaml \ - --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml -python ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml -python ./k8sTools.py down -f ./output -k kubeconfig.yaml -python ./k8sTools.py destroy -d configK3s.yaml -``` - -The final implementation should use CI-specific names and directories so that -parallel or failed runs do not collide with developer experiments. - -## Cleanup Contract - -Cleanup must run with `if: always()` or shell traps. - -Required cleanup steps: - -- `make clean || true` for the SeedEMU namespace. -- `destroyKvmVms.sh || true` for all VMs created by the job. -- remove any temporary local registry container used by the job. -- collect diagnostics before destruction when a failure occurs. - -Do not run broad destructive commands such as `docker system prune` on a shared -self-hosted runner unless the runner is dedicated to this workflow. - -## Failure Artifacts - -Upload these artifacts on failure: - -- compiled `k8s.kube-ovn.yaml` and `images.yaml`. -- generated `setup/*.yaml` and `running/configRunning.yaml`. -- K3s kubeconfig and inventory if they do not contain secrets. -- `kubectl get pods -A -o wide`. -- `kubectl describe pods -n seedemu-k8s-b61`. -- recent Kubernetes events. -- Kube-OVN pod status and relevant logs. -- `virsh list --all`. -- generated `kvmState.yaml`. - -Secrets such as private SSH keys must never be uploaded. - -## Success Criteria - -The Phase 2 job should pass only when: - -- all VMs are created and reachable; -- K3s nodes are Ready; -- Kube-OVN validation passes; -- all images from `images.yaml` build and push to the registry; -- `make up` completes and all SeedEMU workloads become Ready; -- `make clean` succeeds; -- `destroyKvmVms.sh` succeeds or confirms no CI-owned VMs remain. diff --git a/tests/k8s/README.md b/tests/k8s/README.md deleted file mode 100644 index 956c53431..000000000 --- a/tests/k8s/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Native Kubernetes PR Checks - -This directory contains the first-stage GitHub PR checks for the -`seedemu.k8sTools` native Kubernetes workflow. - -## PR Workflow - -`.github/workflows/k8s-check.yaml` runs automatically on PRs that touch native -K8s compiler, k8sTools, B61 example, packaging, Docker image sources, or these -test helpers. - -The workflow has three jobs: - -- `K8s Static`: compiles Python sources, runs `bash -n` on shell resources, - parses YAML files, and rejects checked-in host-local absolute paths. -- `K8s Compile`: compiles the B61 native K8s example and validates generated - Kube-OVN manifests plus `images.yaml`. -- `K8s Build Images`: builds every image listed in `images.yaml`, pushes to a - temporary local registry, then pulls the pushed images back for verification. - This job recompiles the B61 output in its own runner so generated - `base_images/` contexts are present before Docker builds start. - -These jobs do not create KVM VMs, install K3s, or apply Kubernetes resources. -The destructive KVM/K3s E2E plan is documented separately in -`PHASE2_KVM_E2E.md`. - -## Local Reproduction - -Run the non-destructive checks: - -```bash -bash tests/k8s/runNativeK8sPrCheck.sh -``` - -Also build and push workload images to a temporary local registry: - -```bash -bash tests/k8s/runNativeK8sPrCheck.sh --build-images -``` diff --git a/tests/k8s/buildNativeK8sImages.sh b/tests/k8s/buildNativeK8sImages.sh deleted file mode 100755 index 32310bdfb..000000000 --- a/tests/k8s/buildNativeK8sImages.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Build native SeedEMU Kubernetes workload images for PR checks. -# -# Inputs: -# - --output-dir: compile output containing k8s.kube-ovn.yaml, images.yaml, -# base_images/, and per-node Docker build contexts. -# - --registry-prefix: local registry host:port used for image pushes. -# - --image-registry-prefix: logical compiler image prefix to rewrite. -# -# Generated outputs: -# - Docker images tagged under the selected registry prefix. -# -# Side effects: -# - starts a temporary local Docker registry when the registry host is -# 127.0.0.1 or localhost; -# - builds images with docker buildx; -# - pushes workload images to the local registry; -# - removes the temporary registry before exit. -# -# Expected context: -# - GitHub-hosted Ubuntu runner or local developer machine with Docker, -# buildx, Python 3, and PyYAML. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -OUTPUT_DIR="" -REGISTRY_PREFIX="127.0.0.1:5000" -IMAGE_REGISTRY_PREFIX="seedemu" -REGISTRY_NAME="seedemu-native-k8s-ci-registry" -STARTED_REGISTRY="" -ORIGINAL_BUILDER="" -if command -v docker >/dev/null 2>&1; then - ORIGINAL_BUILDER="$( - docker buildx ls 2>/dev/null | - awk 'NR > 1 && $1 ~ /\*$/ {gsub(/\*/, "", $1); builder=$1} END {print builder}' || - true - )" -fi - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - echo "--output-dir is required" >&2 - usage >&2 - exit 1 -fi - -OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" -RUNNING_DIR="${REPO_ROOT}/seedemu/k8sTools/resources/running" -MANIFEST_HELPER="${RUNNING_DIR}/manageK8sManifest.py" -BUILD_SCRIPT="${RUNNING_DIR}/buildRegistryImages.py" -IMAGES_YAML="${OUTPUT_DIR}/images.yaml" - -cleanup() { - if [ -n "${STARTED_REGISTRY}" ]; then - docker rm -f "${STARTED_REGISTRY}" >/dev/null 2>&1 || true - fi - if [ -n "${ORIGINAL_BUILDER}" ]; then - docker buildx use "${ORIGINAL_BUILDER}" >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -requireFile() { - # Args: - # $1: file path that must exist. - local path="$1" - if [ ! -f "${path}" ]; then - echo "Required file not found: ${path}" >&2 - exit 1 - fi -} - -startLocalRegistryIfNeeded() { - # Args: - # $1: registry prefix in host:port form. - local prefix="$1" - local host="${prefix%%:*}" - local port="${prefix##*:}" - if [ "${host}" != "127.0.0.1" ] && [ "${host}" != "localhost" ]; then - return 0 - fi - if [ "${port}" = "${prefix}" ]; then - echo "Local registry prefix must include a port: ${prefix}" >&2 - exit 1 - fi - docker rm -f "${REGISTRY_NAME}" >/dev/null 2>&1 || true - docker run -d --name "${REGISTRY_NAME}" -p "${port}:5000" registry:2 >/dev/null - STARTED_REGISTRY="${REGISTRY_NAME}" -} - -selectDockerDriverBuilder() { - # Use a docker driver builder so --load images are visible as local FROM - # dependencies to later builds in the same check. - local default_driver - default_driver="$(docker buildx inspect default 2>/dev/null | awk -F: '$1 == "Driver" {driver=$2} END {gsub(/^[ \t]+|[ \t]+$/, "", driver); print driver}')" - if [ "${default_driver}" != "docker" ]; then - echo "Docker buildx default builder must use the docker driver for native K8s image checks." >&2 - exit 1 - fi - docker buildx use default >/dev/null -} - -verifyPushedImages() { - # Args: - # $1: images.yaml path. - # $2: logical compiler image prefix. - # $3: pushed registry prefix. - local images_yaml="$1" - local image_registry_prefix="$2" - local registry_prefix="$3" - python3 "${MANIFEST_HELPER}" mapped-images \ - --images-yaml "${images_yaml}" \ - --image-registry-prefix "${image_registry_prefix}" \ - --registry-prefix "${registry_prefix}" | - while IFS=$'\t' read -r image _context; do - [ -n "${image}" ] || continue - echo "+ docker pull ${image}" - docker pull "${image}" >/dev/null - done -} - -requireFile "${IMAGES_YAML}" -requireFile "${MANIFEST_HELPER}" -requireFile "${BUILD_SCRIPT}" - -docker info >/dev/null -docker buildx version >/dev/null - -selectDockerDriverBuilder -startLocalRegistryIfNeeded "${REGISTRY_PREFIX}" - -python3 "${BUILD_SCRIPT}" \ - --output-dir "${OUTPUT_DIR}" \ - --registry-prefix "${REGISTRY_PREFIX}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" - -verifyPushedImages "${IMAGES_YAML}" "${IMAGE_REGISTRY_PREFIX}" "${REGISTRY_PREFIX}" - -echo "Native K8s image build check completed for ${OUTPUT_DIR}." diff --git a/tests/k8s/runNativeK8sPrCheck.sh b/tests/k8s/runNativeK8sPrCheck.sh deleted file mode 100755 index 5c5a39527..000000000 --- a/tests/k8s/runNativeK8sPrCheck.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# Run the non-destructive native Kubernetes PR checks locally. -# -# Inputs: -# - optional --output-dir path for compiler output. -# - optional --build-images to also build/push images to a local registry. -# - optional --registry-prefix host:port for the image build check. -# -# Generated outputs: -# - compiler output under the selected output directory. -# -# Side effects: -# - without --build-images: none outside the output directory. -# - with --build-images: starts a temporary local Docker registry when the -# registry prefix points at localhost, builds images, pushes them, and removes -# the registry container before exit. -# -# Expected context: -# - repository checkout with Python 3.10+, PyYAML, and compile dependencies. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -OUTPUT_DIR="" -BUILD_IMAGES="false" -REGISTRY_PREFIX="127.0.0.1:5000" -IMAGE_REGISTRY_PREFIX="seedemu" -NAMESPACE="seedemu-k8s-b61" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - OUTPUT_DIR="$(mktemp -d)/seedemu-k8s-output" -fi - -cd "${REPO_ROOT}" -source development.env - -python3 tests/k8s/validateK8sStatic.py -python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${OUTPUT_DIR}" -python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${OUTPUT_DIR}" \ - --expected-namespace "${NAMESPACE}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" - -if [ "${BUILD_IMAGES}" = "true" ]; then - bash tests/k8s/buildNativeK8sImages.sh \ - --output-dir "${OUTPUT_DIR}" \ - --registry-prefix "${REGISTRY_PREFIX}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" -fi - -echo "Native K8s PR checks completed for ${OUTPUT_DIR}." diff --git a/tests/k8s/validateK8sStatic.py b/tests/k8s/validateK8sStatic.py deleted file mode 100755 index 8db8f81f6..000000000 --- a/tests/k8s/validateK8sStatic.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python3 -"""Validate SeedEMU native Kubernetes source files before PR build checks. - -Inputs: -- repository source tree, defaulting to the current checkout. - -Outputs: -- console diagnostics only. - -Side effects: -- none; Python files are compiled to syntax trees in memory and shell scripts - are checked with bash -n. - -Expected context: -- GitHub-hosted or local developer machine with Python 3.10+ and bash. -""" -from __future__ import annotations - -import argparse -import py_compile -import subprocess -import sys -from pathlib import Path - -import yaml - - -SOURCE_ROOTS = ( - Path(".github/workflows/k8s-check.yaml"), - Path("MANIFEST.in"), - Path("setup.py"), - Path("seedemu/compiler/kubernetes.py"), - Path("seedemu/compiler/__init__.py"), - Path("seedemu/k8sTools"), - Path("examples/internet/b61_k8s_compile"), - Path("tests/k8s"), -) -GENERATED_EXAMPLE_PREFIXES = ( - "examples/internet/b61_k8s_compile/output/", - "examples/internet/b61_k8s_compile/configK3s.yaml", - "examples/internet/b61_k8s_compile/configK3sTemplate.yaml", - "examples/internet/b61_k8s_compile/kubeconfig.yaml", -) -TEXT_SUFFIXES = {".py", ".sh", ".yaml", ".yml", ".md", ".in"} -FORBIDDEN_TEXT = (str(Path("/home") / "lxl") + "/", str(Path("/home") / "lxl")) - - -def parseArgs() -> argparse.Namespace: - """Parse command-line options.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--repo-root", - type=Path, - default=Path(__file__).resolve().parents[2], - help="SeedEMU repository root.", - ) - return parser.parse_args() - - -def relativePath(path: Path, repo_root: Path) -> str: - """Return a POSIX relative path used for matching and diagnostics. - - Args: - path: Absolute or repository-relative path. - repo_root: Repository root directory. - """ - return path.resolve().relative_to(repo_root.resolve()).as_posix() - - -def isGeneratedExamplePath(path: Path, repo_root: Path) -> bool: - """Return true for generated B61 setup/running/output artifacts. - - Args: - path: Source file candidate. - repo_root: Repository root directory. - """ - rel = relativePath(path, repo_root) - return any(rel.startswith(prefix) for prefix in GENERATED_EXAMPLE_PREFIXES) - - -def isIgnoredPath(path: Path, repo_root: Path) -> bool: - """Return true for cache/build files that should not be source-checked. - - Args: - path: Source file candidate. - repo_root: Repository root directory. - """ - parts = path.relative_to(repo_root).parts - if "__pycache__" in parts: - return True - if path.name == "codex_worklog.md": - return True - return isGeneratedExamplePath(path, repo_root) - - -def iterSourceFiles(repo_root: Path) -> list[Path]: - """Return source files under K8s-related paths. - - Args: - repo_root: Repository root directory. - """ - files: list[Path] = [] - for root in SOURCE_ROOTS: - candidate = repo_root / root - if not candidate.exists(): - continue - if candidate.is_file(): - files.append(candidate) - continue - files.extend(path for path in candidate.rglob("*") if path.is_file()) - return sorted(path for path in files if not isIgnoredPath(path, repo_root)) - - -def validatePythonSyntax(paths: list[Path]) -> None: - """Compile Python source files without importing them. - - Args: - paths: Python files to validate. - """ - for path in paths: - py_compile.compile(str(path), doraise=True) - - -def validateShellSyntax(paths: list[Path]) -> None: - """Run bash -n on shell scripts. - - Args: - paths: Shell script paths to validate. - """ - for path in paths: - subprocess.run(["bash", "-n", str(path)], check=True) - - -def validateYamlSyntax(paths: list[Path]) -> None: - """Parse YAML files without applying YAML 1.1 scalar coercions. - - Args: - paths: YAML files to validate. - """ - for path in paths: - list(yaml.compose_all(path.read_text(encoding="utf-8"))) - - -def validateNoHostLocalArtifacts(paths: list[Path], repo_root: Path) -> None: - """Reject known host-local paths and checked-in image tarballs. - - Args: - paths: K8s-related source files. - repo_root: Repository root directory. - """ - failures: list[str] = [] - for path in paths: - rel = relativePath(path, repo_root) - if path.suffix == ".tar": - failures.append(f"{rel}: image tarball must not be checked in") - continue - if path.suffix not in TEXT_SUFFIXES and path.name not in {"Makefile"}: - continue - try: - text = path.read_text(encoding="utf-8") - except UnicodeDecodeError: - continue - for token in FORBIDDEN_TEXT: - if token in text: - failures.append(f"{rel}: contains host-local path {token}") - if failures: - raise SystemExit("\n".join(failures)) - - -def main() -> int: - """Run static validation for native K8s sources.""" - args = parseArgs() - repo_root = args.repo_root.expanduser().resolve() - files = iterSourceFiles(repo_root) - python_files = [path for path in files if path.suffix == ".py"] - shell_files = [path for path in files if path.suffix == ".sh"] - yaml_files = [path for path in files if path.suffix in {".yaml", ".yml"}] - - validatePythonSyntax(python_files) - validateShellSyntax(shell_files) - validateYamlSyntax(yaml_files) - validateNoHostLocalArtifacts(files, repo_root) - - print( - "Validated " - f"{len(python_files)} Python files, " - f"{len(shell_files)} shell scripts, " - f"and {len(yaml_files)} YAML files." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/k8s/validateNativeK8sOutput.py b/tests/k8s/validateNativeK8sOutput.py deleted file mode 100755 index 2ec955d0c..000000000 --- a/tests/k8s/validateNativeK8sOutput.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env python3 -"""Validate native Kubernetes compiler output used by PR checks. - -Inputs: -- an output directory generated by mini_internet_k8s.py. - -Outputs: -- console diagnostics only. - -Side effects: -- none; this script only parses YAML, JSON, and Docker build contexts. - -Expected context: -- GitHub-hosted or local developer machine with Python 3.10+ and PyYAML. -""" -from __future__ import annotations - -import argparse -import ipaddress -import json -import re -import sys -from pathlib import Path -from typing import Any - -import yaml - - -MANIFEST_NAME = "k8s.kube-ovn.yaml" -IMAGES_NAME = "images.yaml" -LEGACY_MANIFEST_NAME = "k8s.yaml" -FORBIDDEN_TEXT = (str(Path("/home") / "lxl") + "/", str(Path("/home") / "lxl")) -COMPILER_BASE_TAG_RE = re.compile(r"^[0-9a-f]{32}(?::latest)?$") - - -def parseArgs() -> argparse.Namespace: - """Parse command-line options.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output-dir", type=Path, required=True, help="Compiler output directory.") - parser.add_argument("--expected-namespace", required=True, help="Expected Kubernetes namespace name.") - parser.add_argument( - "--image-registry-prefix", - default="seedemu", - help="Logical image prefix written by NativeKubernetesCompiler.", - ) - return parser.parse_args() - - -def fail(message: str) -> None: - """Exit with a validation failure message. - - Args: - message: Human-readable validation error. - """ - raise SystemExit(f"native k8s output validation failed: {message}") - - -def loadYamlDocuments(path: Path) -> list[dict[str, Any]]: - """Load a Kubernetes multi-document YAML file. - - Args: - path: Manifest path. - """ - docs = [doc for doc in yaml.safe_load_all(path.read_text(encoding="utf-8")) if doc is not None] - invalid = [doc for doc in docs if not isinstance(doc, dict)] - if invalid: - fail(f"{path.name} contains non-mapping YAML documents") - return docs - - -def loadImages(path: Path) -> list[dict[str, str]]: - """Load images.yaml and return image entries. - - Args: - path: images.yaml path. - """ - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - if not isinstance(data, dict): - fail("images.yaml root must be a mapping") - images = data.get("images") - if not isinstance(images, list) or not images: - fail("images.yaml must contain a non-empty images list") - for item in images: - if not isinstance(item, dict): - fail("images.yaml entries must be mappings") - if not isinstance(item.get("name"), str) or not item["name"].strip(): - fail("images.yaml entry requires non-empty name") - if not isinstance(item.get("context"), str) or not item["context"].strip(): - fail(f"image {item.get('name')} requires non-empty context") - return images - - -def indexKinds(docs: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: - """Group Kubernetes documents by kind. - - Args: - docs: Parsed Kubernetes manifest documents. - """ - by_kind: dict[str, list[dict[str, Any]]] = {} - for doc in docs: - kind = str(doc.get("kind") or "") - if not kind: - fail(f"manifest document lacks kind: {doc}") - by_kind.setdefault(kind, []).append(doc) - return by_kind - - -def validateNamespace(by_kind: dict[str, list[dict[str, Any]]], expected_namespace: str) -> None: - """Validate the compiler-created namespace. - - Args: - by_kind: Manifest documents grouped by kind. - expected_namespace: Namespace expected from the example compiler. - """ - namespaces = by_kind.get("Namespace", []) - if len(namespaces) != 1: - fail(f"expected exactly one Namespace, got {len(namespaces)}") - namespace = str((namespaces[0].get("metadata") or {}).get("name") or "") - if namespace != expected_namespace: - fail(f"expected namespace {expected_namespace}, got {namespace}") - - -def validateRequiredKinds(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate that Kube-OVN output contains all required resource kinds. - - Args: - by_kind: Manifest documents grouped by kind. - """ - required = ("Namespace", "Vpc", "Subnet", "NetworkAttachmentDefinition", "Deployment") - missing = [kind for kind in required if not by_kind.get(kind)] - if missing: - fail(f"manifest missing required kinds: {', '.join(missing)}") - - -def validateVpcAndSubnets(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate Kube-OVN Vpc and Subnet resource shape. - - Args: - by_kind: Manifest documents grouped by kind. - """ - vpcs = by_kind.get("Vpc", []) - if len(vpcs) != 1: - fail(f"expected exactly one Kube-OVN Vpc, got {len(vpcs)}") - vpc_name = str((vpcs[0].get("metadata") or {}).get("name") or "") - if not vpc_name: - fail("Kube-OVN Vpc requires metadata.name") - for subnet in by_kind.get("Subnet", []): - spec = subnet.get("spec") or {} - if spec.get("vpc") != vpc_name: - fail(f"Subnet {(subnet.get('metadata') or {}).get('name')} does not reference Vpc {vpc_name}") - provider = str(spec.get("provider") or "") - if not provider.endswith(".ovn"): - fail(f"Subnet provider must use Kube-OVN provider suffix: {provider}") - cidr = str(spec.get("cidrBlock") or "") - try: - ipaddress.ip_network(cidr, strict=False) - except ValueError as exc: - fail(f"invalid Subnet cidrBlock {cidr}: {exc}") - if not spec.get("gateway"): - fail(f"Subnet {(subnet.get('metadata') or {}).get('name')} requires gateway") - - -def validateNetworkAttachments(by_kind: dict[str, list[dict[str, Any]]]) -> None: - """Validate that Multus NADs are rendered for Kube-OVN, not macvlan. - - Args: - by_kind: Manifest documents grouped by kind. - """ - for nad in by_kind.get("NetworkAttachmentDefinition", []): - metadata = nad.get("metadata") or {} - name = str(metadata.get("name") or "") - spec = nad.get("spec") or {} - raw_config = spec.get("config") - if not isinstance(raw_config, str): - fail(f"NAD {name} requires string spec.config") - try: - config = json.loads(raw_config) - except json.JSONDecodeError as exc: - fail(f"NAD {name} spec.config is not JSON: {exc}") - if config.get("type") != "kube-ovn": - fail(f"NAD {name} must use type kube-ovn, got {config.get('type')}") - provider = str(config.get("provider") or "") - if not provider.endswith(".ovn"): - fail(f"NAD {name} provider must end with .ovn") - if "master" in config: - fail(f"NAD {name} still contains macvlan master field") - - -def podTemplateAnnotations(workload: dict[str, Any]) -> dict[str, Any]: - """Return workload pod-template annotations. - - Args: - workload: Deployment-like Kubernetes resource. - """ - return ( - ((workload.get("spec") or {}).get("template") or {}) - .get("metadata", {}) - .get("annotations", {}) - or {} - ) - - -def validateDeployments( - by_kind: dict[str, list[dict[str, Any]]], - expected_namespace: str, - image_names: set[str], -) -> None: - """Validate Deployment image and Kube-OVN annotation contracts. - - Args: - by_kind: Manifest documents grouped by kind. - expected_namespace: Namespace expected from the example compiler. - image_names: Logical image names loaded from images.yaml. - """ - for deployment in by_kind.get("Deployment", []): - metadata = deployment.get("metadata") or {} - name = str(metadata.get("name") or "") - if metadata.get("namespace") != expected_namespace: - fail(f"Deployment {name} must be in namespace {expected_namespace}") - containers = ( - ((deployment.get("spec") or {}).get("template") or {}) - .get("spec", {}) - .get("containers", []) - ) - if len(containers) != 1: - fail(f"Deployment {name} expected one main container, got {len(containers)}") - image = str(containers[0].get("image") or "") - if image not in image_names: - fail(f"Deployment {name} image {image} missing from images.yaml") - annotations = podTemplateAnnotations(deployment) - raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") - if not isinstance(raw_networks, str): - fail(f"Deployment {name} lacks Multus network annotation") - try: - networks = json.loads(raw_networks) - except json.JSONDecodeError as exc: - fail(f"Deployment {name} network annotation is not JSON: {exc}") - if not isinstance(networks, list) or not networks: - fail(f"Deployment {name} network annotation must be a non-empty list") - for network in networks: - if not isinstance(network, dict): - fail(f"Deployment {name} contains non-mapping network selection") - if "ips" in network: - fail(f"Deployment {name} still contains macvlan-style ips field") - nad_name = str(network.get("name") or "") - nad_namespace = str(network.get("namespace") or expected_namespace) - key = f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address" - if key not in annotations: - fail(f"Deployment {name} lacks Kube-OVN IP annotation {key}") - - -def validateImages(images: list[dict[str, str]], output_dir: Path, image_registry_prefix: str) -> set[str]: - """Validate image entries and Docker build contexts. - - Args: - images: Parsed images.yaml entries. - output_dir: Compiler output directory. - image_registry_prefix: Expected logical image prefix. - """ - image_names: set[str] = set() - logical_prefix = image_registry_prefix.rstrip("/") + "/" - for item in images: - image = item["name"].strip() - context = item["context"].strip() - if not image.startswith(logical_prefix): - fail(f"image {image} must start with {logical_prefix}") - if image in image_names: - fail(f"duplicate image entry {image}") - image_names.add(image) - context_path = Path(context) - if context_path.is_absolute(): - fail(f"image {image} context must be relative, got {context}") - resolved_context = (output_dir / context_path).resolve() - try: - resolved_context.relative_to(output_dir.resolve()) - except ValueError: - fail(f"image {image} context escapes output directory: {context}") - if not (resolved_context / "Dockerfile").is_file(): - fail(f"image {image} context lacks Dockerfile: {context}") - return image_names - - -def firstFromImage(dockerfile: Path) -> str: - """Return the first FROM image reference in a Dockerfile. - - Args: - dockerfile: Dockerfile path. - """ - for line in dockerfile.read_text(encoding="utf-8").splitlines(): - parts = line.strip().split() - if not parts or parts[0].upper() != "FROM": - continue - if len(parts) >= 3 and parts[1].startswith("--platform="): - return parts[2] - if len(parts) >= 2: - return parts[1] - fail(f"{dockerfile.relative_to(dockerfile.parent.parent)} has malformed FROM line") - fail(f"{dockerfile.relative_to(dockerfile.parent.parent)} lacks FROM line") - - -def validateCompilerBaseImages(output_dir: Path) -> None: - """Validate that hash-tagged node Dockerfiles have staged base contexts. - - Args: - output_dir: Compiler output directory. - """ - required: set[str] = set() - for dockerfile in output_dir.rglob("Dockerfile"): - try: - dockerfile.relative_to(output_dir / "base_images") - continue - except ValueError: - pass - from_image = firstFromImage(dockerfile) - if COMPILER_BASE_TAG_RE.match(from_image): - required.add(from_image.removesuffix(":latest")) - - for base_tag in sorted(required): - context = output_dir / "base_images" / base_tag / "Dockerfile" - if not context.is_file(): - fail(f"missing base_images/{base_tag}/Dockerfile for generated FROM {base_tag}") - - -def validateNoHostLocalText(output_dir: Path) -> None: - """Reject host-local absolute paths in generated text artifacts. - - Args: - output_dir: Compiler output directory. - """ - for path in output_dir.rglob("*"): - if not path.is_file() or path.suffix == ".tar": - continue - try: - text = path.read_text(encoding="utf-8") - except UnicodeDecodeError: - continue - for token in FORBIDDEN_TEXT: - if token in text: - fail(f"{path.relative_to(output_dir)} contains host-local path {token}") - - -def main() -> int: - """Run native K8s compile-output validation.""" - args = parseArgs() - output_dir = args.output_dir.expanduser().resolve() - manifest_path = output_dir / MANIFEST_NAME - images_path = output_dir / IMAGES_NAME - legacy_manifest_path = output_dir / LEGACY_MANIFEST_NAME - - if not manifest_path.is_file(): - fail(f"missing {MANIFEST_NAME}") - if not images_path.is_file(): - fail(f"missing {IMAGES_NAME}") - if legacy_manifest_path.exists(): - fail(f"{LEGACY_MANIFEST_NAME} should not be generated for Kube-OVN output") - - docs = loadYamlDocuments(manifest_path) - images = loadImages(images_path) - by_kind = indexKinds(docs) - image_names = validateImages(images, output_dir, args.image_registry_prefix) - - validateRequiredKinds(by_kind) - validateNamespace(by_kind, args.expected_namespace) - validateVpcAndSubnets(by_kind) - validateNetworkAttachments(by_kind) - validateDeployments(by_kind, args.expected_namespace, image_names) - validateCompilerBaseImages(output_dir) - validateNoHostLocalText(output_dir) - - print( - "Validated native K8s output: " - f"{len(docs)} manifest documents, {len(images)} image contexts." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From d0178ec89ac4bb2bb73d287dfe78742ad5fd2497 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Tue, 9 Jun 2026 12:20:47 +0800 Subject: [PATCH 2/6] update --- seedemu/compiler/__init__.py | 2 +- seedemu/compiler/kubernetes.py | 1627 ++++++++++++----- seedemu/core/AutonomousSystem.py | 101 +- seedemu/core/Node.py | 46 +- seedemu/k8sTools/README.md | 5 +- seedemu/k8sTools/k8sTools.py | 7 +- .../resources/running/_embeddedShell.py | 16 +- .../resources/running/buildRegistryImages.py | 179 +- .../resources/running/buildRegistryImages.sh | 165 -- .../resources/running/manageK8sManifest.py | 180 +- .../resources/running/manageRunningStage.py | 2 + .../running/validateClusterPreflight.py | 221 ++- .../running/validateClusterPreflight.sh | 198 -- seedemu/k8sTools/resources/setup/README.md | 12 +- .../resources/setup/_embeddedShell.py | 16 +- .../resources/setup/ansible/k3s-install.yml | 20 +- .../resources/setup/applyK3sCluster.py | 650 ++++++- .../resources/setup/applyK3sCluster.sh | 589 ------ .../resources/setup/destroyPhysicalCluster.py | 141 +- .../resources/setup/destroyPhysicalCluster.sh | 99 - .../resources/setup/kvm/_embeddedShell.py | 16 +- .../resources/setup/kvm/createKvmVms.py | 471 ++++- .../resources/setup/kvm/createKvmVms.sh | 457 ----- .../resources/setup/kvm/destroyKvmVms.py | 269 ++- .../resources/setup/kvm/destroyKvmVms.sh | 255 --- .../resources/setup/kvm/manageKvmConfig.py | 9 +- .../resources/setup/kvm/prepareHostAssets.py | 312 +++- .../resources/setup/kvm/prepareHostAssets.sh | 298 --- .../resources/setup/kvm/tuneVmLimits.py | 294 ++- .../resources/setup/kvm/tuneVmLimits.sh | 280 --- .../resources/setup/manageK3sConfig.py | 10 +- .../setup/multiHostKvm/_embeddedShell.py | 16 +- .../multiHostKvm/createMultiHostKvmVms.py | 251 ++- .../multiHostKvm/createMultiHostKvmVms.sh | 237 --- .../multiHostKvm/destroyMultiHostKvmVms.py | 161 +- .../multiHostKvm/destroyMultiHostKvmVms.sh | 147 -- .../multiHostKvm/manageMultiHostKvmConfig.py | 5 +- .../multiHostKvm/prepareKvmHypervisors.py | 212 ++- .../multiHostKvm/prepareKvmHypervisors.sh | 198 -- .../resources/setup/ovn/_embeddedShell.py | 16 +- .../resources/setup/ovn/cleanKubeOvnFabric.py | 132 +- .../resources/setup/ovn/cleanKubeOvnFabric.sh | 118 -- .../setup/ovn/installKubeOvnFabric.py | 332 +++- .../setup/ovn/installKubeOvnFabric.sh | 318 ---- .../setup/ovn/validateKubeOvnFabric.py | 57 +- .../setup/ovn/validateKubeOvnFabric.sh | 43 - .../resources/setup/preparePhysicalNodes.py | 81 +- .../resources/setup/preparePhysicalNodes.sh | 67 - .../resources/setup/vxlan/_embeddedShell.py | 16 +- .../setup/vxlan/cleanLinuxVxlanFabric.py | 94 +- .../setup/vxlan/cleanLinuxVxlanFabric.sh | 80 - .../setup/vxlan/configureLinuxVxlanFabric.py | 139 +- .../setup/vxlan/configureLinuxVxlanFabric.sh | 125 -- .../setup/vxlan/validateLinuxVxlanFabric.py | 189 +- .../setup/vxlan/validateLinuxVxlanFabric.sh | 175 -- seedemu/k8sTools/runner.py | 49 +- seedemu/k8sTools/utils.py | 4 +- seedemu/layers/Ebgp.py | 40 +- seedemu/layers/Ibgp.py | 174 +- seedemu/layers/Ospf.py | 6 + seedemu/layers/Routing.py | 57 +- 61 files changed, 5872 insertions(+), 4614 deletions(-) delete mode 100755 seedemu/k8sTools/resources/running/buildRegistryImages.sh delete mode 100755 seedemu/k8sTools/resources/running/validateClusterPreflight.sh delete mode 100755 seedemu/k8sTools/resources/setup/applyK3sCluster.sh delete mode 100755 seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh delete mode 100755 seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh delete mode 100755 seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh delete mode 100755 seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh delete mode 100755 seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh delete mode 100755 seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh delete mode 100755 seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh delete mode 100755 seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh delete mode 100755 seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh delete mode 100755 seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh delete mode 100755 seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh delete mode 100755 seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh delete mode 100755 seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh delete mode 100755 seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh delete mode 100755 seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh diff --git a/seedemu/compiler/__init__.py b/seedemu/compiler/__init__.py index 5dad3a90a..552e5d449 100644 --- a/seedemu/compiler/__init__.py +++ b/seedemu/compiler/__init__.py @@ -5,4 +5,4 @@ from .DistributedDocker import DistributedDocker from .Graphviz import Graphviz from .GcpDistributedDocker import GcpDistributedDocker -from .kubernetes import NativeKubernetesCompiler +from .kubernetes import KubernetesCompiler, NativeKubernetesCompiler, SchedulingStrategy diff --git a/seedemu/compiler/kubernetes.py b/seedemu/compiler/kubernetes.py index 8510a384a..3505197e9 100644 --- a/seedemu/compiler/kubernetes.py +++ b/seedemu/compiler/kubernetes.py @@ -1,386 +1,745 @@ from __future__ import annotations - +from seedemu.core.Emulator import Emulator +from seedemu.core import Node, Network, Compiler, Scope, OptionHandling, BaseVolume +from seedemu.core.enums import NodeRole, NetworkType +from .Docker import Docker, DockerCompilerFileTemplates +from .DockerImage import DockerImage +from typing import Dict, Generator, List, Set, Tuple, Optional, Any +from hashlib import md5 +from os import mkdir, chdir import json import os -import ipaddress -import re import shutil -from hashlib import md5, sha1 -from os import chdir, mkdir -from pathlib import Path -from typing import Any, Dict, List - +import shlex import yaml -from seedemu.compiler.Docker import Docker -from seedemu.core import Network, Node - -INTERNET_MAP_META_PREFIX = "org.seedsecuritylabs.seedemu.meta." -# Border routers are already represented through the router node path in SEED's -# registry. Compiling "brdnode" again creates duplicate Deployment names. -SEEDEMU_NODE_TYPES = ["rnode", "csnode", "hnode", "rs", "snode"] +class SchedulingStrategy: + """Scheduling strategy constants for Kubernetes node placement.""" + NONE = "none" # No scheduling constraints + AUTO = "auto" # Automatic soft grouping + soft spreading + BY_AS = "by_as" # Schedule pods by AS number (soft affinity) + BY_AS_HARD = "by_as_hard" # Same ASN must land on one explicit Kubernetes node + BY_ROLE = "by_role" # Schedule pods by node role (router, host, etc.) + CUSTOM = "custom" # Use custom labels provided by user -class NativeKubernetesCompiler(Docker): - """Minimal Kubernetes compiler for a k8s-native baseline. +class KubernetesCompiler(Docker): + """! + @brief The Kubernetes compiler class. - Design goals: - - no inventory dependency during compile - - no nodeSelector / affinity / topology spreading - - no node-aware preload planning - - generated orchestration artifacts are limited to one manifest file and - images.yaml; build/deploy orchestration lives in seedemu.k8sTools. + Compiles the emulation to Kubernetes manifests with support for: + - Multi-node scheduling (nodeSelector, nodeAffinity) + - Resource management (requests/limits) + - Service generation (ClusterIP, NodePort) + - Configurable CNI types for cross-node networking """ + __registry_prefix: str __namespace: str + __use_multus: bool + __manifests: List[str] + __build_commands: List[str] + _current_node: Node + + # New fields for multi-node support + __scheduling_strategy: str + __node_labels: Dict[str, Dict[str, str]] + __default_resources: Dict[str, Dict[str, str]] __cni_type: str + __local_link_cni_type: Optional[str] __cni_master_interface: str + __generate_services: bool + __service_type: str __image_pull_policy: str - __image_registry_prefix: str - __manifests: List[Dict[str, Any]] - __image_entries: List[Dict[str, str]] def __init__( self, - image_registry_prefix: str = "seedemu", - registry_prefix: str | None = None, - namespace: str = "seedemu-k8s-b61", - cni_type: str = "kube-ovn", - cni_master_interface: str = "ens2", + registry_prefix: str = "localhost:5000", + namespace: str = "seedemu", + use_multus: bool = True, + internetMapEnabled: bool = True, + # New parameters for multi-node deployment + scheduling_strategy: str = SchedulingStrategy.NONE, + node_labels: Dict[str, Dict[str, str]] = None, + default_resources: Dict[str, Dict[str, str]] = None, + cni_type: str = "bridge", + local_link_cni_type: Optional[str] = None, + cni_master_interface: str = "eth0", + generate_services: bool = False, + service_type: str = "ClusterIP", image_pull_policy: str = "Always", - # use_multus: bool = True, - # create_namespace: bool = True, - **kwargs: Any, - ) -> None: - kwargs["selfManagedNetwork"] = True + **kwargs + ): + """! + @brief Kubernetes compiler constructor. + + @param registry_prefix (optional) Registry prefix for docker images. Default "localhost:5000". + @param namespace (optional) Kubernetes namespace. Default "seedemu". + @param use_multus (optional) Use Multus CNI for multiple interfaces. Default True. + @param internetMapEnabled (optional) Enable Internet Map visualization service. Default True. + @param scheduling_strategy (optional) How to schedule pods across nodes. Options: + - "none": No scheduling constraints (default) + - "auto": Let scheduler auto-balance with soft AS/role affinity + - "by_as": Prefer pods with same AS on same node (no node pre-labeling needed) + - "by_role": Prefer same role on same node (no node pre-labeling needed) + - "custom": Use custom node_labels mapping + @param node_labels (optional) Custom node labels for scheduling. Dict mapping AS numbers or + node names to label dicts. Example: {"150": {"kubernetes.io/hostname": "node1"}} + @param default_resources (optional) Default resource requests/limits for all pods. + Example: {"requests": {"cpu": "100m", "memory": "128Mi"}, + "limits": {"cpu": "500m", "memory": "512Mi"}} + @param cni_type (optional) CNI plugin type for NetworkAttachmentDefinition. Options: + - "bridge": Local bridge (default, single-node only) + - "macvlan": macvlan for cross-node with L2 access + - "ipvlan": ipvlan for cross-node networking + - "host-local": Use host-local IPAM + @param local_link_cni_type (optional) Override CNI type for node-local internal links. + When left unset, `by_as_hard` + `macvlan/ipvlan` automatically falls back to + `bridge` for `Local` / `CrossConnect` networks so same-AS internal links stay + isolated on the selected Kubernetes node. + @param cni_master_interface (optional) Master interface for macvlan/ipvlan. Default "eth0". + @param generate_services (optional) Generate K8s Service resources for nodes. Default False. + @param service_type (optional) Service type when generate_services is True. + Options: "ClusterIP", "NodePort". Default "ClusterIP". + @param image_pull_policy (optional) K8s ImagePullPolicy. Default "Always". + """ + # Force selfManagedNetwork=True so that parent logic generates /replace_address.sh call in start.sh + kwargs['selfManagedNetwork'] = True super().__init__(**kwargs) - if registry_prefix is not None: - image_registry_prefix = registry_prefix - self.__image_registry_prefix = image_registry_prefix.strip().strip("/") - self.__namespace = namespace.strip() or "seedemu" - self.__cni_type = cni_type.strip().lower() or "bridge" - self.__cni_master_interface = cni_master_interface.strip() or "eth0" - self.__image_pull_policy = image_pull_policy.strip() or "Always" + self.__registry_prefix = registry_prefix + self.__namespace = namespace + self.__use_multus = use_multus + self.__internet_map_enabled = internetMapEnabled self.__manifests = [] - self.__image_entries = [] + self.__build_commands = [] + self._current_node = None + + # Multi-node deployment settings + strategy = (scheduling_strategy or SchedulingStrategy.NONE).strip().lower() + valid = { + SchedulingStrategy.NONE, + SchedulingStrategy.AUTO, + SchedulingStrategy.BY_AS, + SchedulingStrategy.BY_AS_HARD, + SchedulingStrategy.BY_ROLE, + SchedulingStrategy.CUSTOM, + } + self.__scheduling_strategy = strategy if strategy in valid else SchedulingStrategy.NONE + self.__node_labels = node_labels or {} + self.__default_resources = default_resources or {} + self.__cni_type = cni_type + self.__local_link_cni_type = local_link_cni_type.strip().lower() if isinstance(local_link_cni_type, str) and local_link_cni_type.strip() else None + self.__cni_master_interface = cni_master_interface + self.__generate_services = generate_services + self.__service_type = service_type + self.__image_pull_policy = image_pull_policy def getName(self) -> str: - return "NativeKubernetes" + return "Kubernetes" + + def _resolveInternetMapImages(self) -> Tuple[str, str]: + """Resolve source and deployment image refs for the optional Internet Map service.""" + source_image = os.environ.get( + "SEED_INTERNET_MAP_SOURCE_IMAGE", + "handsonsecurity/seedemu-multiarch-map:buildx-latest", + ).strip() or "handsonsecurity/seedemu-multiarch-map:buildx-latest" + + default_target = source_image + if self.__registry_prefix: + default_target = f"{self.__registry_prefix}/seedemu-internet-map:buildx-latest" + + target_image = os.environ.get("SEED_INTERNET_MAP_IMAGE", default_target).strip() or default_target + return source_image, target_image - def getManifestName(self) -> str: - """Return the manifest filename generated by this compiler.""" - if self._usesKubeOvn(): - return "k8s.kube-ovn.yaml" - return "k8s.yaml" @staticmethod def _safeBridgeName(name: str) -> str: - return f"br-{md5(name.encode()).hexdigest()[:12]}" - - def _usesKubeOvn(self) -> bool: - """Return true when the compiler should emit Kube-OVN resources.""" - return self.__cni_type in {"kube-ovn", "kube_ovn", "ovn"} + """Generate a Linux bridge name that fits the 15-char IFNAMSIZ limit. + + Uses 'br-' prefix (3 chars) + md5 hash truncated to 12 chars = 15 chars total. + This ensures uniqueness while staying within the kernel limit. + """ + h = md5(name.encode()).hexdigest()[:12] + return f"br-{h}" - def _doCompile(self, emulator) -> None: + def _doCompile(self, emulator: Emulator): registry = emulator.getRegistry() self._groupSoftware(emulator) - self.__manifests.append( - { - "apiVersion": "v1", - "kind": "Namespace", - "metadata": {"name": self.__namespace}, - } - ) - - for ((_, obj_type, _), obj) in registry.getAll().items(): - if obj_type == "net": - self.__manifests.append(self._compileNetK8s(obj)) - - for ((_, obj_type, _), obj) in registry.getAll().items(): - if obj_type in SEEDEMU_NODE_TYPES: + # Implementation overview: + # (1) networks -> NetworkAttachmentDefinition (Multus), + # (2) nodes -> Deployment/VirtualMachine, + # (3) output artifacts (k8s.yaml, build_images.sh, .env). + # 1. Generate NetworkAttachmentDefinitions (if Multus) + if self.__use_multus: + for ((scope, type, name), obj) in registry.getAll().items(): + if type == 'net': + self.__manifests.append(self._compileNetK8s(obj)) + + # 2. Compile Nodes + for ((scope, type, name), obj) in registry.getAll().items(): + if type in ['rnode', 'csnode', 'hnode', 'rs', 'snode']: self.__manifests.append(self._compileNodeK8s(obj)) - manifests = ( - self._renderKubeOvnManifests(self.__manifests) - if self._usesKubeOvn() - else self.__manifests - ) - with open(self.getManifestName(), "w", encoding="utf-8") as handle: - yaml.safe_dump_all(manifests, handle, sort_keys=False) + # 3. Generate Output Files - self._stage_base_image_contexts() - with open("images.yaml", "w", encoding="utf-8") as handle: - yaml.safe_dump({"images": self.__image_entries}, handle, sort_keys=False) + # manifests.yaml + with open('k8s.yaml', 'w') as f: + f.write("\n---\n".join(self.__manifests)) - def _compileNetK8s(self, net: Network) -> Dict[str, Any]: - name = self._getRealNetName(net).replace("_", "-").lower() + # Stage local Dockerfile contexts for built-in base images (to avoid relying on Docker Hub mirrors + # that may not whitelist custom images like handsonsecurity/*). + used_images = sorted(getattr(self, "_used_images", set())) + try: + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + base_image_sources = { + # These match DockerImageConstant defaults. + "handsonsecurity/seedemu-multiarch-base:buildx-latest": os.path.join(repo_root, "docker_images", "multiarch", "seedemu-base"), + "handsonsecurity/seedemu-multiarch-router:buildx-latest": os.path.join(repo_root, "docker_images", "multiarch", "seedemu-router"), + } + staged_any = False + for image in used_images: + src = base_image_sources.get(image) + if not src or not os.path.isdir(src): + continue + digest = md5(image.encode("utf-8")).hexdigest() + dst_root = os.path.join("base_images", digest) + os.makedirs(os.path.dirname(dst_root), exist_ok=True) + if os.path.exists(dst_root): + shutil.rmtree(dst_root) + shutil.copytree(src, dst_root) + staged_any = True + if not staged_any and os.path.isdir("base_images"): + shutil.rmtree("base_images") + except Exception: + # Best-effort only: compilation should still succeed even if we can't stage base images. + pass + + # build_images.sh + with open('build_images.sh', 'w') as f: + f.write("#!/usr/bin/env bash\n") + f.write("set -euo pipefail\n") + # Allow callers to override BuildKit/parallelism without editing generated artifacts. + f.write('export DOCKER_BUILDKIT="${SEED_DOCKER_BUILDKIT:-0}"\n') + f.write('PARALLELISM="${SEED_BUILD_PARALLELISM:-1}"\n') + f.write('if ! [[ "${PARALLELISM}" =~ ^[0-9]+$ ]]; then PARALLELISM=1; fi\n') + f.write('export REGISTRY_PUSH_RETRIES="${SEED_REGISTRY_PUSH_RETRIES:-5}"\n') + f.write('if ! [[ "${REGISTRY_PUSH_RETRIES}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_RETRIES=5; fi\n') + f.write('export REGISTRY_PUSH_BACKOFF_SECONDS="${SEED_REGISTRY_PUSH_BACKOFF_SECONDS:-5}"\n') + f.write('if ! [[ "${REGISTRY_PUSH_BACKOFF_SECONDS}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_BACKOFF_SECONDS=5; fi\n') + f.write('export REGISTRY_PUSH_TIMEOUT_SECONDS="${SEED_REGISTRY_PUSH_TIMEOUT_SECONDS:-180}"\n') + f.write('if ! [[ "${REGISTRY_PUSH_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_TIMEOUT_SECONDS=180; fi\n') + f.write('export SEED_IMAGE_DISTRIBUTION_MODE="${SEED_IMAGE_DISTRIBUTION_MODE:-registry}"\n') + f.write('if [[ "${SEED_IMAGE_DISTRIBUTION_MODE}" != "registry" && "${SEED_IMAGE_DISTRIBUTION_MODE}" != "preload" ]]; then export SEED_IMAGE_DISTRIBUTION_MODE="registry"; fi\n') + f.write('export SEED_DOCKER_MAX_CONCURRENT_UPLOADS="${SEED_DOCKER_MAX_CONCURRENT_UPLOADS:-1}"\n') + f.write('if ! [[ "${SEED_DOCKER_MAX_CONCURRENT_UPLOADS}" =~ ^[0-9]+$ ]]; then export SEED_DOCKER_MAX_CONCURRENT_UPLOADS=1; fi\n') + # Export so the inline Python snippet (daemon.json edit) can see it. + f.write('export SEED_DOCKER_IO_MIRROR_ENDPOINT="${SEED_DOCKER_IO_MIRROR_ENDPOINT:-https://docker.m.daocloud.io}"\n') + f.write('MIRROR_HOST="${SEED_DOCKER_IO_MIRROR_ENDPOINT#http://}"\n') + f.write('MIRROR_HOST="${MIRROR_HOST#https://}"\n') + # Export so the inline Python snippet (daemon.json edit) can see it. + f.write(f'export REGISTRY_PREFIX="{self.__registry_prefix}"\n') + f.write('export REGISTRY_LOCAL_ENDPOINT="${SEED_REGISTRY_LOCAL_ENDPOINT:-}"\n') + f.write("\n") + f.write("docker_pull() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if command -v timeout >/dev/null 2>&1; then\n") + f.write(" timeout 180s docker pull \"$image\"\n") + f.write(" else\n") + f.write(" docker pull \"$image\"\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write("mirror_image_name() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if [[ -z \"${MIRROR_HOST}\" ]]; then\n") + f.write(" echo \"$image\"\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" if [[ \"$image\" == *\"/\"* ]]; then\n") + f.write(" echo \"${MIRROR_HOST}/${image}\"\n") + f.write(" else\n") + f.write(" echo \"${MIRROR_HOST}/library/${image}\"\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write("ensure_image_present() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if docker image inspect \"$image\" >/dev/null 2>&1; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" local mirror\n") + f.write(" mirror=\"$(mirror_image_name \"$image\")\"\n") + # In environments where Docker Hub is blocked, try the mirror first. + f.write(" if [[ \"$mirror\" != \"$image\" ]]; then\n") + f.write(" if docker_pull \"$mirror\" >/dev/null 2>&1; then\n") + f.write(" docker tag \"$mirror\" \"$image\" >/dev/null 2>&1 || true\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" fi\n") + f.write(" if docker_pull \"$image\" >/dev/null 2>&1; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" echo \"[build_images] ERROR: cannot pull base image: $image\" >&2\n") + f.write(" echo \"[build_images] Hint: set SEED_DOCKER_IO_MIRROR_ENDPOINT to a reachable mirror\" >&2\n") + f.write(" return 1\n") + f.write("}\n") + f.write("\n") + f.write("ensure_docker_daemon_config() {\n") + f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then return 0; fi\n") + f.write(" if [[ -z \"${REGISTRY_PREFIX}\" ]]; then return 0; fi\n") + f.write(" if [[ \"$(id -u)\" != \"0\" ]]; then return 0; fi\n") + f.write(" if ! command -v systemctl >/dev/null 2>&1; then return 0; fi\n") + f.write(" mkdir -p /etc/docker\n") + f.write(" local result\n") + f.write(" result=\"$(python3 - <<'PY'\n") + f.write("import json\n") + f.write("from pathlib import Path\n") + f.write("import os\n") + f.write("\n") + f.write("registry = os.environ.get('REGISTRY_PREFIX', '')\n") + f.write("mirror = os.environ.get('SEED_DOCKER_IO_MIRROR_ENDPOINT', '')\n") + f.write("max_uploads = os.environ.get('SEED_DOCKER_MAX_CONCURRENT_UPLOADS', '1')\n") + f.write("try:\n") + f.write(" max_uploads_value = max(1, int(max_uploads))\n") + f.write("except Exception:\n") + f.write(" max_uploads_value = 1\n") + f.write("p = Path('/etc/docker/daemon.json')\n") + f.write("data = {}\n") + f.write("if p.exists():\n") + f.write(" try:\n") + f.write(" data = json.loads(p.read_text(encoding='utf-8'))\n") + f.write(" except Exception:\n") + f.write(" data = {}\n") + f.write("\n") + f.write("before = json.dumps(data, sort_keys=True)\n") + f.write("insec = set(data.get('insecure-registries', []) or [])\n") + f.write("if registry and '/' not in registry:\n") + f.write(" insec.add(registry)\n") + f.write(" if ':' in registry:\n") + f.write(" port = registry.rsplit(':', 1)[1]\n") + f.write(" insec.add(f'127.0.0.1:{port}')\n") + f.write(" insec.add(f'localhost:{port}')\n") + f.write("data['insecure-registries'] = sorted(insec)\n") + f.write("\n") + f.write("mirrors = list(data.get('registry-mirrors', []) or [])\n") + f.write("if mirror and mirror not in mirrors:\n") + f.write(" mirrors.append(mirror)\n") + f.write("data['registry-mirrors'] = mirrors\n") + f.write("data['max-concurrent-uploads'] = max_uploads_value\n") + f.write("\n") + f.write("after = json.dumps(data, sort_keys=True)\n") + f.write("if after != before:\n") + f.write(" p.write_text(json.dumps(data, indent=2) + '\\n', encoding='utf-8')\n") + f.write(" print('changed')\n") + f.write("else:\n") + f.write(" print('unchanged')\n") + f.write("PY\n") + f.write(")\"\n") + f.write(" if [[ \"${result}\" == \"changed\" ]]; then\n") + f.write(" systemctl restart docker >/dev/null 2>&1 || true\n") + f.write(" sleep 2\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write('export REGISTRY_ENDPOINT="${REGISTRY_PREFIX%%/*}"\n') + f.write('if [[ -z "${REGISTRY_LOCAL_ENDPOINT}" ]] && [[ "${REGISTRY_ENDPOINT}" == *:* ]] && [[ "${REGISTRY_ENDPOINT}" != 127.0.0.1:* ]] && [[ "${REGISTRY_ENDPOINT}" != localhost:* ]]; then REGISTRY_LOCAL_ENDPOINT="127.0.0.1:${REGISTRY_ENDPOINT##*:}"; fi\n') + f.write('export REGISTRY_PROBE_ENDPOINT="${REGISTRY_LOCAL_ENDPOINT:-${REGISTRY_ENDPOINT}}"\n') + f.write("registry_probe() {\n") + f.write(" if [[ -z \"${REGISTRY_PROBE_ENDPOINT}\" ]]; then return 0; fi\n") + f.write(" if command -v curl >/dev/null 2>&1; then\n") + f.write(" curl -m 5 -fsS \"http://${REGISTRY_PROBE_ENDPOINT}/v2/\" >/dev/null\n") + f.write(" elif command -v wget >/dev/null 2>&1; then\n") + f.write(" wget -q -T 5 -O /dev/null \"http://${REGISTRY_PROBE_ENDPOINT}/v2/\"\n") + f.write(" else\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write("wait_for_registry() {\n") + f.write(" local retries=\"${1:-6}\"\n") + f.write(" local attempt=1\n") + f.write(" while [ \"${attempt}\" -le \"${retries}\" ]; do\n") + f.write(" if registry_probe; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" sleep \"${attempt}\"\n") + f.write(" attempt=$((attempt + 1))\n") + f.write(" done\n") + f.write(" return 1\n") + f.write("}\n") + f.write("\n") + f.write("local_push_image_ref() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if [[ -z \"${REGISTRY_LOCAL_ENDPOINT}\" ]] || [[ -z \"${REGISTRY_PREFIX}\" ]]; then\n") + f.write(" echo \"${image}\"\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" local suffix=\"${image#${REGISTRY_PREFIX}/}\"\n") + f.write(" if [[ \"${suffix}\" == \"${image}\" ]]; then\n") + f.write(" echo \"${image}\"\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" echo \"${REGISTRY_LOCAL_ENDPOINT}/${suffix}\"\n") + f.write("}\n") + f.write("\n") + f.write("docker_push_with_timeout() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if command -v timeout >/dev/null 2>&1; then\n") + f.write(" timeout \"${REGISTRY_PUSH_TIMEOUT_SECONDS}\" docker push \"${image}\"\n") + f.write(" else\n") + f.write(" docker push \"${image}\"\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write("retry_push() {\n") + f.write(" local image=\"$1\"\n") + f.write(" if [[ -z \"${REGISTRY_PREFIX}\" ]]; then return 0; fi\n") + f.write(" local attempt=1\n") + f.write(" while [ \"${attempt}\" -le \"${REGISTRY_PUSH_RETRIES}\" ]; do\n") + f.write(" if ! wait_for_registry 3; then\n") + f.write(" echo \"[build_images] registry probe failed before push attempt ${attempt}/${REGISTRY_PUSH_RETRIES}: ${REGISTRY_PROBE_ENDPOINT}\" >&2\n") + f.write(" fi\n") + f.write(" if docker_push_with_timeout \"${image}\"; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" if [ \"${attempt}\" -ge \"${REGISTRY_PUSH_RETRIES}\" ]; then\n") + f.write(" break\n") + f.write(" fi\n") + f.write(" local sleep_seconds=$((REGISTRY_PUSH_BACKOFF_SECONDS * attempt))\n") + f.write(" echo \"[build_images] retrying push ${image} in ${sleep_seconds}s (${attempt}/${REGISTRY_PUSH_RETRIES})\" >&2\n") + f.write(" sleep \"${sleep_seconds}\"\n") + f.write(" attempt=$((attempt + 1))\n") + f.write(" done\n") + f.write(" echo \"[build_images] ERROR: push failed after ${REGISTRY_PUSH_RETRIES} attempts: ${image}\" >&2\n") + f.write(" return 1\n") + f.write("}\n") + f.write("\n") + f.write("seedemu_build_and_push() {\n") + f.write(" local image=\"$1\"\n") + f.write(" local context_dir=\"$2\"\n") + f.write(" docker build -t \"${image}\" \"${context_dir}\"\n") + f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" if [[ -n \"${REGISTRY_PREFIX}\" ]]; then\n") + f.write(" local push_image\n") + f.write(" push_image=\"$(local_push_image_ref \"${image}\")\"\n") + f.write(" if [[ \"${push_image}\" != \"${image}\" ]]; then\n") + f.write(" docker tag \"${image}\" \"${push_image}\"\n") + f.write(" fi\n") + f.write(" retry_push \"${push_image}\"\n") + f.write(" fi\n") + f.write("}\n") + f.write("\n") + f.write("seedemu_copy_and_push_image() {\n") + f.write(" local source_image=\"$1\"\n") + f.write(" local target_image=\"$2\"\n") + f.write(" ensure_image_present \"${source_image}\"\n") + f.write(" docker tag \"${source_image}\" \"${target_image}\"\n") + f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" if [[ -n \"${REGISTRY_PREFIX}\" ]]; then\n") + f.write(" local push_image\n") + f.write(" push_image=\"$(local_push_image_ref \"${target_image}\")\"\n") + f.write(" if [[ \"${push_image}\" != \"${target_image}\" ]]; then\n") + f.write(" docker tag \"${target_image}\" \"${push_image}\"\n") + f.write(" fi\n") + f.write(" retry_push \"${push_image}\"\n") + f.write(" fi\n") + f.write("}\n") + f.write("export -f registry_probe wait_for_registry local_push_image_ref docker_push_with_timeout retry_push seedemu_build_and_push seedemu_copy_and_push_image\n") + f.write("\n") + f.write("prepare_dummy_image() {\n") + f.write(" local base_image=\"$1\"\n") + f.write(" local dummy_tag=\"$2\"\n") + f.write(" if docker image inspect \"$dummy_tag\" >/dev/null 2>&1; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" if ! docker image inspect \"$base_image\" >/dev/null 2>&1; then\n") + f.write(" local ctx=\"base_images/${dummy_tag}\"\n") + f.write(" if [ -f \"${ctx}/Dockerfile\" ]; then\n") + f.write(" echo \"[build_images] building base image: ${base_image} (from ${ctx})\" >&2\n") + f.write(" # Pre-pull common upstream bases via mirror+tag to avoid Docker Hub outages.\n") + f.write(" ensure_image_present \"ubuntu:20.04\"\n") + f.write(" docker build -t \"${base_image}\" \"${ctx}\"\n") + f.write(" else\n") + f.write(" ensure_image_present \"$base_image\"\n") + f.write(" fi\n") + f.write(" fi\n") + f.write(" mkdir -p dummies\n") + f.write(" local df=\"dummies/${dummy_tag}.Dockerfile\"\n") + f.write(" printf 'FROM %s\\n' \"$base_image\" > \"$df\"\n") + f.write(" docker build -t \"$dummy_tag\" -f \"$df\" dummies >/dev/null\n") + f.write("}\n") + f.write("\n") + f.write("load_prefetched_images() {\n") + f.write(" if [ ! -d prefetched_images ]; then\n") + f.write(" return 0\n") + f.write(" fi\n") + f.write(" local tarball\n") + f.write(" shopt -s nullglob\n") + f.write(" for tarball in prefetched_images/*.tar; do\n") + f.write(" echo \"[build_images] loading prefetched image: ${tarball}\" >&2\n") + f.write(" docker load -i \"${tarball}\" >/dev/null\n") + f.write(" done\n") + f.write(" shopt -u nullglob\n") + f.write("}\n") + f.write("\n") + f.write("ensure_docker_daemon_config\n") + f.write("load_prefetched_images\n") + + if used_images: + f.write('echo "[build_images] preparing base-image dummies"\n') + for image in used_images: + digest = md5(image.encode("utf-8")).hexdigest() + f.write(f'prepare_dummy_image "{image}" "{digest}"\n') + f.write("\n") + f.write('JOBS_FILE="$(mktemp)"\n') + f.write('cleanup() { rm -f "${JOBS_FILE}"; }\n') + f.write('trap cleanup EXIT\n') + f.write("cat > \"${JOBS_FILE}\" <<'JOBS'\n") + f.write("\n".join(self.__build_commands)) + f.write("\nJOBS\n") + f.write('if [ "${PARALLELISM}" -le 1 ]; then\n') + f.write(' while IFS= read -r cmd; do\n') + f.write(' [ -z "${cmd}" ] && continue\n') + f.write(' echo "+ ${cmd}"\n') + f.write(' eval "${cmd}"\n') + f.write(' done < "${JOBS_FILE}"\n') + f.write('else\n') + # Run each full line as one job (preserve spaces in the docker commands). + f.write(" awk 'NF' \"${JOBS_FILE}\" | xargs -P \"${PARALLELISM}\" -d '\\n' -I {} bash -lc 'set -euo pipefail; cmd=\"{}\"; echo \"+ ${cmd}\"; eval \"${cmd}\"'\n") + f.write('fi\n') + os.chmod('build_images.sh', 0o755) + + image_refs = [] + for command in self.__build_commands: + parts = shlex.split(command) + if len(parts) >= 2 and parts[0] == 'seedemu_build_and_push': + image_refs.append(parts[1]) + elif len(parts) >= 3 and parts[0] == 'seedemu_copy_and_push_image': + image_refs.append(parts[2]) + with open('images.txt', 'w') as f: + if image_refs: + f.write("\n".join(image_refs) + "\n") + + # Generate .env file + self.generateEnvFile(Scope(0), '') # Simplified scope handling + + def _compileNetK8s(self, net: Network) -> str: + """Generates NetworkAttachmentDefinition for a network. + + Supports multiple CNI types: + - bridge: Local bridge (single-node) + - macvlan: For cross-node with L2 access + - ipvlan: For cross-node networking + - kube-ovn/ovn: Kube-OVN attached networks through OVS + """ + name = self._getRealNetName(net).replace('_', '-').lower() prefix = str(net.getPrefix()) - if self.__cni_type == "macvlan": + cni_type = self._resolveNetworkCniType(net) + + # Build CNI config based on cni_type + if cni_type == "macvlan": config = { "cniVersion": "0.3.1", "type": "macvlan", "master": self.__cni_master_interface, "mode": "bridge", - "ipam": {"type": "static"}, + "ipam": { + "type": "static" # We manage IPs inside the container + } } - elif self.__cni_type == "ipvlan": + elif cni_type == "ipvlan": config = { "cniVersion": "0.3.1", "type": "ipvlan", "master": self.__cni_master_interface, "mode": "l2", - "ipam": {"type": "static"}, + "ipam": { + "type": "static" + } + } + elif cni_type in {"kube-ovn", "ovn"}: + config = { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": f"{name}.{self.__namespace}.ovn", } - elif self.__cni_type == "host-local": + elif cni_type == "host-local": config = { "cniVersion": "0.3.1", "type": "bridge", + # The Linux bridge device lives at node scope (not K8s namespace scope). + # If we only hash `name`, different namespaces compiling the same topology + # (e.g., multiple smoke runs) will share the same L2 segment and collide on + # IPs/MACs, causing flaky BGP/OSPF behavior. Salt with namespace to isolate. "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), "isGateway": False, - "ipam": {"type": "host-local", "subnet": prefix}, + "ipam": { + "type": "host-local", + "subnet": prefix + } } - else: + else: # Default: bridge (single-node) config = { "cniVersion": "0.3.1", "type": "bridge", + # Same rationale as host-local branch: isolate bridge names per namespace. "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "ipam": {"type": "static"}, + "ipam": {} # No IPAM, we manage IPs inside } - return { + scope, _, registry_name = net.getRegistryInfo() + annotations = { + "org.seedsecuritylabs.seedemu.meta.type": "global" if scope == "ix" else "local", + "org.seedsecuritylabs.seedemu.meta.scope": str(scope), + "org.seedsecuritylabs.seedemu.meta.name": str(registry_name), + "org.seedsecuritylabs.seedemu.meta.prefix": prefix, + } + if net.getDisplayName() is not None: + annotations["org.seedsecuritylabs.seedemu.meta.displayname"] = str(net.getDisplayName()) + if net.getDescription() is not None: + annotations["org.seedsecuritylabs.seedemu.meta.description"] = str(net.getDescription()) + + manifest = { "apiVersion": "k8s.cni.cncf.io/v1", "kind": "NetworkAttachmentDefinition", "metadata": { "name": name, "namespace": self.__namespace, - "annotations": self._getInternetMapNetMeta(net), + "annotations": annotations, }, - "spec": {"config": json.dumps(config)}, - } - - def _renderKubeOvnManifests(self, manifests: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Convert compiler manifests to Kube-OVN secondary-network resources. - - Args: - manifests: Namespace, NAD, and workload manifests generated by the - compiler before fabric-specific conversion. - - Returns: - Manifest list containing a Kube-OVN Vpc, one Subnet per SeedEMU - network, kube-ovn NADs, and workloads with provider IP annotations. - """ - namespace_name = self._findManifestNamespace(manifests) - vpc_name = self._kubeOvnResourceName("vpc", namespace_name) - rendered: List[Dict[str, Any]] = [] - vpc_written = False - - for manifest in manifests: - if manifest.get("kind") == "Namespace" and not vpc_written: - rendered.append(manifest) - rendered.append(self._kubeOvnVpc(namespace_name, vpc_name)) - vpc_written = True - continue - if manifest.get("kind") == "NetworkAttachmentDefinition": - rendered.extend(self._convertNetworkAttachmentToKubeOvn(manifest, namespace_name, vpc_name)) - continue - self._convertWorkloadAnnotationsToKubeOvn(manifest, namespace_name) - rendered.append(manifest) - - if not vpc_written: - rendered.insert(0, self._kubeOvnVpc(namespace_name, vpc_name)) - return rendered - - def _findManifestNamespace(self, manifests: List[Dict[str, Any]]) -> str: - """Return the namespace used by workload manifests.""" - for manifest in manifests: - if manifest.get("kind") == "Namespace": - name = (manifest.get("metadata") or {}).get("name") - if name: - return str(name) - for manifest in manifests: - metadata = manifest.get("metadata") or {} - name = metadata.get("namespace") - if name: - return str(name) - return self.__namespace - - def _kubeOvnResourceName(self, prefix: str, value: str) -> str: - """Return a DNS-safe Kube-OVN cluster-scoped resource name.""" - safe = "".join(ch if ch.isalnum() or ch == "-" else "-" for ch in value.lower()).strip("-") - digest = sha1(value.encode("utf-8")).hexdigest()[:8] - return f"{prefix}-{digest}-{safe}"[:63].rstrip("-") - - def _kubeOvnVpc(self, namespace_name: str, vpc_name: str) -> Dict[str, Any]: - """Return a Kube-OVN Vpc that isolates one SeedEMU namespace.""" - return { - "apiVersion": "kubeovn.io/v1", - "kind": "Vpc", - "metadata": {"name": vpc_name}, - "spec": {"namespaces": [namespace_name]}, - } - - def _convertNetworkAttachmentToKubeOvn( - self, - manifest: Dict[str, Any], - default_namespace: str, - vpc_name: str, - ) -> List[Dict[str, Any]]: - """Return [Subnet, NAD] for one SeedEMU network.""" - metadata = manifest.get("metadata") or {} - nad_name = str(metadata.get("name") or "") - namespace_name = str(metadata.get("namespace") or default_namespace) - if not nad_name: - return [manifest] - annotations = metadata.get("annotations") or {} - prefix = annotations.get(self._metaKey("prefix")) - if not prefix: - raise ValueError(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") - - provider = f"{nad_name}.{namespace_name}.ovn" - subnet_name = self._kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") - subnet = { - "apiVersion": "kubeovn.io/v1", - "kind": "Subnet", - "metadata": {"name": subnet_name}, "spec": { - "protocol": "IPv4", - "provider": provider, - "vpc": vpc_name, - "cidrBlock": str(prefix), - "gateway": self._firstUsableIp(str(prefix)), - "gatewayType": "distributed", - "natOutgoing": False, - "private": False, + "config": json.dumps(config), }, } - - converted = dict(manifest) - converted["spec"] = { - "config": json.dumps( - { - "cniVersion": "0.3.1", - "type": "kube-ovn", - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", - "provider": provider, - }, - separators=(",", ":"), - ) - } - return [subnet, converted] - - def _convertWorkloadAnnotationsToKubeOvn( - self, - manifest: Dict[str, Any], - default_namespace: str, - ) -> None: - """Rewrite Multus static IP annotations for Kube-OVN attached NICs.""" - annotations = self._getPodTemplateAnnotations(manifest) - if not annotations: - return - raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") - if not isinstance(raw_networks, str): - return - try: - networks = json.loads(raw_networks) - except json.JSONDecodeError: - return - if not isinstance(networks, list): - return - - changed = False - for item in networks: - if not isinstance(item, dict): - continue - ips = item.pop("ips", None) - if not ips: - continue - nad_name, nad_namespace = self._parseNetworkSelection(item, default_namespace) - if not nad_name: - continue - ip_values = [str(ip_value).strip().split("/", 1)[0] for ip_value in ips if str(ip_value).strip()] - if not ip_values: - continue - annotations[f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address"] = ",".join(ip_values) - changed = True - - if changed: - annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) - - def _getPodTemplateAnnotations(self, manifest: Dict[str, Any]) -> Dict[str, str] | None: - """Return pod-level annotations from a workload or Pod manifest.""" - kind = manifest.get("kind") - if kind == "Pod": - metadata = manifest.setdefault("metadata", {}) - annotations = metadata.setdefault("annotations", {}) - return annotations if isinstance(annotations, dict) else None - if kind in {"Deployment", "DaemonSet", "StatefulSet", "Job"}: - template = manifest.setdefault("spec", {}).setdefault("template", {}) - metadata = template.setdefault("metadata", {}) - annotations = metadata.setdefault("annotations", {}) - return annotations if isinstance(annotations, dict) else None - return None - - def _parseNetworkSelection(self, item: Dict[str, Any], default_namespace: str) -> tuple[str, str]: - """Return (nad_name, namespace) from one Multus network selection item.""" - raw_name = str(item.get("name") or "") - namespace = str(item.get("namespace") or default_namespace) - if "/" in raw_name: - namespace, raw_name = raw_name.split("/", 1) - return raw_name, namespace - - def _firstUsableIp(self, cidr: str) -> str: - """Return the first usable IPv4 address in a CIDR block.""" - network = ipaddress.ip_network(cidr, strict=False) - if network.version != 4: - raise ValueError(f"Kube-OVN manifest generation currently supports IPv4 only: {cidr}") - hosts = network.hosts() - try: - return str(next(hosts)) - except StopIteration: - return str(network.network_address) - - def _compileNodeK8s(self, node: Node) -> Dict[str, Any]: + return yaml.safe_dump(manifest, sort_keys=False) + + def _resolveNetworkCniType(self, net: Network) -> str: + net_type = net.getType() + if net_type in {NetworkType.Local, NetworkType.CrossConnect}: + if self.__local_link_cni_type: + return self.__local_link_cni_type + if self.__scheduling_strategy == SchedulingStrategy.BY_AS_HARD and self.__cni_type in {"macvlan", "ipvlan"}: + return "bridge" + return self.__cni_type + + def _compileNodeK8s(self, node: Node) -> str: + """Compile a node to Kubernetes Deployment manifest or KubeVirt VirtualMachine. + + Includes: + - Scheduling constraints (nodeSelector based on strategy) + - Resource requests/limits + - Enhanced labels for AS and role identification + - Optional Service generation + """ + self._current_node = node + + # Check virtualization mode + # Virtualization split point: "KubeVirt" -> VirtualMachine, otherwise Deployment. + virtualization_mode = getattr(node, "getVirtualizationMode", lambda: "Container")() + if virtualization_mode == "KubeVirt": + result = self._compileNodeKubeVirt(node) + + if self.__generate_services: + node_name = self._getComposeNodeName(node).replace('_', '-').lower() + asn = str(node.getAsn()) + role = self._nodeRoleToString(node.getRole()) + labels = { + "app": node_name, + "seedemu.io/asn": asn, + "seedemu.io/role": role, + "seedemu.io/name": node.getName(), + "kubevirt.io/domain": node_name + } + service = self._compileServiceK8s(node, node_name, labels) + if service: + result += "\n---\n" + service + + return result + + # 1. Prepare Docker build context (reuse logic from Docker compiler) real_nodename = self._getRealNodeName(node) + + # Ensure the directory exists if not os.path.exists(real_nodename): mkdir(real_nodename) + # We need to change directory to generate the Dockerfile in the right place cwd = os.getcwd() chdir(real_nodename) + + # Generate Dockerfile + # This will call _addFile, which we intercept to generate the address script + image, _ = self._selectImageFor(node) dockerfile = self._computeDockerfile(node) - with open("Dockerfile", "w", encoding="utf-8") as handle: - handle.write(dockerfile) - self._patch_interface_setup_context(Path("Dockerfile")) - chdir(cwd) + with open('Dockerfile', 'w') as f: + f.write(dockerfile) - full_image_name = f"{self.__image_registry_prefix}/{real_nodename}:latest" - self.__image_entries.append( - { - "name": full_image_name, - "context": f"./{real_nodename}", - } - ) + chdir(cwd) - node_name = self._getComposeNodeName(node).replace("_", "-").lower() + # 2. Add build command + # If registry_prefix is empty, don't add slash + if self.__registry_prefix: + full_image_name = f"{self.__registry_prefix}/{real_nodename}:latest" + else: + full_image_name = f"{real_nodename}:latest" + + build_context = shlex.quote(f"./{real_nodename}") + build_image = shlex.quote(full_image_name) + build_cmd = f"seedemu_build_and_push {build_image} {build_context}" + self.__build_commands.append(build_cmd) + + # 3. Generate Deployment Manifest + node_name = self._getComposeNodeName(node).replace('_', '-').lower() # K8s names must be DNS compliant asn = str(node.getAsn()) role = self._nodeRoleToString(node.getRole()) - annotations = self._getInternetMapNodeMeta(node) - net_specs = [] - for iface in node.getInterfaces(): - net = iface.getNet() - net_name = self._getRealNetName(net).replace("_", "-").lower() - prefix_len = net.getPrefix().prefixlen - net_specs.append({"name": net_name, "ips": [f"{iface.getAddress()}/{prefix_len}"]}) - if net_specs: - annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(net_specs) - + # Compute networks annotation + annotations = {} + if self.__use_multus: + nets = [] + net_specs = [] + needs_json_annotation = False + for iface in node.getInterfaces(): + net = iface.getNet() + net_name = self._getRealNetName(net).replace('_', '-').lower() + nets.append(net_name) + resolved_cni_type = self._resolveNetworkCniType(net) + if resolved_cni_type in {"macvlan", "ipvlan"}: + prefix_len = net.getPrefix().prefixlen + net_specs.append({ + "name": net_name, + "ips": [f"{iface.getAddress()}/{prefix_len}"] + }) + needs_json_annotation = True + else: + net_specs.append({ + "name": net_name, + }) + if resolved_cni_type != self.__cni_type: + needs_json_annotation = True + + if needs_json_annotation and net_specs: + # Static IPAM requires IPs in Multus runtime config. Use JSON form annotation + # so each secondary interface gets deterministic seed-emulator addresses. + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(net_specs) + elif nets: + annotations["k8s.v1.cni.cncf.io/networks"] = ", ".join(nets) + + # Compute Envs envs = [{"name": "CONTAINER_NAME", "value": node_name}] - for opt, _scope in node.getScopedRuntimeOptions(): - envs.append({"name": opt.name.upper(), "value": str(opt.value)}) + for o, s in node.getScopedRuntimeOptions(): + envs.append({"name": o.name.upper(), "value": str(o.value)}) + + # Build enhanced labels for identification and service discovery labels = { "app": node_name, "seedemu.io/asn": asn, @@ -389,194 +748,532 @@ def _compileNodeK8s(self, node: Node) -> Dict[str, Any]: "seedemu.io/workload": "seedemu", } - return { + # Build pod spec + pod_spec = { + "containers": [{ + "name": "main", + "image": full_image_name, + "imagePullPolicy": self.__image_pull_policy, + "securityContext": {"privileged": True, "capabilities": {"add": ["ALL"]}}, + "command": ["/start.sh"], + "env": envs, + "volumeMounts": [] + }] + } + + # Add resource requests/limits if configured + if self.__default_resources: + pod_spec["containers"][0]["resources"] = self.__default_resources + + # Add scheduling constraints based on strategy + self._applySchedulingConstraints(pod_spec, node, asn, role, labels) + + manifest = { "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": node_name, "namespace": self.__namespace, - "labels": labels, + "labels": labels }, "spec": { "replicas": 1, "selector": {"matchLabels": {"app": node_name}}, "template": { - "metadata": {"labels": labels, "annotations": annotations}, - "spec": { - "containers": [ - { - "name": "main", - "image": full_image_name, - "imagePullPolicy": self.__image_pull_policy, - "securityContext": { - "privileged": True, - "capabilities": {"add": ["ALL"]}, - }, - "command": ["/start.sh"], - "env": envs, - "volumeMounts": [], - } - ] + "metadata": { + "labels": labels, + "annotations": annotations }, - }, - }, + "spec": pod_spec + } + } } - def _metaKey(self, key: str) -> str: - return f"{INTERNET_MAP_META_PREFIX}{key}" - - def _nodeInternetMapRole(self, node: Node) -> str: - _scope, obj_type, _name = node.getRegistryInfo() - if obj_type == "hnode": - return "Host" - if obj_type == "rnode": - return "Router" - if obj_type == "brdnode": - return "BorderRouter" - if obj_type == "csnode": - return "SCION Control Service" - if obj_type == "snode": - return "Emulator Service Worker" - if obj_type == "rs": - return "Route Server" - return obj_type - - def _getInternetMapNodeMeta(self, node: Node) -> Dict[str, str]: - _scope, _obj_type, name = node.getRegistryInfo() - meta = { - self._metaKey("asn"): str(node.getAsn()), - self._metaKey("nodename"): str(name), - self._metaKey("role"): self._nodeInternetMapRole(node), + result = json.dumps(manifest) + + # Generate Service if enabled + if self.__generate_services: + service = self._compileServiceK8s(node, node_name, labels) + if service: + result += "\n---\n" + service + + return result + + def _computeNodeSelector(self, node: Node, asn: str, role: str) -> Dict[str, str]: + """Compute nodeSelector based on scheduling strategy.""" + if self.__scheduling_strategy in { + SchedulingStrategy.NONE, + SchedulingStrategy.AUTO, + SchedulingStrategy.BY_AS, + SchedulingStrategy.BY_ROLE, + }: + return {} + + node_key = f"{asn}_{node.getName()}" + if node_key in self.__node_labels: + return self.__node_labels[node_key] + if asn in self.__node_labels: + return self.__node_labels[asn] + + if self.__scheduling_strategy == SchedulingStrategy.BY_AS_HARD: + raise AssertionError( + f"SchedulingStrategy.BY_AS_HARD requires an explicit nodeSelector mapping for ASN {asn}. " + f"Pass SEED_NODE_LABELS_JSON with an entry for ASN {asn}, for example: " + f'{{"{asn}": {{"kubernetes.io/hostname": "node-name"}}}}' + ) + + if self.__scheduling_strategy == SchedulingStrategy.CUSTOM: + return {} + + return {} + + def _computePodAffinity(self, asn: str, role: str) -> Dict[str, Any]: + """Compute pod affinity rules for soft grouping strategies.""" + preferred_terms: List[Dict[str, Any]] = [] + + if self.__scheduling_strategy in {SchedulingStrategy.AUTO, SchedulingStrategy.BY_AS}: + preferred_terms.append({ + "weight": 90, + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [{ + "key": "seedemu.io/asn", + "operator": "In", + "values": [asn], + }] + }, + "topologyKey": "kubernetes.io/hostname", + } + }) + + if self.__scheduling_strategy in {SchedulingStrategy.AUTO, SchedulingStrategy.BY_ROLE}: + preferred_terms.append({ + "weight": 40, + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [{ + "key": "seedemu.io/role", + "operator": "In", + "values": [role], + }] + }, + "topologyKey": "kubernetes.io/hostname", + } + }) + + if not preferred_terms: + return {} + + return { + "podAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": preferred_terms + } + } + + def _computeTopologySpreadConstraints(self) -> List[Dict[str, Any]]: + """Compute topology spread constraints for automatic balancing.""" + if self.__scheduling_strategy != SchedulingStrategy.AUTO: + return [] + + return [{ + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": { + "matchLabels": { + "seedemu.io/workload": "seedemu" + } + } + }] + + def _applySchedulingConstraints( + self, + pod_spec: Dict[str, Any], + node: Node, + asn: str, + role: str, + labels: Dict[str, str], + ) -> None: + """Apply nodeSelector/affinity/topology spread to a pod or VMI spec.""" + node_selector = self._computeNodeSelector(node, asn, role) + if node_selector: + pod_spec["nodeSelector"] = node_selector + + affinity = self._computePodAffinity(asn, role) + if affinity: + pod_spec["affinity"] = affinity + + spread = self._computeTopologySpreadConstraints() + if spread: + pod_spec["topologySpreadConstraints"] = spread + + def _compileServiceK8s(self, node: Node, node_name: str, labels: Dict[str, str]) -> Optional[str]: + """Generate a Kubernetes Service for a node if needed.""" + ports = node.getPorts() + + # Only generate service if there are exposed ports or for routers + if not ports and node.getRole() != NodeRole.Router: + return None + + service_ports = [] + + # Add ports defined on the node + for (host_port, node_port, proto) in ports: + service_ports.append({ + "name": f"port-{node_port}-{proto}", + "port": node_port, + "targetPort": node_port, + "protocol": proto.upper() + }) + + # If no ports but it's a router, add a default SSH-like port for management + if not service_ports: + return None + + service = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": f"{node_name}-svc", + "namespace": self.__namespace, + "labels": labels + }, + "spec": { + "type": self.__service_type, + "selector": {"app": node_name}, + "ports": service_ports + } } + + return json.dumps(service) - if node.getDisplayName() is not None: - meta[self._metaKey("displayname")] = str(node.getDisplayName()) - if node.getDescription() is not None: - meta[self._metaKey("description")] = str(node.getDescription()) - if len(node.getClasses()) > 0: - meta[self._metaKey("class")] = json.dumps(node.getClasses()) + def _compileNodeKubeVirt(self, node: Node) -> str: + """Compile a node to KubeVirt VirtualMachine manifest.""" + node_name = self._getComposeNodeName(node).replace('_', '-').lower() + asn = str(node.getAsn()) + role = self._nodeRoleToString(node.getRole()) - for key, value in node.getLabel().items(): - meta[self._metaKey(key)] = str(value) + # Fail-fast boundary for unsupported container-only features. + self._assertKubeVirtCompatibility(node) - for index, iface in enumerate(node.getInterfaces()): + # 1. Build Cloud-Init User Data + user_data = { + "packages": list(node.getSoftware()), + "write_files": [], + "runcmd": [] + } + + # Add files + for f in node.getFiles(): + path, content = f.get() + user_data["write_files"].append({ + "path": path, + "content": content, + "permissions": "0644" + }) + + # Generate and add replace_address.sh to configure static IPs on Multus interfaces. + # For VMs, interface names are not always eth1/eth2 (predictable naming varies), + # so the generated script falls back to auto-detection when needed. + address_script = self._generateK8sAddressScript(interface_prefix="eth") + user_data["write_files"].append({ + "path": "/usr/local/bin/replace_address.sh", + "content": address_script, + "permissions": "0755" + }) + user_data["runcmd"].append(["/usr/local/bin/replace_address.sh"]) + + # Routers need forwarding and relaxed rp_filter for asymmetric paths. + user_data["runcmd"].extend([ + ["sh", "-c", "sysctl -w net.ipv4.ip_forward=1"], + ["sh", "-c", "sysctl -w net.ipv4.conf.all.rp_filter=0"], + ["sh", "-c", "sysctl -w net.ipv4.conf.default.rp_filter=0"], + ]) + + for command in node.getBuildCommands(): + user_data["runcmd"].append(self._toCloudInitRunCommand(command, False)) + + for command in node.getBuildCommandsAtEnd(): + user_data["runcmd"].append(self._toCloudInitRunCommand(command, False)) + + # Add start commands + for cmd, fork in node.getStartCommands(): + user_data["runcmd"].append(self._toCloudInitRunCommand(cmd, fork)) + + for cmd, fork in node.getPostConfigCommands(): + user_data["runcmd"].append(self._toCloudInitRunCommand(cmd, fork)) + + cloud_init_yaml = "#cloud-config\n" + yaml.dump(user_data) + + # 2. Network Configuration + networks = [{"name": "default", "pod": {}}] + interfaces = [{"name": "default", "masquerade": {}}] + + for i, iface in enumerate(node.getInterfaces()): net = iface.getNet() - meta[self._metaKey(f"net.{index}.name")] = str(net.getName()) - meta[self._metaKey(f"net.{index}.address")] = ( - f"{iface.getAddress()}/{net.getPrefix().prefixlen}" - ) + net_name = self._getRealNetName(net).replace('_', '-').lower() + + networks.append({ + "name": f"net{i+1}", + "multus": {"networkName": net_name} + }) + interfaces.append({ + "name": f"net{i+1}", + "bridge": {} + }) + + # 3. Build Manifest + labels = { + "app": node_name, + "seedemu.io/asn": asn, + "seedemu.io/role": role, + "seedemu.io/name": node.getName(), + "seedemu.io/workload": "seedemu", + "kubevirt.io/domain": node_name + } - return meta + # Default base image (Ubuntu 22.04) + # TODO: Make this configurable via options + container_disk_image = "quay.io/containerdisks/ubuntu:22.04" + cloud_init_secret_name = f"{node_name.replace('.', '-')}-cloudinit" - def _getInternetMapNetMeta(self, net: Network) -> Dict[str, str]: - scope, _obj_type, name = net.getRegistryInfo() - meta = { - self._metaKey("type"): "global" if scope == "ix" else "local", - self._metaKey("scope"): str(scope), - self._metaKey("name"): str(name), - self._metaKey("prefix"): str(net.getPrefix()), + cloud_init_secret = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": cloud_init_secret_name, + "namespace": self.__namespace, + "labels": labels + }, + "type": "Opaque", + "stringData": { + "userdata": cloud_init_yaml + } } - if net.getDisplayName() is not None: - meta[self._metaKey("displayname")] = str(net.getDisplayName()) - if net.getDescription() is not None: - meta[self._metaKey("description")] = str(net.getDescription()) - - return meta - - def _patch_interface_setup_context(self, dockerfile_path: Path) -> None: - dockerfile = dockerfile_path.read_text(encoding="utf-8") - match = re.search(r"^COPY\s+(\S+)\s+/interface_setup\s*$", dockerfile, re.MULTILINE) - if not match: - return - - script_path = dockerfile_path.parent / match.group(1) - if not script_path.exists(): - return - - script_path.write_text( - """#!/bin/bash -set -euo pipefail - -cidr_to_net() { - ipcalc -n "$1" | sed -E -n 's/^Network: +([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\/[0-9]{1,2}) +.*/\\1/p' -} - -tmpfile="$(mktemp)" -trap 'rm -f "$tmpfile"' EXIT - -ip -j addr | jq -cr '.[]' | while read -r iface; do - ifname="$(jq -cr '.ifname' <<< "$iface")" - jq -cr '.addr_info[]?' <<< "$iface" | while read -r iaddr; do - addr="$(jq -cr '"\\(.local)/\\(.prefixlen)"' <<< "$iaddr")" - net="$(cidr_to_net "$addr")" - [ -z "$net" ] && continue - line="$(grep -F "$net" ifinfo.txt || true)" - [ -z "$line" ] && continue - new_ifname="$(cut -d: -f1 <<< "$line")" - latency="$(cut -d: -f3 <<< "$line")" - bw="$(cut -d: -f4 <<< "$line")" - loss="$(cut -d: -f5 <<< "$line")" - [ -z "$new_ifname" ] && continue - [ "$bw" = 0 ] && bw=1000000000000 - printf '%s|%s|%s|%s|%s\\n' "$ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "$tmpfile" - done -done - -i=0 -while IFS='|' read -r ifname new_ifname latency bw loss; do - [ -z "$ifname" ] && continue - tmp_ifname="seedtmp${i}" - if [ "$ifname" != "$new_ifname" ]; then - ip link set "$ifname" down - ip link set "$ifname" name "$tmp_ifname" - else - tmp_ifname="$ifname" - fi - printf '%s|%s|%s|%s|%s\\n' "$tmp_ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "${tmpfile}.stage2" - i=$((i + 1)) -done < "$tmpfile" - -while IFS='|' read -r tmp_ifname new_ifname latency bw loss; do - [ -z "$tmp_ifname" ] && continue - if [ "$tmp_ifname" != "$new_ifname" ]; then - ip link set "$tmp_ifname" name "$new_ifname" - fi - ip link set "$new_ifname" up - [ -z "$loss" ] && loss=0 - tc qdisc add dev "$new_ifname" root handle 1:0 tbf rate "${bw}bit" buffer 1000000 limit 1000 - tc qdisc add dev "$new_ifname" parent 1:0 handle 10: netem delay "${latency}ms" loss "${loss}%" -done < "${tmpfile}.stage2" - -rm -f "${tmpfile}.stage2" -""", - encoding="utf-8", - ) - - def _stage_base_image_contexts(self) -> None: - used_images = sorted(getattr(self, "_used_images", set())) - repo_root = Path(__file__).resolve().parents[2] - base_image_sources = { - "handsonsecurity/seedemu-multiarch-base:buildx-latest": os.path.join( - repo_root, "docker_images", "multiarch", "seedemu-base" - ), - "handsonsecurity/seedemu-multiarch-router:buildx-latest": os.path.join( - repo_root, "docker_images", "multiarch", "seedemu-router" - ), + manifest = { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "name": node_name, + "namespace": self.__namespace, + "labels": labels + }, + "spec": { + "runStrategy": "Always", + "template": { + "metadata": { + "labels": labels + }, + "spec": { + "domain": { + "devices": { + "disks": [ + {"name": "containerdisk", "disk": {"bus": "virtio"}}, + {"name": "cloudinitdisk", "disk": {"bus": "virtio"}} + ], + "interfaces": interfaces + }, + "resources": { + "requests": {"memory": "512M"}, + "limits": {"memory": "1024M"} + } + }, + "networks": networks, + "volumes": [ + { + "name": "containerdisk", + "containerDisk": {"image": container_disk_image} + }, + { + "name": "cloudinitdisk", + "cloudInitNoCloud": { + "secretRef": { + "name": cloud_init_secret_name + } + } + } + ] + } + } + } } - staged_any = False - for image in used_images: - src = base_image_sources.get(image) - if not src or not os.path.isdir(src): - continue - digest = md5(image.encode("utf-8")).hexdigest() - dst_root = os.path.join("base_images", digest) - os.makedirs(os.path.dirname(dst_root), exist_ok=True) - if os.path.exists(dst_root): - shutil.rmtree(dst_root) - shutil.copytree(src, dst_root) - staged_any = True - - if not staged_any and os.path.isdir("base_images"): - shutil.rmtree("base_images") + # Add scheduling constraints + self._applySchedulingConstraints(manifest["spec"]["template"]["spec"], node, asn, role, labels) + + return json.dumps(cloud_init_secret) + "\n---\n" + json.dumps(manifest) + + def _assertKubeVirtCompatibility(self, node: Node) -> None: + unsupported_features: List[str] = [] + + if node.getImportedFiles(): + unsupported_features.append("imported files") + if node.getDockerCommands(): + unsupported_features.append("docker commands") + if node.getDockerVolumes(): + unsupported_features.append("docker volumes") + + runtime_options = list(node.getScopedRuntimeOptions()) + if runtime_options: + unsupported_features.append("runtime options") + + if unsupported_features: + unsupported_list = ", ".join(unsupported_features) + raise AssertionError( + f"Node as{node.getAsn()}/{node.getName()} in KubeVirt mode has unsupported features: {unsupported_list}" + ) + + def _toCloudInitRunCommand(self, command: str, fork: bool) -> List[str]: + command_value = f"{command} &" if fork else command + return ["sh", "-c", command_value] + + def _addFile(self, path: str, content: str) -> str: + """Override _addFile to intercept the replacement script generation.""" + if path == '/replace_address.sh' and self.__use_multus and self._current_node: + # Generate K8s specific address replacement script + content = self._generateK8sAddressScript() + return super()._addFile(path, content) + + def _generateK8sAddressScript(self, interface_prefix: str = "net") -> str: + """ + Generate a script to configure static IPs on Multus interfaces. + + - In containers, Multus interfaces are typically named net1/net2/... + - In VMs (KubeVirt), interface names depend on predictable naming rules and + are not reliably eth1/eth2/... + + The generated script prefers deterministic `${prefix}1..N` when present, + otherwise it auto-detects non-default interfaces by ifindex order. + """ + node = self._current_node + iface_count = len(node.getInterfaces()) + + script = "#!/bin/bash\n" + script += "set -euo pipefail\n" + script += "# K8s Address Replacement (Multus)\n\n" + script += f"iface_prefix=\"{interface_prefix}\"\n" + script += f"expected_ifaces={iface_count}\n\n" + + script += "declare -a seed_ifaces=()\n" + script += "if [ -n \"${iface_prefix}\" ] && ip link show \"${iface_prefix}1\" >/dev/null 2>&1; then\n" + script += " for i in $(seq 1 ${expected_ifaces}); do\n" + script += " seed_ifaces+=(\"${iface_prefix}${i}\")\n" + script += " done\n" + script += "else\n" + script += " default_if=\"$(ip route show default 2>/dev/null | awk '/default/ {print $5; exit}')\"\n" + script += " [ -z \"${default_if}\" ] && default_if=\"eth0\"\n" + script += " mapfile -t seed_ifaces < <(\n" + script += " for dev in /sys/class/net/*; do\n" + script += " name=\"$(basename \"$dev\")\"\n" + script += " [ \"$name\" = \"lo\" ] && continue\n" + script += " [ \"$name\" = \"$default_if\" ] && continue\n" + script += " idx=\"$(cat \"$dev/ifindex\" 2>/dev/null || echo 9999)\"\n" + script += " echo \"$idx $name\"\n" + script += " done | sort -n | awk '{print $2}'\n" + script += " )\n" + script += "fi\n\n" + + script += "configure_iface() {\n" + script += " local slot=\"$1\"\n" + script += " local addr=\"$2\"\n" + script += " local iface=\"${seed_ifaces[$slot]:-}\"\n" + script += " if [ -z \"$iface\" ]; then\n" + script += " echo \"Missing data interface index $slot for $addr\" >&2\n" + script += " return 1\n" + script += " fi\n" + script += " echo \"Configuring $iface with $addr\"\n" + script += " ip link set \"$iface\" up\n" + script += " ip addr flush dev \"$iface\" || true\n" + script += " ip addr add \"$addr\" dev \"$iface\"\n" + script += "}\n\n" + + for i, iface in enumerate(node.getInterfaces()): + addr = iface.getAddress() + prefix_len = iface.getNet().getPrefix().prefixlen + script += f"configure_iface {i} \"{addr}/{prefix_len}\"\n" + + return script + + def attachInternetMap(self, asn: int = -1, net: str = '', ip_address: str = '', + port_forwarding: str = '') -> 'KubernetesCompiler': + """! + @brief Add the Internet Map visualization service to the Kubernetes deployment. + + @param asn the autonomous system number (not used in K8s deployment) + @param net the network name (not used in K8s deployment) + @param ip_address the IP address (not used in K8s deployment) + @param port_forwarding the port forwarding (not used in K8s deployment) + + @returns self, for chaining API calls. + """ + if not self.__internet_map_enabled: + return self + + self._log('attaching the Internet Map service to Kubernetes deployment') + source_image, target_image = self._resolveInternetMapImages() + if target_image != source_image: + self.__build_commands.append( + f"seedemu_copy_and_push_image {source_image} {target_image}" + ) + + # Generate Internet Map Kubernetes manifests + internet_map_manifest = f""" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seedemu-internet-map + namespace: {self.__namespace} + labels: + app: seedemu-internet-map +spec: + replicas: 1 + selector: + matchLabels: + app: seedemu-internet-map + template: + metadata: + labels: + app: seedemu-internet-map + spec: + containers: + - name: internet-map + image: {target_image} + ports: + - containerPort: 8080 + imagePullPolicy: {self.__image_pull_policy} + securityContext: + privileged: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +--- +apiVersion: v1 +kind: Service +metadata: + name: seedemu-internet-map-service + namespace: {self.__namespace} +spec: + type: NodePort + selector: + app: seedemu-internet-map + ports: + - port: 8080 + targetPort: 8080 +""" + + self.__manifests.append(internet_map_manifest) + self.__internet_map_enabled = False # Prevent duplicate additions + + return self + + +class NativeKubernetesCompiler(KubernetesCompiler): + """Compatibility name kept for callers using the native compiler import.""" + + pass diff --git a/seedemu/core/AutonomousSystem.py b/seedemu/core/AutonomousSystem.py index b6c3e6ebf..009d4e38b 100644 --- a/seedemu/core/AutonomousSystem.py +++ b/seedemu/core/AutonomousSystem.py @@ -11,7 +11,7 @@ from .Customizable import Customizable from .Node import promote_to_real_world_router from ipaddress import IPv4Network -from typing import Dict, List +from typing import Dict, List, Tuple, Set import requests RIS_PREFIXLIST_URL = 'https://stat.ripe.net/data/announced-prefixes/data.json' @@ -29,6 +29,7 @@ class AutonomousSystem(Printable, Graphable, Configurable, Customizable): __hosts: Dict[str, Node] __nets: Dict[str, Network] __name_servers: List[str] + __clusters: Dict[str, Tuple[Set[str], Set[str]]] # cluster_id -> (set of rr names, set of client names) def __init__(self, asn: int, subnetTemplate: str = "10.{}.0.0/16"): """! @@ -44,8 +45,104 @@ def __init__(self, asn: int, subnetTemplate: str = "10.{}.0.0/16"): self.__asn = asn self.__subnets = None if asn > 255 else list(IPv4Network(subnetTemplate.format(asn)).subnets(new_prefix = 24)) self.__name_servers = [] + self.__clusters = {} + def createCluster(self, address: str) -> 'AutonomousSystem': + """! + @brief Register an iBGP Route Reflector cluster ID for this AS. + + The cluster is created as an empty RR/client membership set. Calling + this method with an existing cluster ID is idempotent. + + @param address cluster ID rendered into BIRD's rr cluster id field. + + @returns self, for chaining API calls. + """ + if address not in self.__clusters: + # Store Route Reflector names and client names separately. + self.__clusters[address] = (set(), set()) + + return self + + def _validate_cluster_integrity(self, data: Dict[str, Tuple[Set[str], Set[str]]]): + """! + @brief Validate aggregated Route Reflector cluster membership. + A single cluster without any RR is treated as the legacy full-mesh iBGP + topology. Multi-cluster topologies, or any topology containing an RR, + are treated as RR topologies and must satisfy the RR/client contract. + + @param data mapping from cluster ID to a tuple of RR names and client + names. + """ + + # A single non-RR cluster represents the legacy full-mesh mode. + if len(data) == 1: + cid, (rrs, clients) = list(data.items())[0] + + # No RR means every router will be handled by the full-mesh renderer. + if len(rrs) == 0: + return + + # RR mode requires every active cluster to have both roles. + for cid, (rr_set, client_set) in data.items(): + + # Each RR cluster needs an RR for client reflection and RR mesh. + assert len(rr_set) > 0, ( + f"[Topology Error] AS{self.__asn} Cluster '{cid}' is invalid: " + f"Missing Route Reflector! In a multi-cluster or RR topology, every cluster must have an RR." + ) + + # A cluster with only RRs is invalid because no clients can use it. + assert len(client_set) > 0, ( + f"[Topology Error] AS{self.__asn} Cluster '{cid}' is invalid: " + f"Missing Clients! The Route Reflector {list(rr_set)} has no clients to serve." + ) + + def _aggregateBgpClusters(self): + """! + @brief Build Route Reflector cluster membership from AS and router state. + + Explicitly registered clusters provide the allowed cluster IDs. Routers + that joined a cluster are assigned to that cluster, while routers + without an explicit cluster are assigned to the default cluster. + + @returns mapping from cluster ID to a tuple of RR names and client names. + """ + # Start from explicitly registered cluster IDs, then add router roles. + merged_data = self.__clusters.copy() + + # Cluster IDs must be registered before routers can join them. + def ensure_key(cid): + assert cid in merged_data, f"Cluster ID {cid} doesn't exists in Cluster!" + + # Routers without explicit cluster membership use the legacy cluster. + default_cluster_id = "10.0.0.0" + + + # Classify each router by its RR flag and cluster membership. + for router in self.__routers.values(): + r_cid = router.getBgpClusterId() + is_rr = router.isRouteReflector() + r_name = router.getName() + + if r_cid is not None: + ensure_key(r_cid) + target_cid = r_cid + else: + if default_cluster_id not in merged_data: + # Create the implicit full-mesh cluster on first use. + merged_data[default_cluster_id] = (set(), set()) + target_cid = default_cluster_id + + if is_rr: + merged_data[target_cid][0].add(r_name) + else: + merged_data[target_cid][1].add(r_name) + print(merged_data) + self._validate_cluster_integrity(merged_data) + self.__clusters = merged_data + return self.__clusters def setNameServers(self, servers: List[str]) -> AutonomousSystem: """! @@ -384,4 +481,4 @@ def createOpenVpnRouter(self, name: str) -> Node: assert name not in self.__routers, 'Router with name {} already exists.'.format(name) self.__routers[name] = Router(name, NodeRole.OpenVpnRouter, self.__asn) - return self.__routers[name] \ No newline at end of file + return self.__routers[name] diff --git a/seedemu/core/Node.py b/seedemu/core/Node.py index 108d12f80..1646b0f30 100644 --- a/seedemu/core/Node.py +++ b/seedemu/core/Node.py @@ -18,7 +18,7 @@ from random import choice from .BaseSystem import BaseSystem -DEFAULT_SOFTWARE: List[str] = ['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat'] +DEFAULT_SOFTWARE: List[str] = ['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat-openbsd'] class File(Printable): """! @@ -1134,14 +1134,55 @@ class Router(Node): __loopback_address: str __is_border_router: bool - + __is_bgp_rr: bool + __bgp_cluster_id: str __extensions: Dict[str, RouterExtension] def __init__(self, name: str, role: NodeRole, asn: int, scope: str = None): self.__is_border_router = False self.__loopback_address = None self.__extensions = {} + self.__is_bgp_rr = False + self.__bgp_cluster_id = None super().__init__( name,role,asn,scope) + + def makeRouteReflector(self, is_rr: bool = True): + """! + @brief Mark this router as an iBGP Route Reflector. + + @param is_rr whether this router should act as a Route Reflector. + + @returns self, for chaining API calls. + """ + self.__is_bgp_rr = is_rr + return self + + def joinBgpCluster(self, cluster_id: str): + """! + @brief Assign this router to an iBGP Route Reflector cluster. + + @param cluster_id cluster ID previously registered on the AS. + + @returns self, for chaining API calls. + """ + self.__bgp_cluster_id = cluster_id + return self + + def getBgpClusterId(self) -> str | None: + """! + @brief Get the configured Route Reflector cluster ID. + + @returns cluster ID, or None if the router uses the default cluster. + """ + return self.__bgp_cluster_id + + def isRouteReflector(self) -> bool: + """! + @brief Check whether this router is configured as a Route Reflector. + + @returns True if the router acts as an iBGP Route Reflector. + """ + return self.__is_bgp_rr def hasExtension(self, name: str) -> bool: return name in self.__extensions @@ -1472,4 +1513,3 @@ def promote_to_scion_router(node: Node): extn.initScionRouter() node.installExtension(extn) return node - diff --git a/seedemu/k8sTools/README.md b/seedemu/k8sTools/README.md index 67ed4da21..a781e495e 100644 --- a/seedemu/k8sTools/README.md +++ b/seedemu/k8sTools/README.md @@ -13,7 +13,8 @@ Build infrastructure: python k8sTools.py build \ --input configKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Deploy workload: @@ -66,7 +67,7 @@ for readiness. No persistent setup/running working directory is created by default. Persistent outputs are limited to the user-requested `configK3s.yaml`, `kubeconfig.yaml`, -and the compiler output directory. +optional `inventory.yaml`, and the compiler output directory. All commands accept `--keep-temp` to leave the copied setup/running resources on disk for debugging. diff --git a/seedemu/k8sTools/k8sTools.py b/seedemu/k8sTools/k8sTools.py index e6830a1f5..79492a2a6 100644 --- a/seedemu/k8sTools/k8sTools.py +++ b/seedemu/k8sTools/k8sTools.py @@ -22,6 +22,7 @@ def build( configK3s: str | Path, kubeconfig: str | Path, *, + inventory: str | Path | None = None, keepTemp: bool = False, ) -> None: """Create infrastructure and write K3s access files. @@ -30,9 +31,10 @@ def build( inputConfig: YAML with kind=kvmOvn, physicalOvn, or multiHostKvmOvn. configK3s: Output configK3s.yaml path. kubeconfig: Output kubeconfig path. + inventory: Optional output inventory.yaml path. keepTemp: Keep temporary setup resources for debugging. """ - buildCluster(inputConfig, configK3s, kubeconfig, keep_temp=keepTemp) + buildCluster(inputConfig, configK3s, kubeconfig, inventory=inventory, keep_temp=keepTemp) def up( self, @@ -93,6 +95,7 @@ def runCli(self, argv: list[str] | None = None) -> int: build.add_argument("--input", required=True, help="input YAML with explicit kind") build.add_argument("--config-k3s", required=True, help="output configK3s.yaml") build.add_argument("--kubeconfig", required=True, help="output kubeconfig.yaml") + build.add_argument("--inventory", help="optional output inventory.yaml with node resources") build.add_argument("--keep-temp", action="store_true", help="keep temporary setup resources") up = sub.add_parser("up", help="build images and deploy workload") @@ -116,7 +119,7 @@ def runCli(self, argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "build": - self.build(args.input, args.config_k3s, args.kubeconfig, keepTemp=args.keep_temp) + self.build(args.input, args.config_k3s, args.kubeconfig, inventory=args.inventory, keepTemp=args.keep_temp) elif args.command == "up": self.up( args.folder, diff --git a/seedemu/k8sTools/resources/running/_embeddedShell.py b/seedemu/k8sTools/resources/running/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/running/_embeddedShell.py +++ b/seedemu/k8sTools/resources/running/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.py b/seedemu/k8sTools/resources/running/buildRegistryImages.py index 13a24552a..470ce8b68 100755 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.py +++ b/seedemu/k8sTools/resources/running/buildRegistryImages.py @@ -1,21 +1,184 @@ #!/usr/bin/env python3 -"""Python entrypoint for buildRegistryImages.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for buildRegistryImages with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Build SeedEMU workload images with BuildKit/buildx and push them to the +# registry selected by command-line arguments or configRunning.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +OUTPUT_DIR="" +REGISTRY_PREFIX="" +IMAGE_REGISTRY_PREFIX="" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +fi +if [ -z "${REGISTRY_PREFIX}" ]; then + REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +fi +if [ -z "${IMAGE_REGISTRY_PREFIX}" ]; then + IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +fi + +IMAGES_YAML="${OUTPUT_DIR}/images.yaml" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +cd "${OUTPUT_DIR}" + +buildImageWithBuildx() { + # Args: + # $1: full destination image reference + # $2: Docker build context directory + local image="$1" + local context="$2" + local log_file + log_file="$(mktemp)" + echo "+ DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t ${image} ${context}" + if DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" 2>&1 | tee "${log_file}"; then + rm -f "${log_file}" + return 0 + fi + + if grep -Eq 'parent snapshot .* does not exist|failed to prepare extraction snapshot' "${log_file}"; then + echo "[k8s_build] buildx export/cache failure while building ${image}; pruning buildx cache and retrying once" >&2 + docker buildx prune -af >/dev/null 2>&1 || true + rm -f "${log_file}" + DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" + return $? + fi + + rm -f "${log_file}" + return 1 +} + +firstFromImage() { + # Args: + # $1: Dockerfile path. + # Prints the first FROM image reference, handling optional --platform. + awk ' + toupper($1) == "FROM" { + if ($2 ~ /^--platform=/) { + print $3 + } else { + print $2 + } + exit + } + ' "$1" +} + +isCompilerBaseTag() { + # Args: + # $1: image reference to check. + # Returns true for Docker compiler hash tags used in generated FROM lines. + [[ "$1" =~ ^[0-9a-f]{32}(:latest)?$ ]] +} + +ensureCompilerBaseImages() { + # Build every hash-tagged base image required by generated node Dockerfiles. + # A missing base_images//Dockerfile means the compiler output is + # incomplete; without this check Docker would try pulling library/. + local tmpfile + local dockerfile + local from_image + local base_tag + local context + local missing=0 + + tmpfile="$(mktemp)" + while IFS= read -r dockerfile; do + from_image="$(firstFromImage "${dockerfile}")" + if isCompilerBaseTag "${from_image}"; then + printf '%s\n' "${from_image%:latest}" >> "${tmpfile}" + fi + done < <(find . -path './base_images/*' -prune -o -name Dockerfile -print | sort) + + while IFS= read -r base_tag; do + [ -n "${base_tag}" ] || continue + context="base_images/${base_tag}" + if [ -f "${context}/Dockerfile" ]; then + buildImageWithBuildx "${base_tag}" "${context}" + elif docker image inspect "${base_tag}" >/dev/null 2>&1; then + continue + else + echo "Missing compiler base image context: ${context}/Dockerfile" >&2 + echo "Re-run compile with the current NativeKubernetesCompiler so base_images/ is generated." >&2 + missing=1 + fi + done < <(sort -u "${tmpfile}") + + rm -f "${tmpfile}" + return "${missing}" +} + +ensureCompilerBaseImages + +python3 "${HELPER}" mapped-images \ + --images-yaml "${IMAGES_YAML}" \ + --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" \ + --registry-prefix "${REGISTRY_PREFIX}" | +while IFS=$'\t' read -r image context; do + [ -n "${image}" ] || continue + [ -n "${context}" ] || { echo "Missing context for ${image}" >&2; exit 1; } + buildImageWithBuildx "${image}" "${context}" + echo "+ docker push ${image}" + docker push "${image}" +done +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.sh b/seedemu/k8sTools/resources/running/buildRegistryImages.sh deleted file mode 100755 index fb88d64ed..000000000 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# Build SeedEMU workload images with BuildKit/buildx and push them to the -# registry selected by command-line arguments or configRunning.yaml. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" -OUTPUT_DIR="" -REGISTRY_PREFIX="" -IMAGE_REGISTRY_PREFIX="" -HELPER="${SCRIPT_DIR}/manageK8sManifest.py" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -if [ -z "${OUTPUT_DIR}" ]; then - OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" -fi -if [ -z "${REGISTRY_PREFIX}" ]; then - REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" -fi -if [ -z "${IMAGE_REGISTRY_PREFIX}" ]; then - IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" -fi - -IMAGES_YAML="${OUTPUT_DIR}/images.yaml" -OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" -cd "${OUTPUT_DIR}" - -buildImageWithBuildx() { - # Args: - # $1: full destination image reference - # $2: Docker build context directory - local image="$1" - local context="$2" - local log_file - log_file="$(mktemp)" - echo "+ DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t ${image} ${context}" - if DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" 2>&1 | tee "${log_file}"; then - rm -f "${log_file}" - return 0 - fi - - if grep -Eq 'parent snapshot .* does not exist|failed to prepare extraction snapshot' "${log_file}"; then - echo "[k8s_build] buildx export/cache failure while building ${image}; pruning buildx cache and retrying once" >&2 - docker buildx prune -af >/dev/null 2>&1 || true - rm -f "${log_file}" - DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" - return $? - fi - - rm -f "${log_file}" - return 1 -} - -firstFromImage() { - # Args: - # $1: Dockerfile path. - # Prints the first FROM image reference, handling optional --platform. - awk ' - toupper($1) == "FROM" { - if ($2 ~ /^--platform=/) { - print $3 - } else { - print $2 - } - exit - } - ' "$1" -} - -isCompilerBaseTag() { - # Args: - # $1: image reference to check. - # Returns true for Docker compiler hash tags used in generated FROM lines. - [[ "$1" =~ ^[0-9a-f]{32}(:latest)?$ ]] -} - -ensureCompilerBaseImages() { - # Build every hash-tagged base image required by generated node Dockerfiles. - # A missing base_images//Dockerfile means the compiler output is - # incomplete; without this check Docker would try pulling library/. - local tmpfile - local dockerfile - local from_image - local base_tag - local context - local missing=0 - - tmpfile="$(mktemp)" - while IFS= read -r dockerfile; do - from_image="$(firstFromImage "${dockerfile}")" - if isCompilerBaseTag "${from_image}"; then - printf '%s\n' "${from_image%:latest}" >> "${tmpfile}" - fi - done < <(find . -path './base_images/*' -prune -o -name Dockerfile -print | sort) - - while IFS= read -r base_tag; do - [ -n "${base_tag}" ] || continue - context="base_images/${base_tag}" - if [ -f "${context}/Dockerfile" ]; then - buildImageWithBuildx "${base_tag}" "${context}" - elif docker image inspect "${base_tag}" >/dev/null 2>&1; then - continue - else - echo "Missing compiler base image context: ${context}/Dockerfile" >&2 - echo "Re-run compile with the current NativeKubernetesCompiler so base_images/ is generated." >&2 - missing=1 - fi - done < <(sort -u "${tmpfile}") - - rm -f "${tmpfile}" - return "${missing}" -} - -ensureCompilerBaseImages - -python3 "${HELPER}" mapped-images \ - --images-yaml "${IMAGES_YAML}" \ - --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" \ - --registry-prefix "${REGISTRY_PREFIX}" | -while IFS=$'\t' read -r image context; do - [ -n "${image}" ] || continue - [ -n "${context}" ] || { echo "Missing context for ${image}" >&2; exit 1; } - buildImageWithBuildx "${image}" "${context}" - echo "+ docker push ${image}" - docker push "${image}" -done diff --git a/seedemu/k8sTools/resources/running/manageK8sManifest.py b/seedemu/k8sTools/resources/running/manageK8sManifest.py index 3d5cb4b5e..105cd43f5 100755 --- a/seedemu/k8sTools/resources/running/manageK8sManifest.py +++ b/seedemu/k8sTools/resources/running/manageK8sManifest.py @@ -18,6 +18,7 @@ import hashlib import ipaddress import json +import os import subprocess from pathlib import Path from typing import Any @@ -266,6 +267,13 @@ def running_context(config_path: str) -> dict[str, str]: if fabric_type in {"linux-vxlan", "vxlan", "linux_vxlan"} else "ens2" ) + attached_cni_type = str( + get_nested( + setup, + "cni.localLinkCniType", + get_nested(setup, "fabric.attachedCniType", os.environ.get("SEED_LOCAL_LINK_CNI_TYPE", "kube-ovn")), + ) + ) return { "setupConfig": str(setup_config_path), "outputDir": str(output_dir), @@ -290,6 +298,7 @@ def running_context(config_path: str) -> dict[str, str]: "masterConnection": str((master_node or {}).get("connection") or "ssh"), "cniMasterInterface": str(get_nested(setup, "cni.defaultMasterInterface", default_cni_master)), "networkBackend": network_backend, + "attachedCniType": attached_cni_type, "rolloutTimeoutSeconds": str(running.get("rolloutTimeoutSeconds") or "1800"), } @@ -409,7 +418,12 @@ def render_kustomization(args: argparse.Namespace) -> None: if network_backend in {"kube-ovn", "ovn"}: rendered_manifest = output_path.parent / "k8s.kube-ovn.yaml" if manifest_path.resolve() != rendered_manifest.resolve(): - renderKubeOvnManifest(args.manifest, rendered_manifest) + renderKubeOvnManifest( + args.manifest, + rendered_manifest, + attached_cni_type=args.attached_cni_type, + cni_master_interface=args.cni_master_interface, + ) payload = {"resources": [rendered_manifest.name], "images": images} else: payload = {"resources": ["k8s.yaml"], "images": images} @@ -419,47 +433,79 @@ def render_kustomization(args: argparse.Namespace) -> None: output_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") -def renderKubeOvnManifest(manifest_path: str, output_path: Path) -> None: - """Render a SeedEMU manifest that uses Kube-OVN secondary L2 networks. +def renderKubeOvnManifest( + manifest_path: str, + output_path: Path, + attached_cni_type: str | None = None, + cni_master_interface: str | None = None, +) -> None: + """Render a SeedEMU manifest that uses Kube-OVN managed secondary networks. Args: manifest_path: Source k8s.yaml generated by the compiler. output_path: Destination manifest referenced by kustomization.yaml. + attached_cni_type: Secondary CNI type. ``kube-ovn`` creates overlay + attached NICs; ``macvlan`` keeps macvlan dataplane and uses + Kube-OVN only for IPAM. + cni_master_interface: Optional macvlan master interface override. The compiler emits macvlan NADs plus Multus network-selection annotations - with static `ips` values in CIDR form. Kube-OVN attached NICs use provider - annotations such as `..ovn.kubernetes.io/ip_address` and - expect plain IP strings. The renderer therefore converts NADs to - type=kube-ovn, creates matching Subnets, and rewrites Pod template - annotations without changing the compiler output. + with static `ips` values in CIDR form. The renderer creates matching + Subnets and rewrites Pod template annotations without changing compiler + output. In macvlan mode, Kube-OVN acts as the centralized IPAM plugin and + avoids creating thousands of OVN logical switches/routes for scale tests. """ with open(manifest_path, "r", encoding="utf-8") as fh: docs = [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] namespace_name = findNamespaceName(docs) + attached_cni = resolveAttachedCniType(attached_cni_type) + use_ovn_attached = isKubeOvnAttachedCni(attached_cni) vpc_name = kubeOvnResourceName("vpc", namespace_name) rendered: list[dict[str, Any]] = [] vpc_written = False for doc in docs: - if doc.get("kind") == "Namespace" and not vpc_written: + if use_ovn_attached and doc.get("kind") == "Namespace" and not vpc_written: rendered.append(doc) rendered.append(kubeOvnVpc(namespace_name, vpc_name)) vpc_written = True continue if doc.get("kind") == "NetworkAttachmentDefinition": - converted = convertNetworkAttachmentToKubeOvn(doc, namespace_name, vpc_name) + converted = convertNetworkAttachmentToKubeOvn( + doc, + namespace_name, + vpc_name, + attached_cni, + cni_master_interface, + ) rendered.extend(converted) continue - convertWorkloadAnnotationsToKubeOvn(doc, namespace_name) + convertWorkloadAnnotationsToKubeOvn(doc, namespace_name, attached_cni) rendered.append(doc) - if not vpc_written: + if use_ovn_attached and not vpc_written: rendered.insert(0, kubeOvnVpc(namespace_name, vpc_name)) output_path.write_text(yaml.safe_dump_all(rendered, sort_keys=False), encoding="utf-8") +def resolveAttachedCniType(attached_cni_type: str | None = None) -> str: + """Return the configured secondary CNI type for Kube-OVN rendering.""" + value = ( + attached_cni_type + or os.environ.get("SEED_LOCAL_LINK_CNI_TYPE") + or os.environ.get("SEED_CNI_TYPE") + or "kube-ovn" + ) + return str(value).strip().lower().replace("_", "-") + + +def isKubeOvnAttachedCni(attached_cni_type: str) -> bool: + """Return true when secondary NICs should use the Kube-OVN CNI directly.""" + return attached_cni_type in {"kube-ovn", "ovn"} + + def findNamespaceName(docs: list[dict[str, Any]]) -> str: """Return the manifest namespace used by SeedEMU workload resources.""" for doc in docs: @@ -497,6 +543,8 @@ def convertNetworkAttachmentToKubeOvn( doc: dict[str, Any], default_namespace: str, vpc_name: str, + attached_cni_type: str, + cni_master_interface: str | None, ) -> list[dict[str, Any]]: """Return [Subnet, NAD] for one compiler-generated macvlan NAD. @@ -504,6 +552,8 @@ def convertNetworkAttachmentToKubeOvn( doc: Original NetworkAttachmentDefinition document. default_namespace: Namespace to use when the NAD omits metadata.namespace. vpc_name: Namespace-scoped Kube-OVN VPC name. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. """ metadata = doc.get("metadata") or {} nad_name = str(metadata.get("name") or "") @@ -515,7 +565,8 @@ def convertNetworkAttachmentToKubeOvn( if not prefix: raise SystemExit(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") - provider = f"{nad_name}.{namespace_name}.ovn" + use_ovn_attached = isKubeOvnAttachedCni(attached_cni_type) + provider = kubeOvnProviderName(nad_name, namespace_name, attached_cni_type) subnet_name = kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") subnet = { "apiVersion": "kubeovn.io/v1", @@ -524,43 +575,86 @@ def convertNetworkAttachmentToKubeOvn( "spec": { "protocol": "IPv4", "provider": provider, - "vpc": vpc_name, "cidrBlock": str(prefix), "gateway": firstUsableIp(str(prefix)), - "gatewayType": "distributed", "natOutgoing": False, "private": False, }, } + if use_ovn_attached: + subnet["spec"]["vpc"] = vpc_name + subnet["spec"]["gatewayType"] = "distributed" converted = dict(doc) - converted["spec"] = { - "config": json.dumps( - { - "cniVersion": "0.3.1", - "type": "kube-ovn", - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", - "provider": provider, - }, - separators=(",", ":"), - ) - } + converted["spec"] = {"config": renderConvertedNadConfig(doc, provider, attached_cni_type, cni_master_interface)} return [subnet, converted] -def convertWorkloadAnnotationsToKubeOvn(doc: dict[str, Any], default_namespace: str) -> None: +def renderConvertedNadConfig( + doc: dict[str, Any], + provider: str, + attached_cni_type: str, + cni_master_interface: str | None, +) -> str: + """Return NetworkAttachmentDefinition spec.config for the selected CNI. + + Args: + doc: Original NetworkAttachmentDefinition document. + provider: Kube-OVN provider string used to match the Subnet. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. + """ + if isKubeOvnAttachedCni(attached_cni_type): + config = { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + spec = doc.get("spec") or {} + raw_config = spec.get("config") + try: + config = json.loads(raw_config) if isinstance(raw_config, str) else {} + except json.JSONDecodeError: + config = {} + if not isinstance(config, dict): + config = {} + config.setdefault("cniVersion", "0.3.1") + config["type"] = attached_cni_type + if attached_cni_type == "macvlan": + if cni_master_interface: + config["master"] = cni_master_interface + config.setdefault("mode", "bridge") + config["ipam"] = { + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + +def convertWorkloadAnnotationsToKubeOvn( + doc: dict[str, Any], + default_namespace: str, + attached_cni_type: str, +) -> None: """Rewrite Multus static IP annotations for Kube-OVN attached NICs. Args: doc: Manifest document to mutate in place. Deployments and raw Pods are supported because both can contain Multus network annotations. default_namespace: Namespace to use when a network selection omits it. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. SeedEMU's compiler writes Multus `ips: ["10.x.x.x/24"]`, which is correct - for macvlan with static IPAM. Kube-OVN non-primary mode uses provider - annotations and expects plain IP values, so each `ips` item is converted to - `..ovn.kubernetes.io/ip_address: 10.x.x.x`. + for static CNI IPAM. Kube-OVN IPAM uses per-provider annotations and + expects plain IP values. Raw Pods use `ip_address`; workload templates use + `ip_pool`, otherwise kube-ovn-controller rejects Deployment pods during + address allocation. """ + kind = str(doc.get("kind") or "") annotations = getPodTemplateAnnotations(doc) if not annotations: return @@ -587,13 +681,36 @@ def convertWorkloadAnnotationsToKubeOvn(doc: dict[str, Any], default_namespace: ip_values = [stripCidr(str(ip_value)) for ip_value in ips if str(ip_value).strip()] if not ip_values: continue - annotations[f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address"] = ",".join(ip_values) + static_field = kubeOvnStaticAddressField(kind) + provider = kubeOvnProviderName(nad_name, nad_namespace, attached_cni_type) + subnet_name = kubeOvnResourceName("subnet", f"{nad_namespace}-{nad_name}") + annotations[f"{provider}.kubernetes.io/{static_field}"] = ",".join(ip_values) + annotations[f"{provider}.kubernetes.io/logical_switch"] = subnet_name changed = True if changed: annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) +def kubeOvnProviderName(nad_name: str, namespace_name: str, attached_cni_type: str) -> str: + """Return the provider name used by Kube-OVN for one NetworkAttachmentDefinition.""" + provider = f"{nad_name}.{namespace_name}" + if isKubeOvnAttachedCni(attached_cni_type): + provider = f"{provider}.ovn" + return provider + + +def kubeOvnStaticAddressField(kind: str) -> str: + """Return the Kube-OVN fixed-address annotation field for a resource kind. + + Args: + kind: Kubernetes resource kind containing the Multus annotation. + """ + if kind == "Pod": + return "ip_address" + return "ip_pool" + + def getPodTemplateAnnotations(doc: dict[str, Any]) -> dict[str, str] | None: """Return pod-level annotations from a workload or Pod document. @@ -793,6 +910,7 @@ def main() -> int: kustomization.add_argument("--registry-prefix", required=True) kustomization.add_argument("--network-backend", default="macvlan") kustomization.add_argument("--cni-master-interface", default="") + kustomization.add_argument("--attached-cni-type", default=None) kustomization.add_argument("--output", required=True) kustomization.set_defaults(func=render_kustomization) diff --git a/seedemu/k8sTools/resources/running/manageRunningStage.py b/seedemu/k8sTools/resources/running/manageRunningStage.py index a261ffb93..5fbd4addc 100755 --- a/seedemu/k8sTools/resources/running/manageRunningStage.py +++ b/seedemu/k8sTools/resources/running/manageRunningStage.py @@ -229,6 +229,8 @@ def renderKustomization(config: Path) -> dict[str, str]: values["networkBackend"], "--cni-master-interface", values["cniMasterInterface"], + "--attached-cni-type", + values["attachedCniType"], "--output", values["kustomization"], ] diff --git a/seedemu/k8sTools/resources/running/validateClusterPreflight.py b/seedemu/k8sTools/resources/running/validateClusterPreflight.py index 7efab1733..7bc9d8eb5 100755 --- a/seedemu/k8sTools/resources/running/validateClusterPreflight.py +++ b/seedemu/k8sTools/resources/running/validateClusterPreflight.py @@ -1,21 +1,226 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateClusterPreflight.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateClusterPreflight with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that compile output, kubeconfig, registry, remote Docker/buildx, and +# all Kubernetes nodes are ready before running make build/up. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +MANIFEST="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key manifest)" +IMAGES_YAML="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imagesYaml)" +KUBECONFIG_PATH="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key kubeconfig)" +REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +SSH_USER="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshUser)" +SSH_KEY="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshKey)" +MASTER_CONNECTION="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key masterConnection)" +NETWORK_BACKEND="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key networkBackend)" + +ssh_args=(-n -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8) + +fail() { + echo "[preflight][ERROR] $*" >&2 + exit 1 +} + +requireCommand() { + # Args: + # $1: command name expected in PATH + command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" +} + +requireFile() { + # Args: + # $1: required file path + [ -f "$1" ] || fail "missing file: $1" +} + +registry_ref="${REGISTRY_PREFIX#http://}" +registry_ref="${registry_ref#https://}" +registry_ref="${registry_ref%%/*}" +registry_host="${registry_ref%%:*}" +registry_url="http://${registry_ref}" + +declare -A node_config_ip=() +declare -A node_connection=() +declare -A node_user=() +declare -A node_key=() +while IFS=$'\t' read -r access_name access_ip access_connection access_user access_key; do + [ -n "${access_name}" ] || continue + node_config_ip["${access_name}"]="${access_ip}" + node_connection["${access_name}"]="${access_connection}" + node_user["${access_name}"]="${access_user}" + node_key["${access_name}"]="${access_key}" +done < <(python3 "${HELPER}" node-access --config "${CONFIG_PATH}") + +echo "=== native-k8s preflight ===" +echo "config=${CONFIG_PATH}" +echo "output_dir=${OUTPUT_DIR}" +echo "manifest=${MANIFEST}" +echo "images_yaml=${IMAGES_YAML}" +echo "kubeconfig=${KUBECONFIG_PATH}" +echo "registry_prefix=${REGISTRY_PREFIX}" +echo "image_registry_prefix=${IMAGE_REGISTRY_PREFIX}" +echo "network_backend=${NETWORK_BACKEND}" + +echo "[1/7] Local tools and compile output" +requireCommand python3 +requireCommand kubectl +requireCommand curl +requireCommand ssh +requireFile "${HELPER}" +requireFile "${MANIFEST}" +requireFile "${IMAGES_YAML}" +requireFile "${KUBECONFIG_PATH}" +[ -d "${OUTPUT_DIR}" ] || fail "missing output directory: ${OUTPUT_DIR}" +[ -s "${MANIFEST}" ] || fail "empty manifest: ${MANIFEST}" +[ -s "${IMAGES_YAML}" ] || fail "empty images yaml: ${IMAGES_YAML}" + +namespace="$(python3 "${HELPER}" namespace --manifest "${MANIFEST}")" +[ -n "${namespace}" ] || fail "cannot determine namespace from ${MANIFEST}" +image_count="$(python3 "${HELPER}" mapped-images --images-yaml "${IMAGES_YAML}" --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" --registry-prefix "${REGISTRY_PREFIX}" | wc -l)" +[ "${image_count}" -gt 0 ] || fail "no images found in ${IMAGES_YAML}" +echo "namespace=${namespace}" +echo "image_count=${image_count}" + +echo "[2/7] Kubeconfig and API server" +kubectl --kubeconfig "${KUBECONFIG_PATH}" version --client=true >/dev/null +kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide + +echo "[3/7] Node readiness" +not_ready="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ + | awk '$2 != "True" {print $1}' +)" +[ -z "${not_ready}" ] || fail "not-ready nodes: ${not_ready//$'\n'/ }" + +echo "[4/7] kube-system baseline" +kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -o wide +bad_system_pods="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods --no-headers 2>/dev/null \ + | awk '$3 !~ /^(Running|Completed)$/ {print $1 ":" $3}' +)" +[ -z "${bad_system_pods}" ] || fail "kube-system has unhealthy pods: ${bad_system_pods//$'\n'/ }" +if [ "${NETWORK_BACKEND}" = "kube-ovn" ]; then + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status deployment/kube-ovn-controller --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/kube-ovn-cni --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/ovs-ovn --timeout=300s +fi + +echo "[5/7] Namespace baseline" +if kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; then + workload_count="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" \ + get pods,deployments.apps,replicasets.apps,statefulsets.apps,daemonsets.apps,jobs.batch,cronjobs.batch \ + --ignore-not-found --no-headers 2>/dev/null | wc -l + )" + if [ "${workload_count}" -gt 0 ]; then + fail "namespace ${namespace} already has ${workload_count} workload resources; run make clean or wait for previous cleanup before make up" + fi + echo "namespace ${namespace} already exists without workload resources; continuing" +else + echo "namespace ${namespace} is absent" +fi + +echo "[6/7] Registry health" +code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" +case "${code}" in + 200|401) echo "local registry_http=${code}" ;; + *) fail "registry ${registry_url}/v2/ is not healthy from local host, http=${code}" ;; +esac + +ssh_target="${SSH_USER}@${registry_host}" +if [ "${MASTER_CONNECTION}" = "local" ]; then + echo "registry-host local registry_http=${code}" +else + requireFile "${SSH_KEY}" + ssh "${ssh_args[@]}" "${ssh_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/'" \ + | awk '{print "registry-host registry_http="$0; if ($0 != "200" && $0 != "401") exit 1}' \ + || fail "registry ${registry_url}/v2/ is not healthy from ${ssh_target}" +fi + +echo "[7/7] Remote build prerequisites and node registry reachability" +if [ "${MASTER_CONNECTION}" = "local" ]; then + sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null \ + || fail "docker/buildx is not ready on local registry host" +else + ssh "${ssh_args[@]}" "${ssh_target}" "sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null" \ + || fail "docker/buildx is not ready on ${ssh_target}" +fi +echo "registry-host docker/buildx ok" + +while IFS=$'\t' read -r node_name node_ip; do + [ -n "${node_name}" ] || continue + node_access="${node_connection[$node_name]:-}" + [ -n "${node_access}" ] || fail "node ${node_name} is not present in configK3s.yaml" + target_ip="${node_ip:-${node_config_ip[$node_name]}}" + [ -n "${target_ip}" ] || fail "cannot determine InternalIP for node ${node_name}" + if [ "${node_access}" = "local" ]; then + code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" + else + requireFile "${node_key[$node_name]}" + node_target="${node_user[$node_name]}@${target_ip}" + node_ssh_args=(-n -i "${node_key[$node_name]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=8) + code="$(ssh "${node_ssh_args[@]}" "${node_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/' || true")" + fi + printf '%s\t%s\tregistry_http=%s\n' "${node_name}" "${node_ip}" "${code}" + case "${code}" in + 200|401) ;; + *) fail "registry ${registry_url}/v2/ is not reachable from ${node_name} (${node_ip}), http=${code}" ;; + esac +done < <( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' +) + +echo "Preflight completed" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/running/validateClusterPreflight.sh b/seedemu/k8sTools/resources/running/validateClusterPreflight.sh deleted file mode 100755 index 226a52468..000000000 --- a/seedemu/k8sTools/resources/running/validateClusterPreflight.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -# Validate that compile output, kubeconfig, registry, remote Docker/buildx, and -# all Kubernetes nodes are ready before running make build/up. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" -HELPER="${SCRIPT_DIR}/manageK8sManifest.py" - -usage() { - cat <&2 - usage >&2 - exit 1 - ;; - esac -done - -OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" -MANIFEST="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key manifest)" -IMAGES_YAML="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imagesYaml)" -KUBECONFIG_PATH="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key kubeconfig)" -REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" -IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" -SSH_USER="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshUser)" -SSH_KEY="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshKey)" -MASTER_CONNECTION="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key masterConnection)" -NETWORK_BACKEND="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key networkBackend)" - -ssh_args=(-n -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8) - -fail() { - echo "[preflight][ERROR] $*" >&2 - exit 1 -} - -requireCommand() { - # Args: - # $1: command name expected in PATH - command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" -} - -requireFile() { - # Args: - # $1: required file path - [ -f "$1" ] || fail "missing file: $1" -} - -registry_ref="${REGISTRY_PREFIX#http://}" -registry_ref="${registry_ref#https://}" -registry_ref="${registry_ref%%/*}" -registry_host="${registry_ref%%:*}" -registry_url="http://${registry_ref}" - -declare -A node_config_ip=() -declare -A node_connection=() -declare -A node_user=() -declare -A node_key=() -while IFS=$'\t' read -r access_name access_ip access_connection access_user access_key; do - [ -n "${access_name}" ] || continue - node_config_ip["${access_name}"]="${access_ip}" - node_connection["${access_name}"]="${access_connection}" - node_user["${access_name}"]="${access_user}" - node_key["${access_name}"]="${access_key}" -done < <(python3 "${HELPER}" node-access --config "${CONFIG_PATH}") - -echo "=== native-k8s preflight ===" -echo "config=${CONFIG_PATH}" -echo "output_dir=${OUTPUT_DIR}" -echo "manifest=${MANIFEST}" -echo "images_yaml=${IMAGES_YAML}" -echo "kubeconfig=${KUBECONFIG_PATH}" -echo "registry_prefix=${REGISTRY_PREFIX}" -echo "image_registry_prefix=${IMAGE_REGISTRY_PREFIX}" -echo "network_backend=${NETWORK_BACKEND}" - -echo "[1/7] Local tools and compile output" -requireCommand python3 -requireCommand kubectl -requireCommand curl -requireCommand ssh -requireFile "${HELPER}" -requireFile "${MANIFEST}" -requireFile "${IMAGES_YAML}" -requireFile "${KUBECONFIG_PATH}" -[ -d "${OUTPUT_DIR}" ] || fail "missing output directory: ${OUTPUT_DIR}" -[ -s "${MANIFEST}" ] || fail "empty manifest: ${MANIFEST}" -[ -s "${IMAGES_YAML}" ] || fail "empty images yaml: ${IMAGES_YAML}" - -namespace="$(python3 "${HELPER}" namespace --manifest "${MANIFEST}")" -[ -n "${namespace}" ] || fail "cannot determine namespace from ${MANIFEST}" -image_count="$(python3 "${HELPER}" mapped-images --images-yaml "${IMAGES_YAML}" --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" --registry-prefix "${REGISTRY_PREFIX}" | wc -l)" -[ "${image_count}" -gt 0 ] || fail "no images found in ${IMAGES_YAML}" -echo "namespace=${namespace}" -echo "image_count=${image_count}" - -echo "[2/7] Kubeconfig and API server" -kubectl --kubeconfig "${KUBECONFIG_PATH}" version --client=true >/dev/null -kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide - -echo "[3/7] Node readiness" -not_ready="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ - | awk '$2 != "True" {print $1}' -)" -[ -z "${not_ready}" ] || fail "not-ready nodes: ${not_ready//$'\n'/ }" - -echo "[4/7] kube-system baseline" -kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -o wide -bad_system_pods="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods --no-headers 2>/dev/null \ - | awk '$3 !~ /^(Running|Completed)$/ {print $1 ":" $3}' -)" -[ -z "${bad_system_pods}" ] || fail "kube-system has unhealthy pods: ${bad_system_pods//$'\n'/ }" -if [ "${NETWORK_BACKEND}" = "kube-ovn" ]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s - kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status deployment/kube-ovn-controller --timeout=300s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/kube-ovn-cni --timeout=300s - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/ovs-ovn --timeout=300s -fi - -echo "[5/7] Namespace baseline" -if kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; then - fail "namespace ${namespace} already exists; run make clean or wait for previous cleanup before make up" -fi -echo "namespace ${namespace} is absent" - -echo "[6/7] Registry health" -code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" -case "${code}" in - 200|401) echo "local registry_http=${code}" ;; - *) fail "registry ${registry_url}/v2/ is not healthy from local host, http=${code}" ;; -esac - -ssh_target="${SSH_USER}@${registry_host}" -if [ "${MASTER_CONNECTION}" = "local" ]; then - echo "registry-host local registry_http=${code}" -else - requireFile "${SSH_KEY}" - ssh "${ssh_args[@]}" "${ssh_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/'" \ - | awk '{print "registry-host registry_http="$0; if ($0 != "200" && $0 != "401") exit 1}' \ - || fail "registry ${registry_url}/v2/ is not healthy from ${ssh_target}" -fi - -echo "[7/7] Remote build prerequisites and node registry reachability" -if [ "${MASTER_CONNECTION}" = "local" ]; then - sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null \ - || fail "docker/buildx is not ready on local registry host" -else - ssh "${ssh_args[@]}" "${ssh_target}" "sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null" \ - || fail "docker/buildx is not ready on ${ssh_target}" -fi -echo "registry-host docker/buildx ok" - -while IFS=$'\t' read -r node_name node_ip; do - [ -n "${node_name}" ] || continue - node_access="${node_connection[$node_name]:-}" - [ -n "${node_access}" ] || fail "node ${node_name} is not present in configK3s.yaml" - target_ip="${node_ip:-${node_config_ip[$node_name]}}" - [ -n "${target_ip}" ] || fail "cannot determine InternalIP for node ${node_name}" - if [ "${node_access}" = "local" ]; then - code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" - else - requireFile "${node_key[$node_name]}" - node_target="${node_user[$node_name]}@${target_ip}" - node_ssh_args=(-n -i "${node_key[$node_name]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=8) - code="$(ssh "${node_ssh_args[@]}" "${node_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/' || true")" - fi - printf '%s\t%s\tregistry_http=%s\n' "${node_name}" "${node_ip}" "${code}" - case "${code}" in - 200|401) ;; - *) fail "registry ${registry_url}/v2/ is not reachable from ${node_name} (${node_ip}), http=${code}" ;; - esac -done < <( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' -) - -echo "Preflight completed" diff --git a/seedemu/k8sTools/resources/setup/README.md b/seedemu/k8sTools/resources/setup/README.md index 949e716ea..b8cd8cbb4 100644 --- a/seedemu/k8sTools/resources/setup/README.md +++ b/seedemu/k8sTools/resources/setup/README.md @@ -2,12 +2,12 @@ 这个目录是 `seedemu.k8sTools` 的内部资源目录。用户通常不会直接看到它，因为 `K8sTools.build()` 会把这些资源复制到临时目录中执行，完成后只保留用户指定的 -`configK3s.yaml` 和 `kubeconfig.yaml`。 +`configK3s.yaml`、`kubeconfig.yaml`，以及可选的 `inventory.yaml`。 -所有用户侧入口都是 Python 文件；每个迁移中的系统操作入口旁边保留同名 `.sh` -资源，Python 入口只负责调用相邻 shell。这样 shell 命令序列可以正常 review 和 -`bash -n` 检查。它们仍会调用系统工具，例如 `virsh`、`docker`、`kubectl`、 -`ssh`、`ansible-playbook` 和 `helm`；这些命令才是真正的 KVM、K3s、OVN/OVS +所有用户侧和内部操作入口都是 Python 文件。部分复杂系统操作仍保留原 shell +命令序列，但正文内嵌在对应 Python 入口中，不再依赖相邻 `.sh` 资源文件。 +这些入口仍会调用系统工具，例如 `virsh`、`docker`、`kubectl`、`ssh`、 +`ansible-playbook` 和 `helm`；这些命令才是真正的 KVM、K3s、OVN/OVS 和镜像操作接口。 ## 资源分组 @@ -74,7 +74,7 @@ ovn/installKubeOvnFabric.py | `kubeconfig.yaml` | `build` 的持久输出，供 `kubectl` 和 `up/down` 使用。 | | `kvmState.yaml` | 临时 KVM 清理状态，会嵌入最终 `configK3s.yaml` 的 `k8sTools.destroy` 元数据中。 | | `multiHostKvmState.yaml` | 临时多物理机 KVM 清理状态，会嵌入最终 `configK3s.yaml`。 | -| `seedemu-k3s.inventory.yaml` | 临时解释性 inventory；当前 `k8sTools` 默认不把它作为用户主产物。 | +| `inventory.yaml` | 可选持久集群 inventory；通过 `k8sTools.py build --inventory inventory.yaml` 写出，包含节点角色、管理 IP 和 CPU/memory/disk 容量。 | ## 手动调试提示 diff --git a/seedemu/k8sTools/resources/setup/_embeddedShell.py b/seedemu/k8sTools/resources/setup/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml index d7ba828d3..16ffa5f2e 100644 --- a/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml +++ b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml @@ -140,6 +140,12 @@ retries: 30 delay: 5 + - name: Fetch K3s binary for offline worker installs + fetch: + src: /usr/local/bin/k3s + dest: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + flat: yes + - name: Install CNI plugins required by Multus (macvlan/ipvlan/static) shell: | set -euo pipefail @@ -359,7 +365,7 @@ command: - /thin_entrypoint args: - - --multus-conf-file=auto + - --multus-conf-file=/tmp/multus-conf/00-multus.conf - --multus-autoconfig-dir=/host/etc/cni/net.d - --cni-conf-dir=/host/etc/cni/net.d resources: @@ -411,7 +417,7 @@ name: multus-cni-config items: - key: cni-conf.json - path: 70-multus.conf + path: 00-multus.conf - name: Install Multus CNI shell: k3s kubectl apply -f /tmp/seedemu-multus-daemonset.yml @@ -548,9 +554,17 @@ state: restarted when: k3s_agent_dir.stat.exists and k3s_agent_config.changed + - name: Copy K3s binary prepared on master to worker + copy: + src: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + dest: /usr/local/bin/k3s + mode: "0755" + when: not k3s_agent_dir.stat.exists or seed_k3s_force_reinstall | bool + - name: Install K3s agent shell: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_install_version }} \ + curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_DOWNLOAD=true \ + INSTALL_K3S_VERSION={{ k3s_install_version }} \ INSTALL_K3S_ARTIFACT_URL={{ seed_k3s_artifact_url }} \ K3S_URL=https://{{ hostvars[groups['master'][0]]['ansible_host'] }}:6443 \ K3S_TOKEN={{ hostvars[groups['master'][0]]['k3s_server_token'] }} \ diff --git a/seedemu/k8sTools/resources/setup/applyK3sCluster.py b/seedemu/k8sTools/resources/setup/applyK3sCluster.py index 1ef62c211..962fbcd7b 100755 --- a/seedemu/k8sTools/resources/setup/applyK3sCluster.py +++ b/seedemu/k8sTools/resources/setup/applyK3sCluster.py @@ -1,21 +1,655 @@ #!/usr/bin/env python3 -"""Python entrypoint for applyK3sCluster.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for applyK3sCluster with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Build a K3s cluster from configK3s.yaml. The script installs K3s with the +# bundled Ansible playbook, creates the master registry, imports bootstrap +# images, writes kubeconfig/inventory outputs, and validates node readiness. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" +if [ -f "${SCRIPT_DIR}/ansible/k3s-install.yml" ]; then + PLAYBOOK_PATH="${SCRIPT_DIR}/ansible/k3s-install.yml" +else + PLAYBOOK_PATH="${REPO_ROOT}/ansible/k3s-install.yml" +fi +CONFIG_PATH="${1:-${SCRIPT_DIR}/configK3s.yaml}" +ansibleTimeout="3600s" + +# Public bootstrap images are prepared on the host and then copied into VMs. +# Do not make fresh VMs pull these images from the public internet during setup. +HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" +HOST_DOCKER_IO_MIRROR="docker.m.daocloud.io" +dockerPullTimeoutSeconds=180 +REGISTRY_BOOTSTRAP_IMAGE="registry:2" +MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" +K3S_SYSTEM_BOOTSTRAP_IMAGES=( + "rancher/mirrored-coredns-coredns:1.10.1" + "rancher/mirrored-metrics-server:v0.6.3" + "rancher/local-path-provisioner:v0.0.24" +) +seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" +seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" +seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" +seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" +seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" +ubuntuBuildImage="ubuntu:20.04" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveInput() { + if [ "${CONFIG_PATH}" = "-h" ] || [ "${CONFIG_PATH}" = "--help" ]; then + usage + exit 0 + fi + if [ ! -s "${CONFIG_PATH}" ]; then + echo "configK3s.yaml not found or empty: ${CONFIG_PATH}" >&2 + exit 1 + fi +} + +helper() { + local command="$1" + shift + python3 "${HELPER}" --config "${CONFIG_PATH}" "${command}" "$@" +} + +loadConfigVars() { + local vars_output + if ! vars_output="$(helper shell-vars)"; then + return 1 + fi + eval "${vars_output}" + MASTER_IS_LOCAL=false + if [ "${k3sMasterConnection:-ssh}" = "local" ]; then + MASTER_IS_LOCAL=true + fi + SSH_OPTS=( + -i "${k3sSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +loadNodeSshContext() { + # Args: + # $1: node name from configK3s.yaml. + # Reads per-node ssh.user/key and prepares NODE_SSH_OPTS for node-specific + # SSH/scp operations. Master-only operations keep using SSH_OPTS. + local name="$1" + eval "$(helper node-ssh-vars --name "${name}")" + if [ "${nodeConnection:-ssh}" = "local" ]; then + NODE_SSH_OPTS=() + return 0 + fi + [ -f "${nodeSshKey}" ] || { + echo "SSH key not found for ${name}: ${nodeSshKey}" >&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +resolveSeedEmulatorDockerDir() { + # Resolve the host path containing seedemu-base and seedemu-router + # Dockerfiles. Existing YAML value wins; common source-tree locations are + # tried as fallbacks for physical-server workflows. + local cursor="${SCRIPT_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch" \ + "${REPO_ROOT}/../seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +runMasterCommand() { + # Args: + # $1: shell command to run on the K3s master. + # Uses local execution when configK3s.yaml points the master at this host; + # otherwise uses the configured master SSH account. + local command="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + bash -lc "${command}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "${command}" + fi +} + +copyFileToMaster() { + # Args: + # $1: local source path. + # $2: destination path on the master. + local source="$1" + local target="$2" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + cp "${source}" "${target}" + else + scp "${SSH_OPTS[@]}" "${source}" "${k3sUser}@${k3sMasterIp}:${target}" >/dev/null + fi +} + +readMasterFile() { + # Args: + # $1: absolute file path to read from the master with sudo. + local path="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + sudo -n cat "${path}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "sudo -n cat '${path}'" + fi +} + +runNodeCommand() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: shell command to run on that node. + local name="$1" + local ip="$2" + local command="$3" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + bash -lc "${command}" + else + ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "${command}" + fi +} + +copyFileToNode() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: local source path. + # $4: destination path on the node. + local name="$1" + local ip="$2" + local source="$3" + local target="$4" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + cp "${source}" "${target}" + else + scp "${NODE_SSH_OPTS[@]}" "${source}" "${nodeSshUser}@${ip}:${target}" >/dev/null + fi +} + +printPlan() { + echo "config=${CONFIG_PATH}" + echo "K3s node plan:" + helper nodes-tsv | awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' + echo "master=${k3sMasterName} (${k3sMasterIp})" + echo "master_connection=${k3sMasterConnection:-ssh}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + echo "kubeconfig=${outputKubeconfig}" +} + +verifyConnectivity() { + echo "[1/10] Verifying SSH and sudo on all nodes" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + echo " ${name} ${ip}" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + runWithTimeout 12s bash -lc "echo local-ok >/dev/null" + runWithTimeout 12s sudo -n true >/dev/null + else + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ssh-ok" >/dev/null + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n true" >/dev/null + fi + done < <(helper nodes-tsv) +} + +runAnsibleInstall() { + echo "[2/10] Installing K3s via generated Ansible inventory" + local inventory_tmp playbook_tmp node_count + mkdir -p "${setupTmpDir}" + inventory_tmp="$(mktemp "${setupTmpDir}/ansible-inventory.XXXXXX.yml")" + playbook_tmp="$(mktemp "${setupTmpDir}/k3s-install.XXXXXX.yml")" + helper write-ansible-inventory --output "${inventory_tmp}" >/dev/null + node_count="$(helper nodes-tsv | wc -l | tr -d ' ')" + sed "s/ready_nodes.stdout | int >= 3/ready_nodes.stdout | int >= ${node_count}/" \ + "${PLAYBOOK_PATH}" > "${playbook_tmp}" + ANSIBLE_HOST_KEY_CHECKING=False \ + runWithTimeout "${ansibleTimeout}" \ + ansible-playbook \ + -i "${inventory_tmp}" \ + "${playbook_tmp}" \ + --ssh-common-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none" + rm -f "${inventory_tmp}" "${playbook_tmp}" +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + local mirror_ref="${HOST_DOCKER_IO_MIRROR}" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${mirror_ref}" "${image}" + else + printf '%s/library/%s\n' "${mirror_ref}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " host docker pull ${image}" >&2 + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " host docker pull ${mirror_image}" >&2 + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare image on host: ${image}" >&2 + echo "This script intentionally does not ask the VM to pull public images." >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + ensureHostDockerImage "${image}" + if [ -s "${tar_path}" ]; then + printf '%s\n' "${tar_path}" + return 0 + fi + echo " host docker save ${image} -> ${tar_path}" >&2 + docker save -o "${tar_path}" "${image}" + printf '%s\n' "${tar_path}" +} + +loadDockerImageToMaster() { + local image="$1" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + remote_tar="/tmp/$(basename "${tar_path}")" + echo " copy ${image} to ${k3sMasterName}:${remote_tar}" + copyFileToMaster "${tar_path}" "${remote_tar}" + runMasterCommand "sudo -n docker load -i '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +importK3sImageToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${node_name}" "${node_ip}" "${tar_path}" +} + +importK3sImageTarToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path="$4" + local remote_tar + remote_tar="/tmp/$(basename "${tar_path}")" + echo " import ${image} to ${node_name}" + copyFileToNode "${node_name}" "${node_ip}" "${tar_path}" "${remote_tar}" + runNodeCommand "${node_name}" "${node_ip}" \ + "sudo -n k3s ctr -n k8s.io images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +ensureRegistry() { + echo "[3/10] Ensuring private registry on master" + echo " preparing ${REGISTRY_BOOTSTRAP_IMAGE} on host and loading it into master Docker" + loadDockerImageToMaster "${REGISTRY_BOOTSTRAP_IMAGE}" + runMasterCommand " + set -euo pipefail + if ! docker buildx version >/dev/null 2>&1; then + sudo -n apt-get update >/dev/null + sudo -n apt-get install -y docker-buildx >/dev/null + fi + sudo -n docker rm -f registry >/dev/null 2>&1 || true + sudo -n docker image inspect '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + sudo -n docker run -d --network host --restart=always --name registry \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:${registryPort} '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + " +} + +ensureSeedemuHostBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedBaseSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedRouterSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareMasterWorkloadBuildImages() { + echo "[4/10] Preparing workload build base images on master Docker" + [ -d "${seedEmulatorDockerDir}/seedemu-base" ] || { + echo "Missing host seedemu base image directory: ${seedEmulatorDockerDir}/seedemu-base" >&2 + exit 1 + } + [ -d "${seedEmulatorDockerDir}/seedemu-router" ] || { + echo "Missing host seedemu router image directory: ${seedEmulatorDockerDir}/seedemu-router" >&2 + exit 1 + } + + ensureSeedemuHostBuildImages + loadDockerImageToMaster "${ubuntuBuildImage}" + loadDockerImageToMaster "${seedBaseSourceImage}" + loadDockerImageToMaster "${seedRouterSourceImage}" + + echo " ensuring stable compiler hash tags on master Docker" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' '${seedBaseHashImage}' + sudo -n docker tag '${seedRouterSourceImage}' '${seedRouterHashImage}' + " + + echo " pushing seedemu base/router images into master local registry" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' \ + '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker tag '${seedRouterSourceImage}' \ + '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + " +} + +fetchKubeconfig() { + echo "[5/10] Fetching kubeconfig" + mkdir -p "$(dirname "${outputKubeconfig}")" + readMasterFile "/etc/rancher/k3s/k3s.yaml" > "${outputKubeconfig}" + sed -i "s|127.0.0.1|${k3sMasterIp}|g" "${outputKubeconfig}" + echo "kubeconfig=${outputKubeconfig}" +} + +preloadK3sBootstrapImagesAllNodes() { + echo "[6/10] Preloading K3s system and Multus images from host into all K3s containerd nodes" + local preload_node_concurrency="${SEED_K8S_PRELOAD_NODE_CONCURRENCY:-3}" + local preload_failures=0 + if ! [[ "${preload_node_concurrency}" =~ ^[0-9]+$ ]] || [ "${preload_node_concurrency}" -lt 1 ]; then + echo "Invalid SEED_K8S_PRELOAD_NODE_CONCURRENCY: ${preload_node_concurrency}" >&2 + exit 1 + fi + echo " preload node concurrency=${preload_node_concurrency}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + saveHostImageTarball "${image}" >/dev/null + done + activePreloadJobs() { + jobs -rp | wc -l | tr -d ' ' + } + preloadNodeImages() { + local name="$1" + local ip="$2" + local image tar_path + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + tar_path="$(hostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${name}" "${ip}" "${tar_path}" + done + } + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + preloadNodeImages "${name}" "${ip}" & + while [ "$(activePreloadJobs)" -ge "${preload_node_concurrency}" ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + done < <(helper nodes-tsv) + while [ "$(activePreloadJobs)" -gt 0 ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + if [ "${preload_failures}" -ne 0 ]; then + echo "K3s bootstrap image preload failed on ${preload_failures} node(s)" >&2 + exit 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=kube-dns \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=metrics-server \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l app=local-path-provisioner \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l name=multus \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true +} + +installOvnFabricIfConfigured() { + # Install Kube-OVN after K3s and Multus are available. OVN is a Kubernetes + # CNI add-on, so it cannot be prepared in the physical-node preflight stage. + if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + return 0 + fi + echo "[7/10] Installing Kube-OVN non-primary CNI" + python3 "${SCRIPT_DIR}/ovn/installKubeOvnFabric.py" "${CONFIG_PATH}" +} + +applyNodeK3sTuning() { + local name="$1" + local ip="$2" + local remote_runner=() + echo " tuning K3s on ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${k3sMaxPods}" \ + "${kubeletRegistryQps}" \ + "${kubeletRegistryBurst}" \ + "${rebootAfterTuning}" <<'EOF_REMOTE' +set -euo pipefail +MAX_PODS="$1" +REGISTRY_QPS="$2" +REGISTRY_BURST="$3" +ASYNC_RESTART="$4" + +if [ ! -f /etc/sysctl.d/99-seed-vm-limits.conf ]; then + echo "warning: /etc/sysctl.d/99-seed-vm-limits.conf not found; run tuneVmLimits.py before large-scale experiments" >&2 +fi + +mkdir -p /etc/rancher/k3s +touch /etc/rancher/k3s/config.yaml +cfg=/etc/rancher/k3s/config.yaml +tmp=$(mktemp) +awk ' + /^[^[:space:]].*:/ { + if ($0 ~ /^(kubelet-arg|kube-apiserver-arg):/) {skip=1; next} + skip=0 + } + skip == 0 {print} +' "${cfg}" > "${tmp}" +cat >> "${tmp}" <> "${tmp}" <<'EOF_MASTER' +kube-apiserver-arg: + - "max-requests-inflight=1000" + - "max-mutating-requests-inflight=500" +EOF_MASTER +fi +cat "${tmp}" > "${cfg}" +rm -f "${tmp}" + +if [ "${ASYNC_RESTART}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true; systemctl restart containerd 2>/dev/null || true" >/dev/null 2>&1 || true +else + systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true + systemctl restart containerd 2>/dev/null || true +fi +EOF_REMOTE +} + +applyTuningAllNodes() { + echo "[8/10] Applying K3s runtime tuning" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + applyNodeK3sTuning "${name}" "${ip}" + done < <(helper nodes-tsv) +} + +verifyCluster() { + echo "[9/10] Waiting for K3s nodes" + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Ready node --all --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system rollout status daemonset/kube-multus-ds --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o wide + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tPodCIDR: "}{.spec.podCIDR}{"\n"}{end}' + echo +} + +writeOutputs() { + echo "[10/10] Writing inventory output" + helper write-cluster-inventory >/dev/null + echo "inventory=${outputInventory}" + echo "kubeconfig=${outputKubeconfig}" + echo "registry=${registryHost}:${registryPort}" +} + +main() { + requireCommand python3 + requireCommand ansible-playbook + requireCommand docker + requireCommand scp + requireCommand ssh + requireCommand kubectl + requireCommand sed + [ -f "${PLAYBOOK_PATH}" ] || { + echo "K3s Ansible playbook not found: ${PLAYBOOK_PATH}" >&2 + exit 1 + } + + resolveInput + loadConfigVars + resolveSeedEmulatorDockerDir + + if [ "${MASTER_IS_LOCAL}" != "true" ] && [ ! -f "${k3sSshKey}" ]; then + echo "SSH key not found: ${k3sSshKey}" >&2 + exit 1 + fi + + printPlan + verifyConnectivity + runAnsibleInstall + ensureRegistry + prepareMasterWorkloadBuildImages + fetchKubeconfig + preloadK3sBootstrapImagesAllNodes + installOvnFabricIfConfigured + applyTuningAllNodes + verifyCluster + writeOutputs + echo "K3s cluster is ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/applyK3sCluster.sh b/seedemu/k8sTools/resources/setup/applyK3sCluster.sh deleted file mode 100755 index 7b61ac728..000000000 --- a/seedemu/k8sTools/resources/setup/applyK3sCluster.sh +++ /dev/null @@ -1,589 +0,0 @@ -#!/usr/bin/env bash -# Build a K3s cluster from configK3s.yaml. The script installs K3s with the -# bundled Ansible playbook, creates the master registry, imports bootstrap -# images, writes kubeconfig/inventory outputs, and validates node readiness. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" -if [ -f "${SCRIPT_DIR}/ansible/k3s-install.yml" ]; then - PLAYBOOK_PATH="${SCRIPT_DIR}/ansible/k3s-install.yml" -else - PLAYBOOK_PATH="${REPO_ROOT}/ansible/k3s-install.yml" -fi -CONFIG_PATH="${1:-${SCRIPT_DIR}/configK3s.yaml}" -ansibleTimeout="3600s" - -# Public bootstrap images are prepared on the host and then copied into VMs. -# Do not make fresh VMs pull these images from the public internet during setup. -HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" -HOST_DOCKER_IO_MIRROR="docker.m.daocloud.io" -dockerPullTimeoutSeconds=180 -REGISTRY_BOOTSTRAP_IMAGE="registry:2" -MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" -K3S_SYSTEM_BOOTSTRAP_IMAGES=( - "rancher/mirrored-coredns-coredns:1.10.1" - "rancher/mirrored-metrics-server:v0.6.3" - "rancher/local-path-provisioner:v0.0.24" -) -seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" -seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" -seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" -seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" -seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" -ubuntuBuildImage="ubuntu:20.04" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -resolveInput() { - if [ "${CONFIG_PATH}" = "-h" ] || [ "${CONFIG_PATH}" = "--help" ]; then - usage - exit 0 - fi - if [ ! -s "${CONFIG_PATH}" ]; then - echo "configK3s.yaml not found or empty: ${CONFIG_PATH}" >&2 - exit 1 - fi -} - -helper() { - local command="$1" - shift - python3 "${HELPER}" --config "${CONFIG_PATH}" "${command}" "$@" -} - -loadConfigVars() { - local vars_output - if ! vars_output="$(helper shell-vars)"; then - return 1 - fi - eval "${vars_output}" - MASTER_IS_LOCAL=false - if [ "${k3sMasterConnection:-ssh}" = "local" ]; then - MASTER_IS_LOCAL=true - fi - SSH_OPTS=( - -i "${k3sSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -loadNodeSshContext() { - # Args: - # $1: node name from configK3s.yaml. - # Reads per-node ssh.user/key and prepares NODE_SSH_OPTS for node-specific - # SSH/scp operations. Master-only operations keep using SSH_OPTS. - local name="$1" - eval "$(helper node-ssh-vars --name "${name}")" - if [ "${nodeConnection:-ssh}" = "local" ]; then - NODE_SSH_OPTS=() - return 0 - fi - [ -f "${nodeSshKey}" ] || { - echo "SSH key not found for ${name}: ${nodeSshKey}" >&2 - exit 1 - } - NODE_SSH_OPTS=( - -i "${nodeSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -resolveSeedEmulatorDockerDir() { - # Resolve the host path containing seedemu-base and seedemu-router - # Dockerfiles. Existing YAML value wins; common source-tree locations are - # tried as fallbacks for physical-server workflows. - local cursor="${SCRIPT_DIR}" - while [ "${cursor}" != "/" ]; do - if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && - [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then - seedEmulatorDockerDir="${cursor}/docker_images/multiarch" - return 0 - fi - cursor="$(dirname "${cursor}")" - done - - local candidate - for candidate in \ - "${seedEmulatorDockerDir}" \ - "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ - "${HOME}/k8s/seed-emulator/docker_images/multiarch" \ - "${REPO_ROOT}/../seed-emulator/docker_images/multiarch"; do - if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then - seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" - return 0 - fi - done -} - -runMasterCommand() { - # Args: - # $1: shell command to run on the K3s master. - # Uses local execution when configK3s.yaml points the master at this host; - # otherwise uses the configured master SSH account. - local command="$1" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - bash -lc "${command}" - else - ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "${command}" - fi -} - -copyFileToMaster() { - # Args: - # $1: local source path. - # $2: destination path on the master. - local source="$1" - local target="$2" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - cp "${source}" "${target}" - else - scp "${SSH_OPTS[@]}" "${source}" "${k3sUser}@${k3sMasterIp}:${target}" >/dev/null - fi -} - -readMasterFile() { - # Args: - # $1: absolute file path to read from the master with sudo. - local path="$1" - if [ "${MASTER_IS_LOCAL}" = "true" ]; then - sudo -n cat "${path}" - else - ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "sudo -n cat '${path}'" - fi -} - -runNodeCommand() { - # Args: - # $1: node name from configK3s.yaml. - # $2: node management IP. - # $3: shell command to run on that node. - local name="$1" - local ip="$2" - local command="$3" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - bash -lc "${command}" - else - ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "${command}" - fi -} - -copyFileToNode() { - # Args: - # $1: node name from configK3s.yaml. - # $2: node management IP. - # $3: local source path. - # $4: destination path on the node. - local name="$1" - local ip="$2" - local source="$3" - local target="$4" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - cp "${source}" "${target}" - else - scp "${NODE_SSH_OPTS[@]}" "${source}" "${nodeSshUser}@${ip}:${target}" >/dev/null - fi -} - -printPlan() { - echo "config=${CONFIG_PATH}" - echo "K3s node plan:" - helper nodes-tsv | awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' - echo "master=${k3sMasterName} (${k3sMasterIp})" - echo "master_connection=${k3sMasterConnection:-ssh}" - echo "seedemu_docker_dir=${seedEmulatorDockerDir}" - echo "kubeconfig=${outputKubeconfig}" -} - -verifyConnectivity() { - echo "[1/10] Verifying SSH and sudo on all nodes" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - echo " ${name} ${ip}" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - runWithTimeout 12s bash -lc "echo local-ok >/dev/null" - runWithTimeout 12s sudo -n true >/dev/null - else - runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ssh-ok" >/dev/null - runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n true" >/dev/null - fi - done < <(helper nodes-tsv) -} - -runAnsibleInstall() { - echo "[2/10] Installing K3s via generated Ansible inventory" - local inventory_tmp playbook_tmp node_count - mkdir -p "${setupTmpDir}" - inventory_tmp="$(mktemp "${setupTmpDir}/ansible-inventory.XXXXXX.yml")" - playbook_tmp="$(mktemp "${setupTmpDir}/k3s-install.XXXXXX.yml")" - helper write-ansible-inventory --output "${inventory_tmp}" >/dev/null - node_count="$(helper nodes-tsv | wc -l | tr -d ' ')" - sed "s/ready_nodes.stdout | int >= 3/ready_nodes.stdout | int >= ${node_count}/" \ - "${PLAYBOOK_PATH}" > "${playbook_tmp}" - ANSIBLE_HOST_KEY_CHECKING=False \ - runWithTimeout "${ansibleTimeout}" \ - ansible-playbook \ - -i "${inventory_tmp}" \ - "${playbook_tmp}" \ - --ssh-common-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none" - rm -f "${inventory_tmp}" "${playbook_tmp}" -} - -imageTarName() { - printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' -} - -dockerIoMirrorRef() { - local image="$1" - local mirror_ref="${HOST_DOCKER_IO_MIRROR}" - - if [[ "${image}" == */*/* ]]; then - return 1 - fi - - if [[ "${image}" == */* ]]; then - printf '%s/%s\n' "${mirror_ref}" "${image}" - else - printf '%s/library/%s\n' "${mirror_ref}" "${image}" - fi -} - -ensureHostDockerImage() { - local image="$1" - local mirror_image="" - - if docker image inspect "${image}" >/dev/null 2>&1; then - return 0 - fi - - echo " host docker pull ${image}" >&2 - if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then - return 0 - fi - - if mirror_image="$(dockerIoMirrorRef "${image}")"; then - echo " host docker pull ${mirror_image}" >&2 - runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null - docker tag "${mirror_image}" "${image}" >/dev/null - return 0 - fi - - echo "Failed to prepare image on host: ${image}" >&2 - echo "This script intentionally does not ask the VM to pull public images." >&2 - return 1 -} - -hostImageTarball() { - local image="$1" - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" -} - -saveHostImageTarball() { - local image="$1" - local tar_path - tar_path="$(hostImageTarball "${image}")" - ensureHostDockerImage "${image}" - echo " host docker save ${image} -> ${tar_path}" >&2 - docker save -o "${tar_path}" "${image}" - printf '%s\n' "${tar_path}" -} - -loadDockerImageToMaster() { - local image="$1" - local tar_path remote_tar - tar_path="$(saveHostImageTarball "${image}")" - remote_tar="/tmp/$(basename "${tar_path}")" - echo " copy ${image} to ${k3sMasterName}:${remote_tar}" - copyFileToMaster "${tar_path}" "${remote_tar}" - runMasterCommand "sudo -n docker load -i '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" -} - -importK3sImageToNode() { - local image="$1" - local node_name="$2" - local node_ip="$3" - local tar_path remote_tar - tar_path="$(saveHostImageTarball "${image}")" - remote_tar="/tmp/$(basename "${tar_path}")" - echo " import ${image} to ${node_name}" - copyFileToNode "${node_name}" "${node_ip}" "${tar_path}" "${remote_tar}" - runNodeCommand "${node_name}" "${node_ip}" \ - "sudo -n k3s ctr -n k8s.io images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" -} - -ensureRegistry() { - echo "[3/10] Ensuring private registry on master" - echo " preparing ${REGISTRY_BOOTSTRAP_IMAGE} on host and loading it into master Docker" - loadDockerImageToMaster "${REGISTRY_BOOTSTRAP_IMAGE}" - runMasterCommand " - set -euo pipefail - if ! docker buildx version >/dev/null 2>&1; then - sudo -n apt-get update >/dev/null - sudo -n apt-get install -y docker-buildx >/dev/null - fi - sudo -n docker rm -f registry >/dev/null 2>&1 || true - sudo -n docker image inspect '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null - sudo -n docker run -d --network host --restart=always --name registry \ - -e REGISTRY_HTTP_ADDR=0.0.0.0:${registryPort} '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null - " -} - -ensureSeedemuHostBuildImages() { - ensureHostDockerImage "${ubuntuBuildImage}" - - if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then - echo " host docker build ${seedBaseSourceImage}" >&2 - DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-base" >/dev/null - fi - - if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then - echo " host docker build ${seedRouterSourceImage}" >&2 - DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-router" >/dev/null - fi - - docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null - docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null -} - -prepareMasterWorkloadBuildImages() { - echo "[4/10] Preparing workload build base images on master Docker" - [ -d "${seedEmulatorDockerDir}/seedemu-base" ] || { - echo "Missing host seedemu base image directory: ${seedEmulatorDockerDir}/seedemu-base" >&2 - exit 1 - } - [ -d "${seedEmulatorDockerDir}/seedemu-router" ] || { - echo "Missing host seedemu router image directory: ${seedEmulatorDockerDir}/seedemu-router" >&2 - exit 1 - } - - ensureSeedemuHostBuildImages - loadDockerImageToMaster "${ubuntuBuildImage}" - loadDockerImageToMaster "${seedBaseSourceImage}" - loadDockerImageToMaster "${seedRouterSourceImage}" - - echo " ensuring stable compiler hash tags on master Docker" - runMasterCommand " - set -euo pipefail - sudo -n docker tag '${seedBaseSourceImage}' '${seedBaseHashImage}' - sudo -n docker tag '${seedRouterSourceImage}' '${seedRouterHashImage}' - " - - echo " pushing seedemu base/router images into master local registry" - runMasterCommand " - set -euo pipefail - sudo -n docker tag '${seedBaseSourceImage}' \ - '127.0.0.1:${registryPort}/${seedBaseSourceImage}' - sudo -n docker push '127.0.0.1:${registryPort}/${seedBaseSourceImage}' - sudo -n docker tag '${seedRouterSourceImage}' \ - '127.0.0.1:${registryPort}/${seedRouterSourceImage}' - sudo -n docker push '127.0.0.1:${registryPort}/${seedRouterSourceImage}' - " -} - -fetchKubeconfig() { - echo "[5/10] Fetching kubeconfig" - mkdir -p "$(dirname "${outputKubeconfig}")" - readMasterFile "/etc/rancher/k3s/k3s.yaml" > "${outputKubeconfig}" - sed -i "s|127.0.0.1|${k3sMasterIp}|g" "${outputKubeconfig}" - echo "kubeconfig=${outputKubeconfig}" -} - -preloadK3sBootstrapImagesAllNodes() { - echo "[6/10] Preloading K3s system and Multus images from host into all K3s containerd nodes" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do - importK3sImageToNode "${image}" "${name}" "${ip}" - done - done < <(helper nodes-tsv) - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=kube-dns \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=metrics-server \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l app=local-path-provisioner \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l name=multus \ - --force --grace-period=0 --wait=false >/dev/null 2>&1 || true -} - -installOvnFabricIfConfigured() { - # Install Kube-OVN after K3s and Multus are available. OVN is a Kubernetes - # CNI add-on, so it cannot be prepared in the physical-node preflight stage. - if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - return 0 - fi - echo "[7/10] Installing Kube-OVN non-primary CNI" - python3 "${SCRIPT_DIR}/ovn/installKubeOvnFabric.py" "${CONFIG_PATH}" -} - -applyNodeK3sTuning() { - local name="$1" - local ip="$2" - local remote_runner=() - echo " tuning K3s on ${name} (${ip})" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - remote_runner=(sudo -n bash -s --) - else - remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) - fi - "${remote_runner[@]}" \ - "${k3sMaxPods}" \ - "${kubeletRegistryQps}" \ - "${kubeletRegistryBurst}" \ - "${rebootAfterTuning}" <<'EOF_REMOTE' -set -euo pipefail -MAX_PODS="$1" -REGISTRY_QPS="$2" -REGISTRY_BURST="$3" -ASYNC_RESTART="$4" - -if [ ! -f /etc/sysctl.d/99-seed-vm-limits.conf ]; then - echo "warning: /etc/sysctl.d/99-seed-vm-limits.conf not found; run tuneVmLimits.py before large-scale experiments" >&2 -fi - -mkdir -p /etc/rancher/k3s -touch /etc/rancher/k3s/config.yaml -cfg=/etc/rancher/k3s/config.yaml -tmp=$(mktemp) -awk ' - /^[^[:space:]].*:/ { - if ($0 ~ /^(kubelet-arg|kube-apiserver-arg):/) {skip=1; next} - skip=0 - } - skip == 0 {print} -' "${cfg}" > "${tmp}" -cat >> "${tmp}" <> "${tmp}" <<'EOF_MASTER' -kube-apiserver-arg: - - "max-requests-inflight=1000" - - "max-mutating-requests-inflight=500" -EOF_MASTER -fi -cat "${tmp}" > "${cfg}" -rm -f "${tmp}" - -if [ "${ASYNC_RESTART}" = "true" ]; then - systemd-run --on-active=2 /bin/bash -c "systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true; systemctl restart containerd 2>/dev/null || true" >/dev/null 2>&1 || true -else - systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true - systemctl restart containerd 2>/dev/null || true -fi -EOF_REMOTE -} - -applyTuningAllNodes() { - echo "[8/10] Applying K3s runtime tuning" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - applyNodeK3sTuning "${name}" "${ip}" - done < <(helper nodes-tsv) -} - -verifyCluster() { - echo "[9/10] Waiting for K3s nodes" - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Ready node --all --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" -n kube-system rollout status daemonset/kube-multus-ds --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" get nodes -o wide - kubectl --kubeconfig "${outputKubeconfig}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tPodCIDR: "}{.spec.podCIDR}{"\n"}{end}' - echo -} - -writeOutputs() { - echo "[10/10] Writing inventory output" - helper write-cluster-inventory >/dev/null - echo "inventory=${outputInventory}" - echo "kubeconfig=${outputKubeconfig}" - echo "registry=${registryHost}:${registryPort}" -} - -main() { - requireCommand python3 - requireCommand ansible-playbook - requireCommand docker - requireCommand scp - requireCommand ssh - requireCommand kubectl - requireCommand sed - [ -f "${PLAYBOOK_PATH}" ] || { - echo "K3s Ansible playbook not found: ${PLAYBOOK_PATH}" >&2 - exit 1 - } - - resolveInput - loadConfigVars - resolveSeedEmulatorDockerDir - - if [ "${MASTER_IS_LOCAL}" != "true" ] && [ ! -f "${k3sSshKey}" ]; then - echo "SSH key not found: ${k3sSshKey}" >&2 - exit 1 - fi - - printPlan - verifyConnectivity - runAnsibleInstall - ensureRegistry - prepareMasterWorkloadBuildImages - fetchKubeconfig - preloadK3sBootstrapImagesAllNodes - installOvnFabricIfConfigured - applyTuningAllNodes - verifyCluster - writeOutputs - echo "K3s cluster is ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py index 7fe6825d5..c695f6d97 100755 --- a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py +++ b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py @@ -1,21 +1,146 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyPhysicalCluster.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyPhysicalCluster with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove a physical SeedEMU K3s cluster and its optional fabric backend. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Runs K3s uninstall scripts on configured nodes, removes the local registry +# container on the master, deletes generated kubeconfig/inventory files, and +# removes the configured linux-vxlan or Kube-OVN fabric if present. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" shell-vars)" + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[cluster-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s <<<"$script" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"$script" +} + +uninstall_node=' +set -euo pipefail +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then + /usr/local/bin/k3s-agent-uninstall.sh || true +fi +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then + /usr/local/bin/k3s-uninstall.sh || true +fi +' + +remove_registry=' +set -euo pipefail +if command -v docker >/dev/null 2>&1; then + docker rm -f registry >/dev/null 2>&1 || true +fi +' + +echo "Cleaning physical K3s cluster from: $CONFIG_PATH" + +destroy_node_concurrency="${SEED_K8S_DESTROY_NODE_CONCURRENCY:-4}" +if ! [[ "$destroy_node_concurrency" =~ ^[0-9]+$ ]] || [ "$destroy_node_concurrency" -lt 1 ]; then + echo "Invalid SEED_K8S_DESTROY_NODE_CONCURRENCY: $destroy_node_concurrency" >&2 + exit 1 +fi + +activeNodeCleanJobs() { + jobs -rp | wc -l | tr -d ' ' +} + +if [ "${fabricType}" = "ovn" ] || [ "${fabricType}" = "kube-ovn" ]; then + if [ -x "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" ]; then + python3 "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" "$CONFIG_PATH" || true + else + echo "warning: OVN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +node_clean_failures=0 +echo "K3s node uninstall concurrency: ${destroy_node_concurrency}" +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "$HELPER" --config "$CONFIG_PATH" node-ssh-vars --name "$name")" + if [ "$role" = "master" ]; then + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$remove_registry" + fi + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$uninstall_node" & + while [ "$(activeNodeCleanJobs)" -ge "$destroy_node_concurrency" ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi + done +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +while [ "$(activeNodeCleanJobs)" -gt 0 ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi +done + +if [ "$node_clean_failures" -ne 0 ]; then + echo "K3s node uninstall failed on ${node_clean_failures} node(s)" >&2 + exit 1 +fi + +if [ "${fabricType}" = "linux-vxlan" ]; then + if [ -x "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" ]; then + python3 "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + else + echo "warning: VXLAN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +rm -f "$outputKubeconfig" "$outputInventory" +echo "Removed generated kubeconfig/inventory:" +echo " $outputKubeconfig" +echo " $outputInventory" +echo "Physical K3s cluster cleanup finished." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh deleted file mode 100755 index 751d6cb51..000000000 --- a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Remove a physical SeedEMU K3s cluster and its optional fabric backend. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Side effects: -# Runs K3s uninstall scripts on configured nodes, removes the local registry -# container on the master, deletes generated kubeconfig/inventory files, and -# removes the configured linux-vxlan or Kube-OVN fabric if present. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${1:-./configK3s.yaml}" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" shell-vars)" - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - - echo "[cluster-clean] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s <<<"$script" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"$script" -} - -uninstall_node=' -set -euo pipefail -if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then - /usr/local/bin/k3s-agent-uninstall.sh || true -fi -if [ -x /usr/local/bin/k3s-uninstall.sh ]; then - /usr/local/bin/k3s-uninstall.sh || true -fi -' - -remove_registry=' -set -euo pipefail -if command -v docker >/dev/null 2>&1; then - docker rm -f registry >/dev/null 2>&1 || true -fi -' - -echo "Cleaning physical K3s cluster from: $CONFIG_PATH" - -if [ "${fabricType}" = "ovn" ] || [ "${fabricType}" = "kube-ovn" ]; then - if [ -x "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" ]; then - python3 "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" "$CONFIG_PATH" || true - else - echo "warning: OVN cleanup script is not present in this generated setup directory" >&2 - fi -fi - -while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "$HELPER" --config "$CONFIG_PATH" node-ssh-vars --name "$name")" - if [ "$role" = "master" ]; then - runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$remove_registry" - fi - runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$uninstall_node" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) - -if [ "${fabricType}" = "linux-vxlan" ]; then - if [ -x "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" ]; then - python3 "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - else - echo "warning: VXLAN cleanup script is not present in this generated setup directory" >&2 - fi -fi - -rm -f "$outputKubeconfig" "$outputInventory" -echo "Removed generated kubeconfig/inventory:" -echo " $outputKubeconfig" -echo " $outputInventory" -echo "Physical K3s cluster cleanup finished." diff --git a/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py index feee5bcb6..fc6f27a93 100755 --- a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py +++ b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py @@ -1,21 +1,476 @@ #!/usr/bin/env python3 -"""Python entrypoint for createKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for createKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create or start KVM VMs from kvm.yaml. The script resolves non-conflicting +# VM names/IPs/MACs, writes user-facing configK3s.yaml plus internal +# kvmState.yaml, and waits for SSH. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +kvmBaseImageReuseMode="copy" +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 + +eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + +SSH_PUB_KEY="" +EXISTING_VMS_TSV="" +PLANNED_NODES_TSV="" +REUSING_KVM_STATE_PLAN="false" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${EXISTING_VMS_TSV}" ] && rm -f "${EXISTING_VMS_TSV}" || true + [ -n "${PLANNED_NODES_TSV}" ] && rm -f "${PLANNED_NODES_TSV}" || true +} + +domainExists() { + virsh dominfo "$1" >/dev/null 2>&1 +} + +domainRunning() { + virsh domstate "$1" 2>/dev/null | grep -qi "running" +} + +prepareDirs() { + mkdir -p "${kvmStorageDir}/base" + mkdir -p "${kvmDiskDir}" + mkdir -p "${kvmCloudInitDir}" + mkdir -p "$(dirname "${kvmBaseImagePath}")" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +ensureNetwork() { + if ! virsh net-info "${kvmNetwork}" >/dev/null 2>&1; then + echo "Libvirt network not found: ${kvmNetwork}" >&2 + exit 1 + fi + virsh net-start "${kvmNetwork}" >/dev/null 2>&1 || true + virsh net-autostart "${kvmNetwork}" >/dev/null 2>&1 || true +} + +collectExistingVms() { + EXISTING_VMS_TSV="$(mktemp "${kvmStorageDir}/existing-vms.XXXXXX.tsv")" + PLANNED_NODES_TSV="$(mktemp "${kvmStorageDir}/planned-vms.XXXXXX.tsv")" + + if [ -f "${outputKvmState}" ]; then + if ! python3 "${HELPER}" "${CONFIG_PATH}" validate-kvm-state --state "${outputKvmState}"; then + echo "Refusing to reuse stale KVM state: ${outputKvmState}" >&2 + echo "Remove it only if you intentionally want this setup directory to generate a new VM plan." >&2 + exit 1 + fi + REUSING_KVM_STATE_PLAN="true" + python3 "${HELPER}" "${CONFIG_PATH}" state-nodes-tsv --state "${outputKvmState}" > "${PLANNED_NODES_TSV}" + echo "Using existing KVM state: ${outputKvmState}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" + return + fi + + virsh list --all --name 2>/dev/null | awk 'NF {print $1 "\t\t"}' >> "${EXISTING_VMS_TSV}" + + virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + print("\t".join([ + host.get("name") or "", + host.get("ip") or "", + (host.get("mac") or "").lower(), + ])) +' >> "${EXISTING_VMS_TSV}" + + virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk ' + NR <= 2 {next} + NF >= 5 { + name=$6 + if (name == "-" || name == "") name="" + ip=$5 + sub("/.*", "", ip) + print name "\t" ip "\t" tolower($3) + } + ' >> "${EXISTING_VMS_TSV}" || true + + python3 "${HELPER}" "${CONFIG_PATH}" nodes-tsv --existing-tsv "${EXISTING_VMS_TSV}" > "${PLANNED_NODES_TSV}" + echo "Planned KVM nodes:" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" +} + +downloadBaseImage() { + if [ -f "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Using existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Using existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + echo "Downloading Ubuntu cloud image to ${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +loadSshPubkey() { + if [ -f "${sshKey}.pub" ]; then + SSH_PUB_KEY="$(tr -d '\n' < "${sshKey}.pub")" + return + fi + if [ -f "${sshKey}" ]; then + SSH_PUB_KEY="$(ssh-keygen -y -f "${sshKey}" 2>/dev/null | tr -d '\n')" + [ -n "${SSH_PUB_KEY}" ] && return + fi + echo "Cannot read SSH key. Set ssh.key in ${CONFIG_PATH} to a valid private key." >&2 + exit 1 +} + +updateDhcpHost() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + virsh net-update "${kvmNetwork}" add ip-dhcp-host "${host_xml}" --live --config >/dev/null +} + +createVmCloudInit() { + local vm_name="$1" + local vm_dir="${kvmCloudInitDir}/${vm_name}" + mkdir -p "${vm_dir}" + + cat > "${vm_dir}/user-data.yaml" < "${vm_dir}/meta-data.yaml" </dev/null +} + +createOrStartVm() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vcpus="$4" + local memory_mb="$5" + local disk_gb="$6" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + local vm_cloud_dir="${kvmCloudInitDir}/${vm_name}" + + if domainExists "${vm_name}"; then + validateExistingDomainBelongsToSetup "${vm_name}" + if [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "Refusing to reuse existing VM '${vm_name}'. Set kvm.allowExisting: true only if this is intentional." >&2 + exit 1 + fi + echo "VM exists: ${vm_name}" + if ! domainRunning "${vm_name}"; then + virsh start "${vm_name}" >/dev/null + fi + return + fi + + echo "Creating VM: ${vm_name} ip=${vm_ip} vcpus=${vcpus} memory_mb=${memory_mb} disk_gb=${disk_gb}" + virt-install \ + --name "${vm_name}" \ + --memory "${memory_mb}" \ + --vcpus "${vcpus}" \ + --cpu host-passthrough \ + --import \ + --os-variant generic \ + --network "network=${kvmNetwork},model=virtio,mac=${vm_mac}" \ + --disk "path=${vm_disk},format=qcow2,bus=virtio" \ + --graphics none \ + --noautoconsole \ + --cloud-init "user-data=${vm_cloud_dir}/user-data.yaml,meta-data=${vm_cloud_dir}/meta-data.yaml" >/dev/null +} + +validateExistingDomainBelongsToSetup() { + # Args: + # $1: VM domain name that already exists. + # Uses kvmDiskDir from kvm.yaml to make stale kvmState.yaml files fail safe. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for existing VM ${vm_name}; refusing to reuse it." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to reuse existing VM ${vm_name}: disk is outside this setup diskDir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +checkNetworkConflict() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + + virsh net-dumpxml "${kvmNetwork}" | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +name, ip, mac = sys.argv[1], sys.argv[2], sys.argv[3].lower() +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + h_name = host.get("name") or "" + h_ip = host.get("ip") or "" + h_mac = (host.get("mac") or "").lower() + same_host = h_name == name and h_ip == ip and h_mac == mac + if h_ip == ip and not same_host: + raise SystemExit(f"IP {ip} is already reserved by host name={h_name} mac={h_mac}") + if h_mac == mac and not same_host: + raise SystemExit(f"MAC {mac} is already reserved by host name={h_name} ip={h_ip}") +' "${vm_name}" "${vm_ip}" "${vm_mac}" + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk -v ip="${vm_ip}" -v mac="${vm_mac,,}" -v name="${vm_name}" ' + NR <= 2 {next} + { + lease_mac=tolower($3); lease_ip=$5; sub("/.*", "", lease_ip); lease_name=$6 + if ((lease_ip == ip || lease_mac == mac) && !(lease_ip == ip && lease_mac == mac && lease_name == name)) { + print "DHCP lease conflict: name=" lease_name " mac=" lease_mac " ip=" lease_ip > "/dev/stderr" + exit 1 + } + } + '; then + return + fi + exit 1 +} + +checkCreateConflicts() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + + if domainExists "${vm_name}" && [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "VM name conflict: ${vm_name} already exists. Choose a new name or set kvm.allowExisting: true intentionally." >&2 + exit 1 + fi + if [ -e "${vm_disk}" ] && ! domainExists "${vm_name}" ]; then + echo "Disk conflict: ${vm_disk} already exists but VM ${vm_name} is not defined." >&2 + exit 1 + fi + checkNetworkConflict "${vm_name}" "${vm_ip}" "${vm_mac}" +} + +waitForSsh() { + local vm_name="$1" + local vm_ip="$2" + local elapsed=0 + while [ "${elapsed}" -lt "${kvmBootTimeoutSeconds}" ]; do + if ssh -o StrictHostKeyChecking=no \ + -n \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -o ConnectTimeout=5 \ + -i "${sshKey}" \ + "${sshUser}@${vm_ip}" "echo ok" >/dev/null 2>&1; then + echo "SSH ready: ${vm_name} (${vm_ip})" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + echo "Timeout waiting for SSH on ${vm_name} (${vm_ip})" >&2 + return 1 +} + +writeK3sConfig() { + # Args: none. + # Writes the user-facing configK3s.yaml from the transient planned node + # list. It intentionally omits KVM resource details and output paths. + python3 "${HELPER}" "${CONFIG_PATH}" write-k3s-config \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputK3sConfig}" + echo "K3s config: ${outputK3sConfig}" +} + +writeKvmState() { + # Args: none. + # Writes internal kvmState.yaml with VM MAC/resource/disk metadata. Cleanup + # uses this file instead of overloading the user-facing configK3s.yaml. + python3 "${HELPER}" "${CONFIG_PATH}" write-kvm-state \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputKvmState}" + echo "KVM state: ${outputKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand virsh + requireCommand virt-install + requireCommand qemu-img + requireCommand curl + requireCommand ssh + requireCommand ssh-keygen + requireCommand awk + + [ -f "${sshKey}" ] || { + echo "SSH key not found: ${sshKey}" >&2 + exit 1 + } + trap cleanupTmp EXIT + + prepareDirs + ensureNetwork + collectExistingVms + downloadBaseImage + loadSshPubkey + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + checkCreateConflicts "${name}" "${ip}" "${mac}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + updateDhcpHost "${name}" "${ip}" "${mac}" + createVmCloudInit "${name}" + createVmDisk "${name}" "${disk_gb}" + createOrStartVm "${name}" "${ip}" "${mac}" "${vcpus}" "${memory_mb}" "${disk_gb}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < "${PLANNED_NODES_TSV}" + + if [ "${kvmSkipK3sConfig}" = "true" ]; then + echo "Skipping host-local configK3s.yaml generation because kvm.skipK3sConfig=true." + else + writeK3sConfig + fi + writeKvmState + echo "KVM VMs are ready." + echo "Next step: ${SCRIPT_DIR}/tuneVmLimits.py ${outputK3sConfig}" +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh deleted file mode 100755 index 90d2a5bfe..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.sh +++ /dev/null @@ -1,457 +0,0 @@ -#!/usr/bin/env bash -# Create or start KVM VMs from kvm.yaml. The script resolves non-conflicting -# VM names/IPs/MACs, writes user-facing configK3s.yaml plus internal -# kvmState.yaml, and waits for SSH. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" -kvmBaseImageSearchDirs="${HOME}/k8s/output" -kvmBaseImageReuseMode="copy" -baseImageLowSpeedSeconds=60 -baseImageLowSpeedLimit=1024 - -eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" - -SSH_PUB_KEY="" -EXISTING_VMS_TSV="" -PLANNED_NODES_TSV="" -REUSING_KVM_STATE_PLAN="false" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -cleanupTmp() { - [ -n "${EXISTING_VMS_TSV}" ] && rm -f "${EXISTING_VMS_TSV}" || true - [ -n "${PLANNED_NODES_TSV}" ] && rm -f "${PLANNED_NODES_TSV}" || true -} - -domainExists() { - virsh dominfo "$1" >/dev/null 2>&1 -} - -domainRunning() { - virsh domstate "$1" 2>/dev/null | grep -qi "running" -} - -prepareDirs() { - mkdir -p "${kvmStorageDir}/base" - mkdir -p "${kvmDiskDir}" - mkdir -p "${kvmCloudInitDir}" - mkdir -p "$(dirname "${kvmBaseImagePath}")" -} - -validBaseImage() { - # Args: - # $1: qcow2 image path to validate. - local image_path="$1" - [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 -} - -downloadBaseImageFile() { - # Args: - # $1: URL, $2: destination path. - local image_url="$1" - local image_path="$2" - local partial_path="${image_path}.part" - mkdir -p "$(dirname "${image_path}")" - curl -fL \ - --retry 5 \ - --retry-delay 5 \ - --retry-all-errors \ - --connect-timeout 20 \ - --speed-time "${baseImageLowSpeedSeconds}" \ - --speed-limit "${baseImageLowSpeedLimit}" \ - -C - \ - -o "${partial_path}" \ - "${image_url}" - validBaseImage "${partial_path}" || { - echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 - return 1 - } - mv -f "${partial_path}" "${image_path}" -} - -ensureNetwork() { - if ! virsh net-info "${kvmNetwork}" >/dev/null 2>&1; then - echo "Libvirt network not found: ${kvmNetwork}" >&2 - exit 1 - fi - virsh net-start "${kvmNetwork}" >/dev/null 2>&1 || true - virsh net-autostart "${kvmNetwork}" >/dev/null 2>&1 || true -} - -collectExistingVms() { - EXISTING_VMS_TSV="$(mktemp "${kvmStorageDir}/existing-vms.XXXXXX.tsv")" - PLANNED_NODES_TSV="$(mktemp "${kvmStorageDir}/planned-vms.XXXXXX.tsv")" - - if [ -f "${outputKvmState}" ]; then - if ! python3 "${HELPER}" "${CONFIG_PATH}" validate-kvm-state --state "${outputKvmState}"; then - echo "Refusing to reuse stale KVM state: ${outputKvmState}" >&2 - echo "Remove it only if you intentionally want this setup directory to generate a new VM plan." >&2 - exit 1 - fi - REUSING_KVM_STATE_PLAN="true" - python3 "${HELPER}" "${CONFIG_PATH}" state-nodes-tsv --state "${outputKvmState}" > "${PLANNED_NODES_TSV}" - echo "Using existing KVM state: ${outputKvmState}" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" - return - fi - - virsh list --all --name 2>/dev/null | awk 'NF {print $1 "\t\t"}' >> "${EXISTING_VMS_TSV}" - - virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c ' -import sys -import xml.etree.ElementTree as ET - -root = ET.fromstring(sys.stdin.read()) -for host in root.findall(".//host"): - print("\t".join([ - host.get("name") or "", - host.get("ip") or "", - (host.get("mac") or "").lower(), - ])) -' >> "${EXISTING_VMS_TSV}" - - virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk ' - NR <= 2 {next} - NF >= 5 { - name=$6 - if (name == "-" || name == "") name="" - ip=$5 - sub("/.*", "", ip) - print name "\t" ip "\t" tolower($3) - } - ' >> "${EXISTING_VMS_TSV}" || true - - python3 "${HELPER}" "${CONFIG_PATH}" nodes-tsv --existing-tsv "${EXISTING_VMS_TSV}" > "${PLANNED_NODES_TSV}" - echo "Planned KVM nodes:" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" -} - -downloadBaseImage() { - if [ -f "${kvmBaseImagePath}" ]; then - if validBaseImage "${kvmBaseImagePath}"; then - return - fi - echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" - rm -f "${kvmBaseImagePath}" - fi - if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then - echo "Using existing base image from ${kvmLegacyBaseImagePath}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - fi - return - fi - local output_image="" - local search_dir="" - for search_dir in ${kvmBaseImageSearchDirs}; do - [ -d "${search_dir}" ] || continue - output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" - if [ -n "${output_image}" ]; then - echo "Using existing base image from ${output_image}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${output_image}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" - fi - return - fi - done - echo "Downloading Ubuntu cloud image to ${kvmBaseImagePath}" - downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" -} - -loadSshPubkey() { - if [ -f "${sshKey}.pub" ]; then - SSH_PUB_KEY="$(tr -d '\n' < "${sshKey}.pub")" - return - fi - if [ -f "${sshKey}" ]; then - SSH_PUB_KEY="$(ssh-keygen -y -f "${sshKey}" 2>/dev/null | tr -d '\n')" - [ -n "${SSH_PUB_KEY}" ] && return - fi - echo "Cannot read SSH key. Set ssh.key in ${CONFIG_PATH} to a valid private key." >&2 - exit 1 -} - -updateDhcpHost() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local host_xml="" - virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true - virsh net-update "${kvmNetwork}" add ip-dhcp-host "${host_xml}" --live --config >/dev/null -} - -createVmCloudInit() { - local vm_name="$1" - local vm_dir="${kvmCloudInitDir}/${vm_name}" - mkdir -p "${vm_dir}" - - cat > "${vm_dir}/user-data.yaml" < "${vm_dir}/meta-data.yaml" </dev/null -} - -createOrStartVm() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local vcpus="$4" - local memory_mb="$5" - local disk_gb="$6" - local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" - local vm_cloud_dir="${kvmCloudInitDir}/${vm_name}" - - if domainExists "${vm_name}"; then - validateExistingDomainBelongsToSetup "${vm_name}" - if [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then - echo "Refusing to reuse existing VM '${vm_name}'. Set kvm.allowExisting: true only if this is intentional." >&2 - exit 1 - fi - echo "VM exists: ${vm_name}" - if ! domainRunning "${vm_name}"; then - virsh start "${vm_name}" >/dev/null - fi - return - fi - - echo "Creating VM: ${vm_name} ip=${vm_ip} vcpus=${vcpus} memory_mb=${memory_mb} disk_gb=${disk_gb}" - virt-install \ - --name "${vm_name}" \ - --memory "${memory_mb}" \ - --vcpus "${vcpus}" \ - --cpu host-passthrough \ - --import \ - --os-variant generic \ - --network "network=${kvmNetwork},model=virtio,mac=${vm_mac}" \ - --disk "path=${vm_disk},format=qcow2,bus=virtio" \ - --graphics none \ - --noautoconsole \ - --cloud-init "user-data=${vm_cloud_dir}/user-data.yaml,meta-data=${vm_cloud_dir}/meta-data.yaml" >/dev/null -} - -validateExistingDomainBelongsToSetup() { - # Args: - # $1: VM domain name that already exists. - # Uses kvmDiskDir from kvm.yaml to make stale kvmState.yaml files fail safe. - local vm_name="$1" - local disk_path="" - disk_path="$( - virsh domblklist "${vm_name}" --details 2>/dev/null \ - | awk '$2 == "disk" && $4 != "-" {print $4; exit}' - )" - if [ -z "${disk_path}" ]; then - echo "Cannot determine disk path for existing VM ${vm_name}; refusing to reuse it." >&2 - exit 1 - fi - case "${disk_path}" in - "${kvmDiskDir}"/*) ;; - *) - echo "Refusing to reuse existing VM ${vm_name}: disk is outside this setup diskDir." >&2 - echo " vm_disk=${disk_path}" >&2 - echo " setup_disk_dir=${kvmDiskDir}" >&2 - echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 - exit 1 - ;; - esac -} - -checkNetworkConflict() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - - virsh net-dumpxml "${kvmNetwork}" | python3 -c ' -import sys -import xml.etree.ElementTree as ET - -name, ip, mac = sys.argv[1], sys.argv[2], sys.argv[3].lower() -root = ET.fromstring(sys.stdin.read()) -for host in root.findall(".//host"): - h_name = host.get("name") or "" - h_ip = host.get("ip") or "" - h_mac = (host.get("mac") or "").lower() - same_host = h_name == name and h_ip == ip and h_mac == mac - if h_ip == ip and not same_host: - raise SystemExit(f"IP {ip} is already reserved by host name={h_name} mac={h_mac}") - if h_mac == mac and not same_host: - raise SystemExit(f"MAC {mac} is already reserved by host name={h_name} ip={h_ip}") -' "${vm_name}" "${vm_ip}" "${vm_mac}" - - if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk -v ip="${vm_ip}" -v mac="${vm_mac,,}" -v name="${vm_name}" ' - NR <= 2 {next} - { - lease_mac=tolower($3); lease_ip=$5; sub("/.*", "", lease_ip); lease_name=$6 - if ((lease_ip == ip || lease_mac == mac) && !(lease_ip == ip && lease_mac == mac && lease_name == name)) { - print "DHCP lease conflict: name=" lease_name " mac=" lease_mac " ip=" lease_ip > "/dev/stderr" - exit 1 - } - } - '; then - return - fi - exit 1 -} - -checkCreateConflicts() { - local vm_name="$1" - local vm_ip="$2" - local vm_mac="$3" - local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" - - if domainExists "${vm_name}" && [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then - echo "VM name conflict: ${vm_name} already exists. Choose a new name or set kvm.allowExisting: true intentionally." >&2 - exit 1 - fi - if [ -e "${vm_disk}" ] && ! domainExists "${vm_name}" ]; then - echo "Disk conflict: ${vm_disk} already exists but VM ${vm_name} is not defined." >&2 - exit 1 - fi - checkNetworkConflict "${vm_name}" "${vm_ip}" "${vm_mac}" -} - -waitForSsh() { - local vm_name="$1" - local vm_ip="$2" - local elapsed=0 - while [ "${elapsed}" -lt "${kvmBootTimeoutSeconds}" ]; do - if ssh -o StrictHostKeyChecking=no \ - -n \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -o ConnectTimeout=5 \ - -i "${sshKey}" \ - "${sshUser}@${vm_ip}" "echo ok" >/dev/null 2>&1; then - echo "SSH ready: ${vm_name} (${vm_ip})" - return 0 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done - echo "Timeout waiting for SSH on ${vm_name} (${vm_ip})" >&2 - return 1 -} - -writeK3sConfig() { - # Args: none. - # Writes the user-facing configK3s.yaml from the transient planned node - # list. It intentionally omits KVM resource details and output paths. - python3 "${HELPER}" "${CONFIG_PATH}" write-k3s-config \ - --nodes-tsv "${PLANNED_NODES_TSV}" \ - --output "${outputK3sConfig}" - echo "K3s config: ${outputK3sConfig}" -} - -writeKvmState() { - # Args: none. - # Writes internal kvmState.yaml with VM MAC/resource/disk metadata. Cleanup - # uses this file instead of overloading the user-facing configK3s.yaml. - python3 "${HELPER}" "${CONFIG_PATH}" write-kvm-state \ - --nodes-tsv "${PLANNED_NODES_TSV}" \ - --output "${outputKvmState}" - echo "KVM state: ${outputKvmState}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand virsh - requireCommand virt-install - requireCommand qemu-img - requireCommand curl - requireCommand ssh - requireCommand ssh-keygen - requireCommand awk - - [ -f "${sshKey}" ] || { - echo "SSH key not found: ${sshKey}" >&2 - exit 1 - } - trap cleanupTmp EXIT - - prepareDirs - ensureNetwork - collectExistingVms - downloadBaseImage - loadSshPubkey - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - checkCreateConflicts "${name}" "${ip}" "${mac}" - done < "${PLANNED_NODES_TSV}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - updateDhcpHost "${name}" "${ip}" "${mac}" - createVmCloudInit "${name}" - createVmDisk "${name}" "${disk_gb}" - createOrStartVm "${name}" "${ip}" "${mac}" "${vcpus}" "${memory_mb}" "${disk_gb}" - done < "${PLANNED_NODES_TSV}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - waitForSsh "${name}" "${ip}" - done < "${PLANNED_NODES_TSV}" - - if [ "${kvmSkipK3sConfig}" = "true" ]; then - echo "Skipping host-local configK3s.yaml generation because kvm.skipK3sConfig=true." - else - writeK3sConfig - fi - writeKvmState - echo "KVM VMs are ready." - echo "Next step: ${SCRIPT_DIR}/tuneVmLimits.py ${outputK3sConfig}" -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py index fa35995c0..45a1cb0f6 100755 --- a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py +++ b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py @@ -1,21 +1,274 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs recorded in kvmState.yaml. +# Inputs: kvmState.yaml generated by createKvmVms.py. +# Outputs/side effects: destroys matching libvirt domains, removes DHCP +# reservations/leases, disks, cloud-init files, generated kubeconfig/inventory, +# and then verifies the cleanup. The disk-dir guard prevents deleting unrelated +# VMs whose names happen to match. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +NODES_TSV="" +VERIFY_NODES_TSV="" +dnsmasqStatus="/var/lib/libvirt/dnsmasq/virbr0.status" +setupOutputGlob="${SETUP_DIR}/seedemu-k3s.*" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${NODES_TSV}" ] && rm -f "${NODES_TSV}" || true + [ -n "${VERIFY_NODES_TSV}" ] && rm -f "${VERIFY_NODES_TSV}" || true +} + +loadConfig() { + if [ ! -s "${CONFIG_PATH}" ]; then + echo "KVM state not found or empty: ${CONFIG_PATH}" >&2 + echo "Run createKvmVms.py first so it generates kvmState.yaml." >&2 + exit 1 + fi + eval "$(python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-vars --state "${CONFIG_PATH}")" + kvmBridgeName="$(virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c 'import sys, xml.etree.ElementTree as ET; data=sys.stdin.read(); print((ET.fromstring(data).find(".//bridge").get("name") if data.strip() else "") or "")' 2>/dev/null || true)" + if [ -n "${kvmBridgeName}" ]; then + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmBridgeName}.status" + else + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmNetwork}.status" + fi + NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-nodes.XXXXXX.tsv")" + VERIFY_NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-verify-nodes.XXXXXX.tsv")" + python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-nodes-tsv --state "${CONFIG_PATH}" > "${NODES_TSV}" + cp "${NODES_TSV}" "${VERIFY_NODES_TSV}" +} + +printPlan() { + echo "Cleaning KVM nodes from: ${CONFIG_PATH}" + echo "KVM network: ${kvmNetwork}" + echo "Disk dir: ${kvmDiskDir}" + echo "Cloud-init dir: ${kvmCloudInitDir}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' "${NODES_TSV}" +} + +cleanupDomainsReservationsAndFiles() { + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + echo "Cleaning VM ${name} (${ip}, ${mac})" + + if virsh dominfo "${name}" >/dev/null 2>&1; then + validateDestroyTargetBelongsToSetup "${name}" + virsh destroy "${name}" >/dev/null 2>&1 || true + virsh undefine "${name}" --nvram --remove-all-storage >/dev/null 2>&1 \ + || virsh undefine "${name}" --nvram >/dev/null 2>&1 \ + || virsh undefine "${name}" >/dev/null + fi + + if [ -n "${ip}" ] && [ -n "${mac}" ]; then + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + fi + + rm -rf "${kvmCloudInitDir}/${name}" + rm -f "${kvmDiskDir}/${name}.qcow2" + done < "${NODES_TSV}" +} + +validateDestroyTargetBelongsToSetup() { + # Args: + # $1: VM domain name to validate before destructive cleanup. + # Uses kvmDiskDir from kvmState.yaml. This prevents a stale state file from + # deleting a pre-existing cluster VM whose disk lives elsewhere. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for ${vm_name}; refusing destructive cleanup." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to destroy ${vm_name}: disk is outside this setup disk dir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +cleanupDnsmasqStaleLeases() { + [ -f "${dnsmasqStatus}" ] || return 0 + + local tmp_nodes + tmp_nodes="$(mktemp)" + awk -F '\t' 'NF >= 4 {print $1 "\t" $3 "\t" tolower($4)}' "${NODES_TSV}" > "${tmp_nodes}" + + sudo cp "${dnsmasqStatus}" "${dnsmasqStatus}.bak.$(date +%Y%m%d_%H%M%S)" + sudo python3 - "${dnsmasqStatus}" "${tmp_nodes}" <<'PY' +import json +import sys +from pathlib import Path + +status_path = Path(sys.argv[1]) +nodes_path = Path(sys.argv[2]) + +remove_names = set() +remove_ips = set() +remove_macs = set() +for line in nodes_path.read_text(encoding="utf-8").splitlines(): + parts = line.split("\t") + if len(parts) < 3: + continue + name, ip, mac = parts[:3] + if name: + remove_names.add(name) + if ip: + remove_ips.add(ip) + if mac: + remove_macs.add(mac.lower()) + +try: + data = json.loads(status_path.read_text(encoding="utf-8") or "[]") +except json.JSONDecodeError: + raise SystemExit(f"Invalid dnsmasq status JSON: {status_path}") + +filtered = [ + item for item in data + if not ( + item.get("hostname") in remove_names + or item.get("ip-address") in remove_ips + or str(item.get("mac-address", "")).lower() in remove_macs + ) +] +status_path.write_text(json.dumps(filtered, indent=2) + "\n", encoding="utf-8") +PY + rm -f "${tmp_nodes}" +} + +cleanupSetupOutputs() { + echo "Cleaning stale setup outputs matching: ${setupOutputGlob}" + rm -f ${setupOutputGlob} + rm -f "${CONFIG_PATH}" "${outputK3sConfig}" "${outputKubeconfig}" "${outputInventory}" + rm -f "${SETUP_DIR}/configRunning.yaml" +} + +verifyClean() { + local failed=0 + + echo "Verifying cleanup..." + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + if virsh dominfo "${name}" >/dev/null 2>&1; then + echo "Residual domain: ${name}" >&2 + failed=1 + fi + if [ -e "${kvmDiskDir}/${name}.qcow2" ]; then + echo "Residual disk: ${kvmDiskDir}/${name}.qcow2" >&2 + failed=1 + fi + if [ -e "${kvmCloudInitDir}/${name}" ]; then + echo "Residual cloud-init dir: ${kvmCloudInitDir}/${name}" >&2 + failed=1 + fi + + if virsh net-dumpxml "${kvmNetwork}" 2>/dev/null \ + | grep -E "]*(mac=['\"]${mac}['\"]|name=['\"]${name}['\"]|ip=['\"]${ip}['\"])" >/dev/null 2>&1; then + echo "Residual DHCP reservation in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null \ + | awk -v name="${name}" -v ip="${ip}/24" -v mac="$(printf '%s' "${mac}" | tr '[:upper:]' '[:lower:]')" ' + NR > 2 { + row = tolower($0) + if (row ~ mac && $5 == ip && $6 == name) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "Residual DHCP lease in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + done < "${VERIFY_NODES_TSV}" + + if [ -e "${CONFIG_PATH}" ]; then + echo "Residual KVM state: ${CONFIG_PATH}" >&2 + failed=1 + fi + + if compgen -G "${setupOutputGlob}" >/dev/null; then + echo "Residual setup output files matching ${setupOutputGlob}" >&2 + compgen -G "${setupOutputGlob}" >&2 || true + failed=1 + fi + + if [ "${failed}" -ne 0 ]; then + echo "Cleanup verification failed." >&2 + exit 1 + fi + echo "Cleanup verification passed." +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand awk + requireCommand grep + requireCommand mktemp + requireCommand python3 + requireCommand sudo + requireCommand virsh + + trap cleanupTmp EXIT + loadConfig + printPlan + cleanupDomainsReservationsAndFiles + cleanupDnsmasqStaleLeases + cleanupSetupOutputs + verifyClean +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh deleted file mode 100755 index f99d3fd2b..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.sh +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env bash -# Destroy KVM VMs recorded in kvmState.yaml. -# Inputs: kvmState.yaml generated by createKvmVms.py. -# Outputs/side effects: destroys matching libvirt domains, removes DHCP -# reservations/leases, disks, cloud-init files, generated kubeconfig/inventory, -# and then verifies the cleanup. The disk-dir guard prevents deleting unrelated -# VMs whose names happen to match. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvmState.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" -NODES_TSV="" -VERIFY_NODES_TSV="" -dnsmasqStatus="/var/lib/libvirt/dnsmasq/virbr0.status" -setupOutputGlob="${SETUP_DIR}/seedemu-k3s.*" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -cleanupTmp() { - [ -n "${NODES_TSV}" ] && rm -f "${NODES_TSV}" || true - [ -n "${VERIFY_NODES_TSV}" ] && rm -f "${VERIFY_NODES_TSV}" || true -} - -loadConfig() { - if [ ! -s "${CONFIG_PATH}" ]; then - echo "KVM state not found or empty: ${CONFIG_PATH}" >&2 - echo "Run createKvmVms.py first so it generates kvmState.yaml." >&2 - exit 1 - fi - eval "$(python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-vars --state "${CONFIG_PATH}")" - kvmBridgeName="$(virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c 'import sys, xml.etree.ElementTree as ET; data=sys.stdin.read(); print((ET.fromstring(data).find(".//bridge").get("name") if data.strip() else "") or "")' 2>/dev/null || true)" - if [ -n "${kvmBridgeName}" ]; then - dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmBridgeName}.status" - else - dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmNetwork}.status" - fi - NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-nodes.XXXXXX.tsv")" - VERIFY_NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-verify-nodes.XXXXXX.tsv")" - python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-nodes-tsv --state "${CONFIG_PATH}" > "${NODES_TSV}" - cp "${NODES_TSV}" "${VERIFY_NODES_TSV}" -} - -printPlan() { - echo "Cleaning KVM nodes from: ${CONFIG_PATH}" - echo "KVM network: ${kvmNetwork}" - echo "Disk dir: ${kvmDiskDir}" - echo "Cloud-init dir: ${kvmCloudInitDir}" - awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' "${NODES_TSV}" -} - -cleanupDomainsReservationsAndFiles() { - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - [ -n "${name}" ] || continue - echo "Cleaning VM ${name} (${ip}, ${mac})" - - if virsh dominfo "${name}" >/dev/null 2>&1; then - validateDestroyTargetBelongsToSetup "${name}" - virsh destroy "${name}" >/dev/null 2>&1 || true - virsh undefine "${name}" --nvram --remove-all-storage >/dev/null 2>&1 \ - || virsh undefine "${name}" --nvram >/dev/null 2>&1 \ - || virsh undefine "${name}" >/dev/null - fi - - if [ -n "${ip}" ] && [ -n "${mac}" ]; then - local host_xml="" - virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true - fi - - rm -rf "${kvmCloudInitDir}/${name}" - rm -f "${kvmDiskDir}/${name}.qcow2" - done < "${NODES_TSV}" -} - -validateDestroyTargetBelongsToSetup() { - # Args: - # $1: VM domain name to validate before destructive cleanup. - # Uses kvmDiskDir from kvmState.yaml. This prevents a stale state file from - # deleting a pre-existing cluster VM whose disk lives elsewhere. - local vm_name="$1" - local disk_path="" - disk_path="$( - virsh domblklist "${vm_name}" --details 2>/dev/null \ - | awk '$2 == "disk" && $4 != "-" {print $4; exit}' - )" - if [ -z "${disk_path}" ]; then - echo "Cannot determine disk path for ${vm_name}; refusing destructive cleanup." >&2 - exit 1 - fi - case "${disk_path}" in - "${kvmDiskDir}"/*) ;; - *) - echo "Refusing to destroy ${vm_name}: disk is outside this setup disk dir." >&2 - echo " vm_disk=${disk_path}" >&2 - echo " setup_disk_dir=${kvmDiskDir}" >&2 - echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 - exit 1 - ;; - esac -} - -cleanupDnsmasqStaleLeases() { - [ -f "${dnsmasqStatus}" ] || return 0 - - local tmp_nodes - tmp_nodes="$(mktemp)" - awk -F '\t' 'NF >= 4 {print $1 "\t" $3 "\t" tolower($4)}' "${NODES_TSV}" > "${tmp_nodes}" - - sudo cp "${dnsmasqStatus}" "${dnsmasqStatus}.bak.$(date +%Y%m%d_%H%M%S)" - sudo python3 - "${dnsmasqStatus}" "${tmp_nodes}" <<'PY' -import json -import sys -from pathlib import Path - -status_path = Path(sys.argv[1]) -nodes_path = Path(sys.argv[2]) - -remove_names = set() -remove_ips = set() -remove_macs = set() -for line in nodes_path.read_text(encoding="utf-8").splitlines(): - parts = line.split("\t") - if len(parts) < 3: - continue - name, ip, mac = parts[:3] - if name: - remove_names.add(name) - if ip: - remove_ips.add(ip) - if mac: - remove_macs.add(mac.lower()) - -try: - data = json.loads(status_path.read_text(encoding="utf-8") or "[]") -except json.JSONDecodeError: - raise SystemExit(f"Invalid dnsmasq status JSON: {status_path}") - -filtered = [ - item for item in data - if not ( - item.get("hostname") in remove_names - or item.get("ip-address") in remove_ips - or str(item.get("mac-address", "")).lower() in remove_macs - ) -] -status_path.write_text(json.dumps(filtered, indent=2) + "\n", encoding="utf-8") -PY - rm -f "${tmp_nodes}" -} - -cleanupSetupOutputs() { - echo "Cleaning stale setup outputs matching: ${setupOutputGlob}" - rm -f ${setupOutputGlob} - rm -f "${CONFIG_PATH}" "${outputK3sConfig}" "${outputKubeconfig}" "${outputInventory}" - rm -f "${SETUP_DIR}/configRunning.yaml" -} - -verifyClean() { - local failed=0 - - echo "Verifying cleanup..." - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - [ -n "${name}" ] || continue - if virsh dominfo "${name}" >/dev/null 2>&1; then - echo "Residual domain: ${name}" >&2 - failed=1 - fi - if [ -e "${kvmDiskDir}/${name}.qcow2" ]; then - echo "Residual disk: ${kvmDiskDir}/${name}.qcow2" >&2 - failed=1 - fi - if [ -e "${kvmCloudInitDir}/${name}" ]; then - echo "Residual cloud-init dir: ${kvmCloudInitDir}/${name}" >&2 - failed=1 - fi - - if virsh net-dumpxml "${kvmNetwork}" 2>/dev/null \ - | grep -E "]*(mac=['\"]${mac}['\"]|name=['\"]${name}['\"]|ip=['\"]${ip}['\"])" >/dev/null 2>&1; then - echo "Residual DHCP reservation in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 - failed=1 - fi - - if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null \ - | awk -v name="${name}" -v ip="${ip}/24" -v mac="$(printf '%s' "${mac}" | tr '[:upper:]' '[:lower:]')" ' - NR > 2 { - row = tolower($0) - if (row ~ mac && $5 == ip && $6 == name) { - found = 1 - } - } - END { exit(found ? 0 : 1) } - '; then - echo "Residual DHCP lease in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 - failed=1 - fi - done < "${VERIFY_NODES_TSV}" - - if [ -e "${CONFIG_PATH}" ]; then - echo "Residual KVM state: ${CONFIG_PATH}" >&2 - failed=1 - fi - - if compgen -G "${setupOutputGlob}" >/dev/null; then - echo "Residual setup output files matching ${setupOutputGlob}" >&2 - compgen -G "${setupOutputGlob}" >&2 || true - failed=1 - fi - - if [ "${failed}" -ne 0 ]; then - echo "Cleanup verification failed." >&2 - exit 1 - fi - echo "Cleanup verification passed." -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - - requireCommand awk - requireCommand grep - requireCommand mktemp - requireCommand python3 - requireCommand sudo - requireCommand virsh - - trap cleanupTmp EXIT - loadConfig - printPlan - cleanupDomainsReservationsAndFiles - cleanupDnsmasqStaleLeases - cleanupSetupOutputs - verifyClean -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py index 8b4c50a85..db26816ea 100755 --- a/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py +++ b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py @@ -559,6 +559,9 @@ def to_k3s_node_payload(node: dict[str, Any]) -> dict[str, Any]: "name": node["name"], "role": node["role"], "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), "ssh": { "user": node.get("sshUser", ""), "key": node.get("sshKey", ""), @@ -625,7 +628,7 @@ def write_k3s_config(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.nodes_tsv: Transient node TSV generated by createKvmVms.sh. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. args.output: Destination configK3s.yaml path. """ data = load_config(args.config) @@ -655,7 +658,7 @@ def write_kvm_state(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.nodes_tsv: Transient node TSV generated by createKvmVms.sh. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. args.output: Destination kvmState.yaml path. """ data = load_config(args.config) @@ -734,7 +737,7 @@ def validate_kvm_state(args: argparse.Namespace) -> None: Args: args.config: Source kvm.yaml. - args.state: Existing kvmState.yaml that createKvmVms.sh may reuse. + args.state: Existing kvmState.yaml that createKvmVms.py may reuse. """ data = load_config(args.config) expected = nodes(data) diff --git a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py index 55a600669..73f5e5e27 100755 --- a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py +++ b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py @@ -1,21 +1,317 @@ #!/usr/bin/env python3 -"""Python entrypoint for prepareHostAssets.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for prepareHostAssets with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare host-side assets required before KVM creation: +# Ubuntu cloud image, registry/K3s bootstrap image tarballs, and SeedEMU base +# image tags. All user-configurable paths come from kvm.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" + +HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" +hostDockerIoMirror="docker.m.daocloud.io" +dockerPullTimeoutSeconds=180 +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 +prepareForce="false" +kvmBaseImageReuseMode="copy" +REGISTRY_BOOTSTRAP_IMAGE="registry:2" +MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" +K3S_SYSTEM_BOOTSTRAP_IMAGES=( + "rancher/mirrored-coredns-coredns:1.10.1" + "rancher/mirrored-metrics-server:v0.6.3" + "rancher/local-path-provisioner:v0.0.24" +) +seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" +seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" +seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" +seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" +ubuntuBuildImage="ubuntu:20.04" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveSeedEmulatorDockerDir() { + # Resolve the source-tree path containing seedemu-base and seedemu-router. + # kvm.yaml seedemu.dockerImagesDir wins through manageKvmConfig.py; the + # ancestor scan keeps examples runnable directly from this repository. + local cursor="${SETUP_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${hostDockerIoMirror}" "${image}" + else + printf '%s/library/%s\n' "${hostDockerIoMirror}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " docker pull ${image}" + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " docker pull ${mirror_image}" + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare Docker image on host: ${image}" >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + if [ "${prepareForce}" != "true" ] && [ -s "${tar_path}" ]; then + echo " cache exists: ${tar_path}" + return + fi + ensureHostDockerImage "${image}" + echo " docker save ${image} -> ${tar_path}" + docker save -o "${tar_path}" "${image}" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + # Downloads through a resumable .part file and validates qcow2 metadata + # before publishing the final path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +prepareBaseImage() { + mkdir -p "$(dirname "${kvmBaseImagePath}")" + + if [ -e "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + echo "Base image already exists: ${kvmBaseImagePath}" + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Reusing existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Reusing existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + + echo "Downloading Ubuntu cloud image:" + echo " url=${kvmBaseImageUrl}" + echo " output=${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +prepareSeedemuBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-base" ]; then + echo " docker build ${seedBaseSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + else + ensureHostDockerImage "${seedBaseSourceImage}" + fi + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-router" ]; then + echo " docker build ${seedRouterSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + else + ensureHostDockerImage "${seedRouterSourceImage}" + fi + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareImageCache() { + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + prepareSeedemuBuildImages + + saveHostImageTarball "${REGISTRY_BOOTSTRAP_IMAGE}" + saveHostImageTarball "${MULTUS_BOOTSTRAP_IMAGE}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}"; do + saveHostImageTarball "${image}" + done + saveHostImageTarball "${ubuntuBuildImage}" + saveHostImageTarball "${seedBaseSourceImage}" + saveHostImageTarball "${seedRouterSourceImage}" + + echo " hash tags prepared locally, not saved into image-cache:" + echo " ${seedBaseHashImage}" + echo " ${seedRouterHashImage}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand python3 + requireCommand curl + requireCommand docker + requireCommand find + requireCommand qemu-img + + eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + resolveSeedEmulatorDockerDir + + echo "Preparing setup assets using config: ${CONFIG_PATH}" + echo "base_image_path=${kvmBaseImagePath}" + echo "image_cache_dir=${HOST_IMAGE_CACHE_DIR}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + prepareBaseImage + prepareImageCache + echo "Setup assets are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh deleted file mode 100755 index 65d602426..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.sh +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env bash -# Prepare host-side assets required before KVM creation: -# Ubuntu cloud image, registry/K3s bootstrap image tarballs, and SeedEMU base -# image tags. All user-configurable paths come from kvm.yaml. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageKvmConfig.py" - -HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" -hostDockerIoMirror="docker.m.daocloud.io" -dockerPullTimeoutSeconds=180 -baseImageLowSpeedSeconds=60 -baseImageLowSpeedLimit=1024 -prepareForce="false" -kvmBaseImageReuseMode="copy" -REGISTRY_BOOTSTRAP_IMAGE="registry:2" -MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" -K3S_SYSTEM_BOOTSTRAP_IMAGES=( - "rancher/mirrored-coredns-coredns:1.10.1" - "rancher/mirrored-metrics-server:v0.6.3" - "rancher/local-path-provisioner:v0.0.24" -) -seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" -kvmBaseImageSearchDirs="${HOME}/k8s/output" -seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" -seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" -seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" -seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" -ubuntuBuildImage="ubuntu:20.04" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -resolveSeedEmulatorDockerDir() { - # Resolve the source-tree path containing seedemu-base and seedemu-router. - # kvm.yaml seedemu.dockerImagesDir wins through manageKvmConfig.py; the - # ancestor scan keeps examples runnable directly from this repository. - local cursor="${SETUP_DIR}" - while [ "${cursor}" != "/" ]; do - if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && - [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then - seedEmulatorDockerDir="${cursor}/docker_images/multiarch" - return 0 - fi - cursor="$(dirname "${cursor}")" - done - local candidate - for candidate in \ - "${seedEmulatorDockerDir}" \ - "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ - "${HOME}/k8s/seed-emulator/docker_images/multiarch"; do - if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then - seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" - return 0 - fi - done -} - -imageTarName() { - printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' -} - -dockerIoMirrorRef() { - local image="$1" - - if [[ "${image}" == */*/* ]]; then - return 1 - fi - - if [[ "${image}" == */* ]]; then - printf '%s/%s\n' "${hostDockerIoMirror}" "${image}" - else - printf '%s/library/%s\n' "${hostDockerIoMirror}" "${image}" - fi -} - -ensureHostDockerImage() { - local image="$1" - local mirror_image="" - - if docker image inspect "${image}" >/dev/null 2>&1; then - return 0 - fi - - echo " docker pull ${image}" - if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then - return 0 - fi - - if mirror_image="$(dockerIoMirrorRef "${image}")"; then - echo " docker pull ${mirror_image}" - runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null - docker tag "${mirror_image}" "${image}" >/dev/null - return 0 - fi - - echo "Failed to prepare Docker image on host: ${image}" >&2 - return 1 -} - -hostImageTarball() { - local image="$1" - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" -} - -saveHostImageTarball() { - local image="$1" - local tar_path - tar_path="$(hostImageTarball "${image}")" - if [ "${prepareForce}" != "true" ] && [ -s "${tar_path}" ]; then - echo " cache exists: ${tar_path}" - return - fi - ensureHostDockerImage "${image}" - echo " docker save ${image} -> ${tar_path}" - docker save -o "${tar_path}" "${image}" -} - -validBaseImage() { - # Args: - # $1: qcow2 image path to validate. - local image_path="$1" - [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 -} - -downloadBaseImageFile() { - # Args: - # $1: URL, $2: destination path. - # Downloads through a resumable .part file and validates qcow2 metadata - # before publishing the final path. - local image_url="$1" - local image_path="$2" - local partial_path="${image_path}.part" - mkdir -p "$(dirname "${image_path}")" - curl -fL \ - --retry 5 \ - --retry-delay 5 \ - --retry-all-errors \ - --connect-timeout 20 \ - --speed-time "${baseImageLowSpeedSeconds}" \ - --speed-limit "${baseImageLowSpeedLimit}" \ - -C - \ - -o "${partial_path}" \ - "${image_url}" - validBaseImage "${partial_path}" || { - echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 - return 1 - } - mv -f "${partial_path}" "${image_path}" -} - -prepareBaseImage() { - mkdir -p "$(dirname "${kvmBaseImagePath}")" - - if [ -e "${kvmBaseImagePath}" ]; then - if validBaseImage "${kvmBaseImagePath}"; then - echo "Base image already exists: ${kvmBaseImagePath}" - return - fi - echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" - rm -f "${kvmBaseImagePath}" - fi - - if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then - echo "Reusing existing base image from ${kvmLegacyBaseImagePath}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" - fi - return - fi - - local output_image="" - local search_dir="" - for search_dir in ${kvmBaseImageSearchDirs}; do - [ -d "${search_dir}" ] || continue - output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" - if [ -n "${output_image}" ]; then - echo "Reusing existing base image from ${output_image}" - if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then - ln -s "${output_image}" "${kvmBaseImagePath}" - else - cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" - fi - return - fi - done - - echo "Downloading Ubuntu cloud image:" - echo " url=${kvmBaseImageUrl}" - echo " output=${kvmBaseImagePath}" - downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" -} - -prepareSeedemuBuildImages() { - ensureHostDockerImage "${ubuntuBuildImage}" - - if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then - if [ -d "${seedEmulatorDockerDir}/seedemu-base" ]; then - echo " docker build ${seedBaseSourceImage}" - DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-base" >/dev/null - else - ensureHostDockerImage "${seedBaseSourceImage}" - fi - fi - - if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then - if [ -d "${seedEmulatorDockerDir}/seedemu-router" ]; then - echo " docker build ${seedRouterSourceImage}" - DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ - "${seedEmulatorDockerDir}/seedemu-router" >/dev/null - else - ensureHostDockerImage "${seedRouterSourceImage}" - fi - fi - - docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null - docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null -} - -prepareImageCache() { - mkdir -p "${HOST_IMAGE_CACHE_DIR}" - prepareSeedemuBuildImages - - saveHostImageTarball "${REGISTRY_BOOTSTRAP_IMAGE}" - saveHostImageTarball "${MULTUS_BOOTSTRAP_IMAGE}" - for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}"; do - saveHostImageTarball "${image}" - done - saveHostImageTarball "${ubuntuBuildImage}" - saveHostImageTarball "${seedBaseSourceImage}" - saveHostImageTarball "${seedRouterSourceImage}" - - echo " hash tags prepared locally, not saved into image-cache:" - echo " ${seedBaseHashImage}" - echo " ${seedRouterHashImage}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - - requireCommand python3 - requireCommand curl - requireCommand docker - requireCommand find - requireCommand qemu-img - - eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" - resolveSeedEmulatorDockerDir - - echo "Preparing setup assets using config: ${CONFIG_PATH}" - echo "base_image_path=${kvmBaseImagePath}" - echo "image_cache_dir=${HOST_IMAGE_CACHE_DIR}" - echo "seedemu_docker_dir=${seedEmulatorDockerDir}" - prepareBaseImage - prepareImageCache - echo "Setup assets are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py index a29d87221..3ec843914 100755 --- a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py +++ b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py @@ -1,21 +1,299 @@ #!/usr/bin/env python3 -"""Python entrypoint for tuneVmLimits.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for tuneVmLimits with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Tune OS-level limits on each VM listed in configK3s.yaml. +# Inputs: configK3s.yaml with nodes and SSH settings. +# Outputs/side effects: writes sysctl, limits, systemd drop-ins, and cni0 +# hash_max service on each VM through SSH. No K3s installation is performed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +# Keep these defaults in the script, not in configK3s.yaml. The YAML describes VMs; +# this script describes the current high-density OS limit policy. +vmLimitNofile="10485760" +vmLimitNproc="4194304" +vmLimitMaxNetNamespaces="65536" +vmLimitNeighGcThresh1="1048576" +vmLimitNeighGcThresh2="4194304" +vmLimitNeighGcThresh3="8388608" +vmLimitNetdevMaxBacklog="1000000" +vmLimitOptmemMax="25165824" +vmLimitCni0HashMax="16384" +vmLimitReboot="false" + +NODE_SSH_OPTS=() +nodeSshUser="" +nodeSshKey="" +nodeConnection="" + +usage() { + cat <&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +requireCommand() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +waitForSsh() { + local name="$1" + local ip="$2" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + sudo -n true >/dev/null 2>&1 || { + echo "Local sudo failed for ${name} (${ip})" >&2 + return 1 + } + return 0 + fi + if ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ok" >/dev/null 2>&1; then + return 0 + fi + echo "SSH failed for ${name} (${ip})" >&2 + return 1 +} + +unlockOneVm() { + local name="$1" + local ip="$2" + local remote_runner=() + echo "Unlocking VM limits: ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${vmLimitNofile}" \ + "${vmLimitNproc}" \ + "${vmLimitMaxNetNamespaces}" \ + "${vmLimitNeighGcThresh1}" \ + "${vmLimitNeighGcThresh2}" \ + "${vmLimitNeighGcThresh3}" \ + "${vmLimitNetdevMaxBacklog}" \ + "${vmLimitOptmemMax}" \ + "${vmLimitCni0HashMax}" \ + "${vmLimitReboot}" <<'EOF_REMOTE' +set -euo pipefail + +LIMIT_NOFILE="$1" +LIMIT_NPROC="$2" +MAX_NET_NS="$3" +NEIGH1="$4" +NEIGH2="$5" +NEIGH3="$6" +NETDEV_BACKLOG="$7" +OPTMEM_MAX="$8" +CNI0_HASH_MAX="$9" +DO_REBOOT="${10}" + +appendIfMissing() { + local file="$1" + local pattern="$2" + local line="$3" + touch "${file}" + if ! grep -qF "${pattern}" "${file}"; then + printf '%s\n' "${line}" >> "${file}" + fi +} + +echo ">> limits.conf" +appendIfMissing /etc/security/limits.conf "* soft nofile" "* soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* hard nofile" "* hard nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* soft nproc" "* soft nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "* hard nproc" "* hard nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "root soft nofile" "root soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "root hard nofile" "root hard nofile ${LIMIT_NOFILE}" + +echo ">> sysctl" +cat > /etc/sysctl.d/99-seed-vm-limits.conf </dev/null 2>&1 || true +ip -s -s neigh flush all >/dev/null 2>&1 || true + +echo ">> systemd default limits" +mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d +cat > /etc/systemd/system.conf.d/99-seed-vm-limits.conf < /etc/systemd/user.conf.d/99-seed-vm-limits.conf <> known service drop-ins if present" +for service in k3s k3s-agent containerd docker; do + if systemctl list-unit-files | grep -q "^${service}.service"; then + mkdir -p "/etc/systemd/system/${service}.service.d" + cat > "/etc/systemd/system/${service}.service.d/99-seed-vm-limits.conf" <> cni0 hash_max service for future K3s/flannel bridge" +cat > /usr/local/sbin/seed-vm-cni0-hashmax.sh <<'EOF_TUNE' +#!/usr/bin/env bash +set -euo pipefail +target="${1:-16384}" +for _ in $(seq 1 60); do + if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + printf '%s\n' "${target}" > /sys/class/net/cni0/bridge/hash_max + exit 0 + fi + sleep 2 +done +exit 0 +EOF_TUNE +chmod +x /usr/local/sbin/seed-vm-cni0-hashmax.sh + +cat > /etc/systemd/system/seed-vm-cni0-hashmax.service </dev/null 2>&1 || true +if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + /usr/local/sbin/seed-vm-cni0-hashmax.sh "${CNI0_HASH_MAX}" >/dev/null 2>&1 || true +fi + +if [ "${DO_REBOOT}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "reboot" >/dev/null 2>&1 || true +fi + +echo "VM limit unlock completed on $(hostname)" +EOF_REMOTE +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + echo "Using K3s config: ${CONFIG_PATH}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < <(nodesInput) + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + unlockOneVm "${name}" "${ip}" + done < <(nodesInput) + + echo "VM limit unlock completed for all nodes in ${CONFIG_PATH}" +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh deleted file mode 100755 index 7ce9a4cfa..000000000 --- a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env bash -# Tune OS-level limits on each VM listed in configK3s.yaml. -# Inputs: configK3s.yaml with nodes and SSH settings. -# Outputs/side effects: writes sysctl, limits, systemd drop-ins, and cni0 -# hash_max service on each VM through SSH. No K3s installation is performed. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" - -# Keep these defaults in the script, not in configK3s.yaml. The YAML describes VMs; -# this script describes the current high-density OS limit policy. -vmLimitNofile="10485760" -vmLimitNproc="4194304" -vmLimitMaxNetNamespaces="65536" -vmLimitNeighGcThresh1="1048576" -vmLimitNeighGcThresh2="4194304" -vmLimitNeighGcThresh3="8388608" -vmLimitNetdevMaxBacklog="1000000" -vmLimitOptmemMax="25165824" -vmLimitCni0HashMax="16384" -vmLimitReboot="false" - -NODE_SSH_OPTS=() -nodeSshUser="" -nodeSshKey="" -nodeConnection="" - -usage() { - cat <&2 - exit 1 - } - NODE_SSH_OPTS=( - -i "${nodeSshKey}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o BatchMode=yes - -o IdentitiesOnly=yes - -o IdentityAgent=none - -o ConnectTimeout=10 - -o ServerAliveInterval=30 - -o ServerAliveCountMax=3 - ) -} - -requireCommand() { - command -v "$1" >/dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -waitForSsh() { - local name="$1" - local ip="$2" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - sudo -n true >/dev/null 2>&1 || { - echo "Local sudo failed for ${name} (${ip})" >&2 - return 1 - } - return 0 - fi - if ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ok" >/dev/null 2>&1; then - return 0 - fi - echo "SSH failed for ${name} (${ip})" >&2 - return 1 -} - -unlockOneVm() { - local name="$1" - local ip="$2" - local remote_runner=() - echo "Unlocking VM limits: ${name} (${ip})" - loadNodeSshContext "${name}" - if [ "${nodeConnection:-ssh}" = "local" ]; then - remote_runner=(sudo -n bash -s --) - else - remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) - fi - "${remote_runner[@]}" \ - "${vmLimitNofile}" \ - "${vmLimitNproc}" \ - "${vmLimitMaxNetNamespaces}" \ - "${vmLimitNeighGcThresh1}" \ - "${vmLimitNeighGcThresh2}" \ - "${vmLimitNeighGcThresh3}" \ - "${vmLimitNetdevMaxBacklog}" \ - "${vmLimitOptmemMax}" \ - "${vmLimitCni0HashMax}" \ - "${vmLimitReboot}" <<'EOF_REMOTE' -set -euo pipefail - -LIMIT_NOFILE="$1" -LIMIT_NPROC="$2" -MAX_NET_NS="$3" -NEIGH1="$4" -NEIGH2="$5" -NEIGH3="$6" -NETDEV_BACKLOG="$7" -OPTMEM_MAX="$8" -CNI0_HASH_MAX="$9" -DO_REBOOT="${10}" - -appendIfMissing() { - local file="$1" - local pattern="$2" - local line="$3" - touch "${file}" - if ! grep -qF "${pattern}" "${file}"; then - printf '%s\n' "${line}" >> "${file}" - fi -} - -echo ">> limits.conf" -appendIfMissing /etc/security/limits.conf "* soft nofile" "* soft nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "* hard nofile" "* hard nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "* soft nproc" "* soft nproc ${LIMIT_NPROC}" -appendIfMissing /etc/security/limits.conf "* hard nproc" "* hard nproc ${LIMIT_NPROC}" -appendIfMissing /etc/security/limits.conf "root soft nofile" "root soft nofile ${LIMIT_NOFILE}" -appendIfMissing /etc/security/limits.conf "root hard nofile" "root hard nofile ${LIMIT_NOFILE}" - -echo ">> sysctl" -cat > /etc/sysctl.d/99-seed-vm-limits.conf </dev/null 2>&1 || true -ip -s -s neigh flush all >/dev/null 2>&1 || true - -echo ">> systemd default limits" -mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d -cat > /etc/systemd/system.conf.d/99-seed-vm-limits.conf < /etc/systemd/user.conf.d/99-seed-vm-limits.conf <> known service drop-ins if present" -for service in k3s k3s-agent containerd docker; do - if systemctl list-unit-files | grep -q "^${service}.service"; then - mkdir -p "/etc/systemd/system/${service}.service.d" - cat > "/etc/systemd/system/${service}.service.d/99-seed-vm-limits.conf" <> cni0 hash_max service for future K3s/flannel bridge" -cat > /usr/local/sbin/seed-vm-cni0-hashmax.sh <<'EOF_TUNE' -#!/usr/bin/env bash -set -euo pipefail -target="${1:-16384}" -for _ in $(seq 1 60); do - if [ -w /sys/class/net/cni0/bridge/hash_max ]; then - printf '%s\n' "${target}" > /sys/class/net/cni0/bridge/hash_max - exit 0 - fi - sleep 2 -done -exit 0 -EOF_TUNE -chmod +x /usr/local/sbin/seed-vm-cni0-hashmax.sh - -cat > /etc/systemd/system/seed-vm-cni0-hashmax.service </dev/null 2>&1 || true -if [ -w /sys/class/net/cni0/bridge/hash_max ]; then - /usr/local/sbin/seed-vm-cni0-hashmax.sh "${CNI0_HASH_MAX}" >/dev/null 2>&1 || true -fi - -if [ "${DO_REBOOT}" = "true" ]; then - systemd-run --on-active=2 /bin/bash -c "reboot" >/dev/null 2>&1 || true -fi - -echo "VM limit unlock completed on $(hostname)" -EOF_REMOTE -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - echo "Using K3s config: ${CONFIG_PATH}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - waitForSsh "${name}" "${ip}" - done < <(nodesInput) - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - unlockOneVm "${name}" "${ip}" - done < <(nodesInput) - - echo "VM limit unlock completed for all nodes in ${CONFIG_PATH}" -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/manageK3sConfig.py b/seedemu/k8sTools/resources/setup/manageK3sConfig.py index 3bd63788b..3566b2c08 100644 --- a/seedemu/k8sTools/resources/setup/manageK3sConfig.py +++ b/seedemu/k8sTools/resources/setup/manageK3sConfig.py @@ -233,7 +233,7 @@ def masterNode(nodes: list[dict[str, Any]]) -> dict[str, Any]: def configValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, Any]: - """Build the local shell variable values consumed by applyK3sCluster.sh.""" + """Build the local shell variable values consumed by applyK3sCluster.py.""" cluster_name = str(data.get("clusterName") or data.get("cluster_name") or "seedemu-k3s") master = masterNode(nodes) registry_host = getNested(data, "registry.host", master["ip"]) @@ -285,7 +285,7 @@ def fabricValues(data: dict[str, Any]) -> dict[str, Any]: The first implementation intentionally supports a two-node Linux VXLAN fabric. Larger physical fabrics should use a real L2/VLAN/EVPN/OVS design - instead of an implicit shell-script full mesh. + instead of an implicit command-driven full mesh. """ values = { "fabricType": str(getNested(data, "fabric.type", "none")), @@ -309,7 +309,7 @@ def ovnValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, An data: Parsed configK3s.yaml mapping. nodes: Normalized node list used to discover the master IP. - These values are consumed by ovn/installKubeOvnFabric.sh. Kube-OVN runs as + These values are consumed by ovn/installKubeOvnFabric.py. Kube-OVN runs as a secondary CNI while K3s/flannel remains the primary eth0 network. """ vals = configValues(data, nodes) @@ -432,7 +432,7 @@ def fabricNodeRows(data: dict[str, Any], nodes: list[dict[str, Any]]) -> list[di The user may set fabric.nodes..underlayInterface explicitly. If it is omitted, the helper detects the underlay by asking the node which interface it uses to route to the peer management IP. Test IPs are internal defaults - and are only used by validateLinuxVxlanFabric.sh. + and are only used by validateLinuxVxlanFabric.py. """ values = fabricValues(data) if values["fabricType"] == "none": @@ -582,7 +582,7 @@ def ansibleHostVars(node: dict[str, Any], k3s_role: str, as_group: str) -> dict[ def commandWriteAnsibleInventory(args: argparse.Namespace) -> None: - """Write the temporary Ansible inventory used by applyK3sCluster.sh.""" + """Write the temporary Ansible inventory used by applyK3sCluster.py.""" data = loadYaml(args.config) nodes = yamlNodes(data) vals = configValues(data, nodes) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py index 29fd1da73..c9b5d51e0 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py @@ -1,21 +1,256 @@ #!/usr/bin/env python3 -"""Python entrypoint for createMultiHostKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for createMultiHostKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create KVM VMs across multiple physical hypervisors. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Outputs: +# - Host-local kvm.yaml files under setup/tmp/multiHostKvm/. +# - Remote host-local setup directories under hypervisors[].remoteWorkDir. +# - Global configK3s.yaml for applyK3sCluster.py. +# - Global multiHostKvmState.yaml for destroyMultiHostKvmVms.py. +# +# Execution context: +# Run after prepareKvmHypervisors.py. This script creates VMs on the target +# hypervisors but does not install K3s. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +LOCAL_WORK_DIR="" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +copyVmSshKeyToHost() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key. + # + # Remote hypervisors run kvm/createKvmVms.py locally, so they need a local + # copy of the VM SSH private key to inject its public key into cloud-init + # and wait for SSH readiness. The global configK3s.yaml still keeps the + # original control-host key path for later K3s installation. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local source_key target_key source_pub target_pub + + IFS=$'\t' read -r source_key target_key source_pub target_pub < <( + python3 "${HELPER}" --config "${CONFIG_PATH}" host-vm-ssh-key-tsv --host "${name}" + ) + [ -f "${source_key}" ] || { + echo "VM SSH key not found on control host: ${source_key}" >&2 + exit 1 + } + if [ "${connection}" = "local" ]; then + return + fi + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "$(dirname "${target_key}")")" + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_key}" "${ssh_user}@${ip}:${target_key}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 600 $(printf '%q' "${target_key}")" + if [ -f "${source_pub}" ]; then + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_pub}" "${ssh_user}@${ip}:${target_pub}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 644 $(printf '%q' "${target_pub}")" + fi +} + +prepareLocalWorkDir() { + mkdir -p "${outputTmpDir}" + LOCAL_WORK_DIR="${outputTmpDir}/multiHostKvm" + rm -rf "${LOCAL_WORK_DIR}" + mkdir -p "${LOCAL_WORK_DIR}" +} + +createHostVms() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local remote_work_dir="${10}" + local host_dir="${LOCAL_WORK_DIR}/${name}" + local host_kvm_yaml="${host_dir}/kvm.yaml" + + echo "[multi-host-create] ${name} (${ip})" + mkdir -p "${host_dir}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-local-kvm \ + --host "${name}" \ + --output "${host_kvm_yaml}" >/dev/null + + syncDirToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${SETUP_DIR}/kvm" "${remote_work_dir}/kvm" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${host_kvm_yaml}" "${remote_work_dir}/kvm.yaml" + copyVmSshKeyToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + cd $(printf '%q' "${remote_work_dir}") + python3 ./kvm/prepareHostAssets.py ./kvm.yaml + python3 ./kvm/createKvmVms.py ./kvm.yaml + " +} + +writeGlobalOutputs() { + python3 "${HELPER}" --config "${CONFIG_PATH}" write-global-k3s-config \ + --output "${outputConfigK3s}" >/dev/null + python3 "${HELPER}" --config "${CONFIG_PATH}" write-state \ + --output "${outputMultiHostKvmState}" >/dev/null + echo "K3s config: ${outputConfigK3s}" + echo "Multi-host KVM state: ${outputMultiHostKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand rsync + requireCommand cp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + prepareLocalWorkDir + python3 "${HELPER}" --config "${CONFIG_PATH}" validate + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + createHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + writeGlobalOutputs + echo "Multi-host KVM VMs are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh deleted file mode 100755 index 3a148544a..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash -# Create KVM VMs across multiple physical hypervisors. -# -# Inputs: -# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. -# -# Outputs: -# - Host-local kvm.yaml files under setup/tmp/multiHostKvm/. -# - Remote host-local setup directories under hypervisors[].remoteWorkDir. -# - Global configK3s.yaml for buildK3sCluster.sh. -# - Global multiHostKvmState.yaml for destroyMultiHostKvmVms.py. -# -# Execution context: -# Run after prepareKvmHypervisors.py. This script creates VMs on the target -# hypervisors but does not install K3s. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" -LOCAL_WORK_DIR="" - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null -} - -copyVmSshKeyToHost() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key. - # - # Remote hypervisors run kvm/createKvmVms.py locally, so they need a local - # copy of the VM SSH private key to inject its public key into cloud-init - # and wait for SSH readiness. The global configK3s.yaml still keeps the - # original control-host key path for later K3s installation. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local source_key target_key source_pub target_pub - - IFS=$'\t' read -r source_key target_key source_pub target_pub < <( - python3 "${HELPER}" --config "${CONFIG_PATH}" host-vm-ssh-key-tsv --host "${name}" - ) - [ -f "${source_key}" ] || { - echo "VM SSH key not found on control host: ${source_key}" >&2 - exit 1 - } - if [ "${connection}" = "local" ]; then - return - fi - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "mkdir -p $(printf '%q' "$(dirname "${target_key}")")" - # shellcheck disable=SC2046 - scp $(sshOptions "${ssh_key}") "${source_key}" "${ssh_user}@${ip}:${target_key}" >/dev/null - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "chmod 600 $(printf '%q' "${target_key}")" - if [ -f "${source_pub}" ]; then - # shellcheck disable=SC2046 - scp $(sshOptions "${ssh_key}") "${source_pub}" "${ssh_user}@${ip}:${target_pub}" >/dev/null - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "chmod 644 $(printf '%q' "${target_pub}")" - fi -} - -prepareLocalWorkDir() { - mkdir -p "${outputTmpDir}" - LOCAL_WORK_DIR="${outputTmpDir}/multiHostKvm" - rm -rf "${LOCAL_WORK_DIR}" - mkdir -p "${LOCAL_WORK_DIR}" -} - -createHostVms() { - # Args are read from one hosts-tsv row. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local remote_work_dir="${10}" - local host_dir="${LOCAL_WORK_DIR}/${name}" - local host_kvm_yaml="${host_dir}/kvm.yaml" - - echo "[multi-host-create] ${name} (${ip})" - mkdir -p "${host_dir}" - python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-local-kvm \ - --host "${name}" \ - --output "${host_kvm_yaml}" >/dev/null - - syncDirToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${SETUP_DIR}/kvm" "${remote_work_dir}/kvm" - copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${host_kvm_yaml}" "${remote_work_dir}/kvm.yaml" - copyVmSshKeyToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - cd $(printf '%q' "${remote_work_dir}") - python3 ./kvm/prepareHostAssets.py ./kvm.yaml - python3 ./kvm/createKvmVms.py ./kvm.yaml - " -} - -writeGlobalOutputs() { - python3 "${HELPER}" --config "${CONFIG_PATH}" write-global-k3s-config \ - --output "${outputConfigK3s}" >/dev/null - python3 "${HELPER}" --config "${CONFIG_PATH}" write-state \ - --output "${outputMultiHostKvmState}" >/dev/null - echo "K3s config: ${outputConfigK3s}" - echo "Multi-host KVM state: ${outputMultiHostKvmState}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - requireCommand scp - requireCommand rsync - requireCommand cp - - [ -s "${CONFIG_PATH}" ] || { - echo "Missing config: ${CONFIG_PATH}" >&2 - exit 1 - } - - prepareLocalWorkDir - python3 "${HELPER}" --config "${CONFIG_PATH}" validate - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - createHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) - writeGlobalOutputs - echo "Multi-host KVM VMs are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py index 8d8a21890..9437d7b16 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py @@ -1,21 +1,166 @@ #!/usr/bin/env python3 -"""Python entrypoint for destroyMultiHostKvmVms.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for destroyMultiHostKvmVms with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs and routed networks created by the multi-host KVM flow. +# +# Inputs: +# $1: multiHostKvmState.yaml. Defaults to ../multiHostKvmState.yaml. +# +# Side effects: +# - Runs each host-local kvm/destroyKvmVms.py on its owning hypervisor. +# - Removes static routes between hypervisor VM subnets. +# - Removes optional point-to-point VXLAN route tunnels. +# - Destroys and undefines the generated libvirt routed networks. +# - Removes generated local configK3s/kubeconfig/inventory/state outputs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +STATE_PATH="${1:-${SETUP_DIR}/multiHostKvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" &2; sleep 2; python3 ./kvm/destroyKvmVms.py ./kvmState.yaml; } + fi + virsh net-destroy $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-undefine $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -t nat -D POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + fi + rm -rf $(printf '%q' "${remote_work_dir}") + " + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + echo " delete route ${remote_cidr} via ${peer_ip} (${peer_name})" + if [ -n "${route_dev}" ]; then + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") dev $(printf '%q' "${route_dev}") >/dev/null 2>&1 || sudo -n ip route del $(printf '%q' "${remote_cidr}") >/dev/null 2>&1 || true" + else + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") >/dev/null 2>&1 || true" + fi + done < <(python3 "${HELPER}" --config /dev/null state-routes-tsv --state "${STATE_PATH}" --host "${name}") + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " delete tunnel ${tunnel_name} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true" + done < <(python3 "${HELPER}" --config /dev/null state-tunnels-tsv --state "${STATE_PATH}" --host "${name}") +} + +cleanupLocalOutputs() { + eval "$(python3 "${HELPER}" --config /dev/null state-output-vars --state "${STATE_PATH}")" + rm -f "${stateConfigK3s}" + rm -f "${stateKubeconfig}" + rm -f "${stateInventory}" + rm -f "${STATE_PATH}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + + [ -s "${STATE_PATH}" ] || { + echo "Missing state file: ${STATE_PATH}" >&2 + exit 1 + } + + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + destroyHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config /dev/null state-hosts-tsv --state "${STATE_PATH}") + cleanupLocalOutputs + echo "Multi-host KVM cleanup completed." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh deleted file mode 100755 index a7051f182..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -# Destroy KVM VMs and routed networks created by the multi-host KVM flow. -# -# Inputs: -# $1: multiHostKvmState.yaml. Defaults to ../multiHostKvmState.yaml. -# -# Side effects: -# - Runs each host-local kvm/destroyKvmVms.py on its owning hypervisor. -# - Removes static routes between hypervisor VM subnets. -# - Removes optional point-to-point VXLAN route tunnels. -# - Destroys and undefines the generated libvirt routed networks. -# - Removes generated local configK3s/kubeconfig/inventory/state outputs. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -STATE_PATH="${1:-${SETUP_DIR}/multiHostKvmState.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" &2; sleep 2; python3 ./kvm/destroyKvmVms.py ./kvmState.yaml; } - fi - virsh net-destroy $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - virsh net-undefine $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - if command -v iptables >/dev/null 2>&1; then - sudo -n iptables -t nat -D POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || true - sudo -n iptables -D FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true - sudo -n iptables -D FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true - fi - rm -rf $(printf '%q' "${remote_work_dir}") - " - - while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do - [ -n "${remote_cidr}" ] || continue - echo " delete route ${remote_cidr} via ${peer_ip} (${peer_name})" - if [ -n "${route_dev}" ]; then - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route del $(printf '%q' "${remote_cidr}") dev $(printf '%q' "${route_dev}") >/dev/null 2>&1 || sudo -n ip route del $(printf '%q' "${remote_cidr}") >/dev/null 2>&1 || true" - else - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route del $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") >/dev/null 2>&1 || true" - fi - done < <(python3 "${HELPER}" --config /dev/null state-routes-tsv --state "${STATE_PATH}" --host "${name}") - - while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do - [ -n "${tunnel_name}" ] || continue - echo " delete tunnel ${tunnel_name} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true" - done < <(python3 "${HELPER}" --config /dev/null state-tunnels-tsv --state "${STATE_PATH}" --host "${name}") -} - -cleanupLocalOutputs() { - eval "$(python3 "${HELPER}" --config /dev/null state-output-vars --state "${STATE_PATH}")" - rm -f "${stateConfigK3s}" - rm -f "${stateKubeconfig}" - rm -f "${stateInventory}" - rm -f "${STATE_PATH}" -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - - [ -s "${STATE_PATH}" ] || { - echo "Missing state file: ${STATE_PATH}" >&2 - exit 1 - } - - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - destroyHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config /dev/null state-hosts-tsv --state "${STATE_PATH}") - cleanupLocalOutputs - echo "Multi-host KVM cleanup completed." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py index 094f5c0c0..2bd7bb324 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py @@ -576,7 +576,7 @@ def writeHostNetworkXml(args: argparse.Namespace) -> None: def writeHostLocalKvm(args: argparse.Namespace) -> None: - """Write host-local kvm.yaml consumed by kvm/createKvmVms.sh.""" + """Write host-local kvm.yaml consumed by kvm/createKvmVms.py.""" data = loadYaml(args.config) host = requireHost(hostList(data), args.host) vm_ssh = data.get("vmSsh") if isinstance(data.get("vmSsh"), dict) else {} @@ -631,6 +631,9 @@ def writeGlobalK3sConfig(args: argparse.Namespace) -> None: "name": node["name"], "role": node["role"], "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), "ssh": {"user": ssh_user, "key": ssh_key}, } for node in vmPlan(data) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py index cd1d90f07..ac9d86ebb 100755 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py @@ -1,21 +1,217 @@ #!/usr/bin/env python3 -"""Python entrypoint for prepareKvmHypervisors.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for prepareKvmHypervisors with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare physical KVM hypervisors for a multi-host KVM SeedEMU cluster. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Generated/modified state: +# - Creates one routed libvirt network per hypervisor. +# - Enables IPv4 forwarding on each hypervisor. +# - Optionally creates point-to-point VXLAN route tunnels. +# - Installs static routes between the hypervisor VM subnets. +# +# Execution context: +# Run from the generated setup directory before createMultiHostKvmVms.py. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +TMP_DIR="" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +cleanupTmp() { + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" || true +} + +validateConfig() { + python3 "${HELPER}" --config "${CONFIG_PATH}" validate +} + +prepareHost() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local network_name="$6" + local bridge_name="$7" + local cidr="$8" + local remote_work_dir="${10}" + local network_xml="${TMP_DIR}/${name}.xml" + local remote_xml="${remote_work_dir}/network.xml" + + echo "[hypervisor-prepare] ${name} (${ip}) network=${network_name} bridge=${bridge_name}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-network-xml \ + --host "${name}" \ + --output "${network_xml}" >/dev/null + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n true && command -v virsh >/dev/null && command -v virt-install >/dev/null && command -v qemu-img >/dev/null && command -v docker >/dev/null && command -v curl >/dev/null && command -v ssh >/dev/null" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "${remote_work_dir}")" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" "${network_xml}" "${remote_xml}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + virsh net-info $(printf '%q' "${network_name}") >/dev/null 2>&1 || virsh net-define $(printf '%q' "${remote_xml}") >/dev/null + virsh net-start $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-autostart $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + sudo -n sysctl -w net.ipv4.ip_forward=1 >/dev/null + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -C FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -C FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -t nat -C POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || sudo -n iptables -t nat -A POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE + fi + " + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " tunnel ${tunnel_name}: ${local_tunnel_cidr} -> ${peer_tunnel_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + underlay_dev=\$(ip route get $(printf '%q' "${peer_ip}") | awk '{for (i=1;i<=NF;i++) if (\$i==\"dev\") {print \$(i+1); exit}}') + [ -n \"\${underlay_dev}\" ] || { echo 'Cannot determine underlay dev for peer $(printf '%q' "${peer_ip}")' >&2; exit 1; } + sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true + sudo -n ip link add $(printf '%q' "${tunnel_name}") type vxlan id $(printf '%q' "${vni}") local $(printf '%q' "${local_ip}") remote $(printf '%q' "${peer_ip}") dstport $(printf '%q' "${dst_port}") dev \"\${underlay_dev}\" + sudo -n ip addr add $(printf '%q' "${local_tunnel_cidr}") dev $(printf '%q' "${tunnel_name}") + sudo -n ip link set $(printf '%q' "${tunnel_name}") up + " + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-tunnels-tsv --host "${name}") + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + if [ -n "${route_dev}" ]; then + echo " route ${remote_cidr} via ${peer_ip} dev ${route_dev} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") dev $(printf '%q' "${route_dev}")" + else + echo " route ${remote_cidr} via ${peer_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}")" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-routes-tsv --host "${name}") +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand cp + requireCommand mktemp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + TMP_DIR="$(mktemp -d "${SETUP_DIR}/tmp.multi-host-kvm.XXXXXX")" + trap cleanupTmp EXIT + + validateConfig + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + prepareHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + + echo "KVM hypervisors are ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh deleted file mode 100755 index 436dcbfb7..000000000 --- a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -# Prepare physical KVM hypervisors for a multi-host KVM SeedEMU cluster. -# -# Inputs: -# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. -# -# Generated/modified state: -# - Creates one routed libvirt network per hypervisor. -# - Enables IPv4 forwarding on each hypervisor. -# - Optionally creates point-to-point VXLAN route tunnels. -# - Installs static routes between the hypervisor VM subnets. -# -# Execution context: -# Run from the generated setup directory before createMultiHostKvmVms.py. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" -HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" -TMP_DIR="" - -usage() { - cat </dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -sshOptions() { - # Args: - # $1: SSH private key path. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o IdentityAgent=none \ - -i "${ssh_key}" -} - -runHostCommand() { - # Args: - # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" - - if [ "${connection}" = "local" ]; then - bash -lc "${libvirt_command}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null -} - -cleanupTmp() { - [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" || true -} - -validateConfig() { - python3 "${HELPER}" --config "${CONFIG_PATH}" validate -} - -prepareHost() { - # Args are read from one hosts-tsv row. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local network_name="$6" - local bridge_name="$7" - local cidr="$8" - local remote_work_dir="${10}" - local network_xml="${TMP_DIR}/${name}.xml" - local remote_xml="${remote_work_dir}/network.xml" - - echo "[hypervisor-prepare] ${name} (${ip}) network=${network_name} bridge=${bridge_name}" - python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-network-xml \ - --host "${name}" \ - --output "${network_xml}" >/dev/null - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n true && command -v virsh >/dev/null && command -v virt-install >/dev/null && command -v qemu-img >/dev/null && command -v docker >/dev/null && command -v curl >/dev/null && command -v ssh >/dev/null" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "mkdir -p $(printf '%q' "${remote_work_dir}")" - copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" "${network_xml}" "${remote_xml}" - - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - virsh net-info $(printf '%q' "${network_name}") >/dev/null 2>&1 || virsh net-define $(printf '%q' "${remote_xml}") >/dev/null - virsh net-start $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - virsh net-autostart $(printf '%q' "${network_name}") >/dev/null 2>&1 || true - sudo -n sysctl -w net.ipv4.ip_forward=1 >/dev/null - if command -v iptables >/dev/null 2>&1; then - sudo -n iptables -C FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT - sudo -n iptables -C FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT - sudo -n iptables -t nat -C POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || sudo -n iptables -t nat -A POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE - fi - " - - while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do - [ -n "${tunnel_name}" ] || continue - echo " tunnel ${tunnel_name}: ${local_tunnel_cidr} -> ${peer_tunnel_ip} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " - set -euo pipefail - underlay_dev=\$(ip route get $(printf '%q' "${peer_ip}") | awk '{for (i=1;i<=NF;i++) if (\$i==\"dev\") {print \$(i+1); exit}}') - [ -n \"\${underlay_dev}\" ] || { echo 'Cannot determine underlay dev for peer $(printf '%q' "${peer_ip}")' >&2; exit 1; } - sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true - sudo -n ip link add $(printf '%q' "${tunnel_name}") type vxlan id $(printf '%q' "${vni}") local $(printf '%q' "${local_ip}") remote $(printf '%q' "${peer_ip}") dstport $(printf '%q' "${dst_port}") dev \"\${underlay_dev}\" - sudo -n ip addr add $(printf '%q' "${local_tunnel_cidr}") dev $(printf '%q' "${tunnel_name}") - sudo -n ip link set $(printf '%q' "${tunnel_name}") up - " - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-tunnels-tsv --host "${name}") - - while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do - [ -n "${remote_cidr}" ] || continue - if [ -n "${route_dev}" ]; then - echo " route ${remote_cidr} via ${peer_ip} dev ${route_dev} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") dev $(printf '%q' "${route_dev}")" - else - echo " route ${remote_cidr} via ${peer_ip} (${peer_name})" - runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}")" - fi - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-routes-tsv --host "${name}") -} - -main() { - if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 - fi - requireCommand python3 - requireCommand ssh - requireCommand scp - requireCommand cp - requireCommand mktemp - - [ -s "${CONFIG_PATH}" ] || { - echo "Missing config: ${CONFIG_PATH}" >&2 - exit 1 - } - - TMP_DIR="$(mktemp -d "${SETUP_DIR}/tmp.multi-host-kvm.XXXXXX")" - trap cleanupTmp EXIT - - validateConfig - while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do - prepareHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ - "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) - - echo "KVM hypervisors are ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py index e05f462d5..ddbc243b2 100755 --- a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py @@ -1,21 +1,137 @@ #!/usr/bin/env python3 -"""Python entrypoint for cleanKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for cleanKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove Kube-OVN non-primary CNI artifacts from a physical K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# Uninstalls the Kube-OVN Helm release when a live cluster is available and +# removes Kube-OVN/OVS/OVN runtime files from configured nodes. It does not +# reboot nodes; users can reboot manually if they need a fully cold network +# state as recommended by Kube-OVN upstream docs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to run as root. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[ovn-clean] ${name} (${ip})" + if [ "${connection}" = "local" ]; then + sudo -n bash -s <<<"${script}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"${script}" +} + +ensureHelm() { + # Print an existing helm binary or the private binary downloaded by + # installKubeOvnFabric.py. + if command -v helm >/dev/null 2>&1; then + command -v helm + return + fi + local helm_bin="${ovnHelmCacheDir}/bin/helm" + if [ -x "${helm_bin}" ]; then + echo "${helm_bin}" + return + fi + echo "" +} + +uninstallHelmRelease() { + # Remove Kubernetes resources while the API server is still available. + [ -s "${outputKubeconfig}" ] || return 0 + local helm_bin + helm_bin="$(ensureHelm)" + if [ -n "${helm_bin}" ]; then + "${helm_bin}" uninstall "${ovnReleaseName}" -n "${ovnNamespace}" --kubeconfig "${outputKubeconfig}" >/dev/null 2>&1 || true + else + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete deploy,ds,sts,svc,cm,sa,role,rolebinding \ + -l app.kubernetes.io/instance="${ovnReleaseName}" --ignore-not-found >/dev/null 2>&1 || true + fi +} + +clean_node_script=' +set -euo pipefail +rm -rf /var/run/openvswitch +rm -rf /var/run/ovn +rm -rf /etc/origin/openvswitch +rm -rf /etc/origin/ovn +rm -rf /etc/cni/net.d/00-kube-ovn.conflist +rm -rf /etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/00-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/log/openvswitch +rm -rf /var/log/ovn +rm -rf /var/log/kube-ovn +ip link del ovn0 2>/dev/null || true +' + +uninstallHelmRelease +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + runNodeRootScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "${clean_node_script}" +done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) + +echo "Kube-OVN fabric cleaned." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh deleted file mode 100755 index a87412506..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# Remove Kube-OVN non-primary CNI artifacts from a physical K3s cluster. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml and the kubeconfig generated by buildK3sCluster.sh. -# -# Side effects: -# Uninstalls the Kube-OVN Helm release when a live cluster is available and -# removes Kube-OVN/OVS/OVN runtime files from configured nodes. It does not -# reboot nodes; users can reboot manually if they need a fully cold network -# state as recommended by Kube-OVN upstream docs. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; nothing to clean." - exit 0 -fi - -sshOptions() { - # Args: - # $1: SSH private key used for the target node. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - # Args: - # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, - # $5: SSH key, $6: shell script to run as root. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - - echo "[ovn-clean] ${name} (${ip})" - if [ "${connection}" = "local" ]; then - sudo -n bash -s <<<"${script}" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"${script}" -} - -ensureHelm() { - # Print an existing helm binary or the private binary downloaded by - # installKubeOvnFabric.py. - if command -v helm >/dev/null 2>&1; then - command -v helm - return - fi - local helm_bin="${ovnHelmCacheDir}/bin/helm" - if [ -x "${helm_bin}" ]; then - echo "${helm_bin}" - return - fi - echo "" -} - -uninstallHelmRelease() { - # Remove Kubernetes resources while the API server is still available. - [ -s "${outputKubeconfig}" ] || return 0 - local helm_bin - helm_bin="$(ensureHelm)" - if [ -n "${helm_bin}" ]; then - "${helm_bin}" uninstall "${ovnReleaseName}" -n "${ovnNamespace}" --kubeconfig "${outputKubeconfig}" >/dev/null 2>&1 || true - else - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete deploy,ds,sts,svc,cm,sa,role,rolebinding \ - -l app.kubernetes.io/instance="${ovnReleaseName}" --ignore-not-found >/dev/null 2>&1 || true - fi -} - -clean_node_script=' -set -euo pipefail -rm -rf /var/run/openvswitch -rm -rf /var/run/ovn -rm -rf /etc/origin/openvswitch -rm -rf /etc/origin/ovn -rm -rf /etc/cni/net.d/00-kube-ovn.conflist -rm -rf /etc/cni/net.d/01-kube-ovn.conflist -rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/00-kube-ovn.conflist -rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/01-kube-ovn.conflist -rm -rf /var/log/openvswitch -rm -rf /var/log/ovn -rm -rf /var/log/kube-ovn -ip link del ovn0 2>/dev/null || true -' - -uninstallHelmRelease -while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - runNodeRootScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "${clean_node_script}" -done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) - -echo "Kube-OVN fabric cleaned." diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py index 4d09b25d0..db591c5bc 100755 --- a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py @@ -1,21 +1,337 @@ #!/usr/bin/env python3 -"""Python entrypoint for installKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for installKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Install Kube-OVN as a non-primary CNI for SeedEMU secondary networks. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml with fabric.type=ovn and optional ovn.* settings. +# +# Outputs/side effects: +# Installs the Kube-OVN Helm release into the configured K3s cluster. K3s +# flannel remains the primary CNI for eth0; Kube-OVN only serves Multus +# secondary interfaces used by SeedEMU simulated networks. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN install." + exit 0 +fi + +requireCommand() { + # Args: + # $1: Command name required by this script. + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + # Args: + # $1: duration accepted by timeout, remaining args: command to run. + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to execute on the node. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + if [ "${connection}" = "local" ]; then + bash -s <<<"${script}" + return + fi + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "bash -s" <<<"${script}" +} + +preloadKubeOvnImage() { + # Pull Kube-OVN once on the setup host, then import the image tar into each + # K3s node's containerd. This avoids repeated node-side Docker Hub pulls. + local image="docker.io/kubeovn/kube-ovn:${ovnChartVersion}" + local mirror_image="docker.m.daocloud.io/kubeovn/kube-ovn:${ovnChartVersion}" + local tar_path="${ovnHelmCacheDir}/kube-ovn_${ovnChartVersion}.tar" + mkdir -p "$(dirname "${tar_path}")" + echo "[ovn] preloading ${image} into K3s containerd nodes" + if ! runWithTimeout 180s sudo -n docker pull "${image}"; then + sudo -n docker pull "${mirror_image}" + sudo -n docker tag "${mirror_image}" "${image}" + fi + sudo -n docker save "${image}" -o "${tar_path}" + sudo -n chmod 0644 "${tar_path}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " import ${image} to ${name}" + if [ "${nodeConnection}" = "local" ]; then + sudo -n k3s ctr images import "${tar_path}" + else + local remote_tar="/tmp/$(basename "${tar_path}")" + # shellcheck disable=SC2046 + scp $(sshOptions "${nodeSshKey}") "${tar_path}" "${nodeSshUser}@${ip}:${remote_tar}" + # shellcheck disable=SC2046 + ssh -n $(sshOptions "${nodeSshKey}") "${nodeSshUser}@${ip}" \ + "sudo -n k3s ctr images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +prepareKubeOvnCniBinDir() { + # Prepare each node's K3s CNI binary directory for Kube-OVN install-cni. + # Args: none. Reads nodes and ovnCniBinDir from configK3s.yaml. + # + # K3s often keeps CNI plugins as symlinks under /var/lib/rancher/k3s/data/cni. + # Kube-OVN's install-cni init container copies these exact binaries into + # that same mounted path and fails if a target is a dangling symlink inside + # the container mount. Remove only the known overwrite targets. + echo "[ovn] preparing K3s CNI binary directory on all nodes: ${ovnCniBinDir}" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " prepare ${name}" + runNodeScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "$(cat </dev/null 2>&1; then + command -v helm + return + fi + + local helm_dir="${ovnHelmCacheDir}/bin" + local helm_bin="${helm_dir}/helm" + if [ -x "${helm_bin}" ]; then + echo "${helm_bin}" + return + fi + + mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" + local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" + echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 + curl -fsSL https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive}" + tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" + cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" + chmod +x "${helm_bin}" + echo "${helm_bin}" +} + +installKubeOvn() { + # Install Kube-OVN in non-primary mode. Kube-OVN's CNI binary is placed into + # the K3s CNI bin dir so Multus can invoke type=kube-ovn delegates. + local helm_bin="$1" + + "${helm_bin}" repo add "${ovnHelmRepoName}" "${ovnHelmRepoUrl}" >/dev/null 2>&1 || true + "${helm_bin}" repo update >/dev/null + + kubectl --kubeconfig "${outputKubeconfig}" label node "${k3sMasterName}" kube-ovn/role=master --overwrite + + echo "[ovn] installing Kube-OVN ${ovnChartVersion} in non-primary CNI mode" + "${helm_bin}" upgrade --install "${ovnReleaseName}" "${ovnHelmRepoName}/kube-ovn" \ + --kubeconfig "${outputKubeconfig}" \ + --namespace "${ovnNamespace}" \ + --version "${ovnChartVersion}" \ + --set cni_conf.NON_PRIMARY_CNI=true \ + --set cni_conf.CNI_CONF_DIR="${ovnCniConfDir}" \ + --set cni_conf.MOUNT_CNI_CONF_DIR="${ovnMountCniConfDir}" \ + --set cni_conf.CNI_BIN_DIR="${ovnCniBinDir}" \ + --set networking.TUNNEL_TYPE="${ovnTunnelType}" \ + --set networking.IFACE="${ovnIface}" \ + --set ipv4.POD_CIDR="${ovnPodCidr}" \ + --set ipv4.POD_GATEWAY="${ovnPodGateway}" \ + --set ipv4.SVC_CIDR="${ovnServiceCidr}" \ + --set ipv4.JOIN_CIDR="${ovnJoinCidr}" \ + --set MASTER_NODES="${ovnMasterNodes}" \ + --wait \ + --timeout 10m +} + +waitKubeOvnReady() { + # Confirm the controller, OVS/OVN daemonset, and CNI daemonset are rolled + # out before the running stage creates Kube-OVN NAD/Subnet resources. + local resources=( + "deployment/kube-ovn-controller" + "deployment/ovn-central" + "daemonset/ovs-ovn" + "daemonset/kube-ovn-cni" + ) + for resource in "${resources[@]}"; do + if kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get "${resource}" >/dev/null 2>&1; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status "${resource}" --timeout=600s + fi + done + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=300s +} + +repairK3sCniBinDirAfterKubeOvnInstall() { + # Kube-OVN's install-cni init container runs inside a container mount of + # the K3s CNI bin dir. On K3s this can leave the host dir without the + # primary loopback/portmap links or without the kube-ovn delegate binary. + # Use a short-lived hostNetwork pod per node so this repair does not depend + # on the CNI plugin path that it is fixing. + echo "[ovn] verifying K3s CNI binaries after Kube-OVN install" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + local pod_name + pod_name="seedemu-ovn-cni-repair-$(printf '%s' "${name}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')" + pod_name="${pod_name%-}" + echo " repair ${name} with pod/${pod_name}" + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" \ + --ignore-not-found=true --wait=false >/dev/null 2>&1 || true + cat </dev/null || true)" + if [ "${phase}" = "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=80 || true + break + fi + if [ "${phase}" = "Failed" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=120 || true + return 1 + fi + sleep 2 + done + local final_phase + final_phase="$(kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + if [ "${final_phase}" != "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" describe pod "${pod_name}" || true + return 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" --wait=false >/dev/null 2>&1 || true + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +main() { + requireCommand curl + requireCommand kubectl + requireCommand python3 + requireCommand scp + requireCommand ssh + requireCommand docker + requireCommand tar + + if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + echo "Run applyK3sCluster.py or k8sTools.py build before installing Kube-OVN." >&2 + exit 1 + fi + + local helm_bin + helm_bin="$(ensureHelm)" + preloadKubeOvnImage + prepareKubeOvnCniBinDir + installKubeOvn "${helm_bin}" + waitKubeOvnReady + repairK3sCniBinDirAfterKubeOvnInstall + echo "Kube-OVN non-primary CNI is ready." +} + +main "$@" +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh deleted file mode 100755 index 597dfd34f..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash -# Install Kube-OVN as a non-primary CNI for SeedEMU secondary networks. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml with fabric.type=ovn and optional ovn.* settings. -# -# Outputs/side effects: -# Installs the Kube-OVN Helm release into the configured K3s cluster. K3s -# flannel remains the primary CNI for eth0; Kube-OVN only serves Multus -# secondary interfaces used by SeedEMU simulated networks. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; skipping OVN install." - exit 0 -fi - -requireCommand() { - # Args: - # $1: Command name required by this script. - command -v "$1" >/dev/null 2>&1 || { - echo "Missing required command: $1" >&2 - exit 1 - } -} - -runWithTimeout() { - # Args: - # $1: duration accepted by timeout, remaining args: command to run. - local duration="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "${duration}" "$@" - else - "$@" - fi -} - -sshOptions() { - # Args: - # $1: SSH private key used for the target node. - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeScript() { - # Args: - # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, - # $5: SSH key, $6: shell script to execute on the node. - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - if [ "${connection}" = "local" ]; then - bash -s <<<"${script}" - return - fi - # shellcheck disable=SC2046 - ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "bash -s" <<<"${script}" -} - -preloadKubeOvnImage() { - # Pull Kube-OVN once on the setup host, then import the image tar into each - # K3s node's containerd. This avoids repeated node-side Docker Hub pulls. - local image="docker.io/kubeovn/kube-ovn:${ovnChartVersion}" - local mirror_image="docker.m.daocloud.io/kubeovn/kube-ovn:${ovnChartVersion}" - local tar_path="${ovnHelmCacheDir}/kube-ovn_${ovnChartVersion}.tar" - mkdir -p "$(dirname "${tar_path}")" - echo "[ovn] preloading ${image} into K3s containerd nodes" - if ! runWithTimeout 180s sudo -n docker pull "${image}"; then - sudo -n docker pull "${mirror_image}" - sudo -n docker tag "${mirror_image}" "${image}" - fi - sudo -n docker save "${image}" -o "${tar_path}" - sudo -n chmod 0644 "${tar_path}" - - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - echo " import ${image} to ${name}" - if [ "${nodeConnection}" = "local" ]; then - sudo -n k3s ctr images import "${tar_path}" - else - local remote_tar="/tmp/$(basename "${tar_path}")" - # shellcheck disable=SC2046 - scp $(sshOptions "${nodeSshKey}") "${tar_path}" "${nodeSshUser}@${ip}:${remote_tar}" - # shellcheck disable=SC2046 - ssh -n $(sshOptions "${nodeSshKey}") "${nodeSshUser}@${ip}" \ - "sudo -n k3s ctr images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" - fi - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) -} - -prepareKubeOvnCniBinDir() { - # Prepare each node's K3s CNI binary directory for Kube-OVN install-cni. - # Args: none. Reads nodes and ovnCniBinDir from configK3s.yaml. - # - # K3s often keeps CNI plugins as symlinks under /var/lib/rancher/k3s/data/cni. - # Kube-OVN's install-cni init container copies these exact binaries into - # that same mounted path and fails if a target is a dangling symlink inside - # the container mount. Remove only the known overwrite targets. - echo "[ovn] preparing K3s CNI binary directory on all nodes: ${ovnCniBinDir}" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" - echo " prepare ${name}" - runNodeScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "$(cat </dev/null 2>&1; then - command -v helm - return - fi - - local helm_dir="${ovnHelmCacheDir}/bin" - local helm_bin="${helm_dir}/helm" - if [ -x "${helm_bin}" ]; then - echo "${helm_bin}" - return - fi - - mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" - local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" - echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 - curl -fsSL https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive}" - tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" - cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" - chmod +x "${helm_bin}" - echo "${helm_bin}" -} - -installKubeOvn() { - # Install Kube-OVN in non-primary mode. Kube-OVN's CNI binary is placed into - # the K3s CNI bin dir so Multus can invoke type=kube-ovn delegates. - local helm_bin="$1" - - "${helm_bin}" repo add "${ovnHelmRepoName}" "${ovnHelmRepoUrl}" >/dev/null 2>&1 || true - "${helm_bin}" repo update >/dev/null - - kubectl --kubeconfig "${outputKubeconfig}" label node "${k3sMasterName}" kube-ovn/role=master --overwrite - - echo "[ovn] installing Kube-OVN ${ovnChartVersion} in non-primary CNI mode" - "${helm_bin}" upgrade --install "${ovnReleaseName}" "${ovnHelmRepoName}/kube-ovn" \ - --kubeconfig "${outputKubeconfig}" \ - --namespace "${ovnNamespace}" \ - --version "${ovnChartVersion}" \ - --set cni_conf.NON_PRIMARY_CNI=true \ - --set cni_conf.CNI_CONF_DIR="${ovnCniConfDir}" \ - --set cni_conf.MOUNT_CNI_CONF_DIR="${ovnMountCniConfDir}" \ - --set cni_conf.CNI_BIN_DIR="${ovnCniBinDir}" \ - --set networking.TUNNEL_TYPE="${ovnTunnelType}" \ - --set networking.IFACE="${ovnIface}" \ - --set ipv4.POD_CIDR="${ovnPodCidr}" \ - --set ipv4.POD_GATEWAY="${ovnPodGateway}" \ - --set ipv4.SVC_CIDR="${ovnServiceCidr}" \ - --set ipv4.JOIN_CIDR="${ovnJoinCidr}" \ - --set MASTER_NODES="${ovnMasterNodes}" \ - --wait \ - --timeout 10m -} - -waitKubeOvnReady() { - # Confirm the controller, OVS/OVN daemonset, and CNI daemonset are rolled - # out before the running stage creates Kube-OVN NAD/Subnet resources. - local resources=( - "deployment/kube-ovn-controller" - "deployment/ovn-central" - "daemonset/ovs-ovn" - "daemonset/kube-ovn-cni" - ) - for resource in "${resources[@]}"; do - if kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get "${resource}" >/dev/null 2>&1; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status "${resource}" --timeout=600s - fi - done - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=300s - kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=300s -} - -repairK3sCniBinDirAfterKubeOvnInstall() { - # Kube-OVN's install-cni init container runs inside a container mount of - # the K3s CNI bin dir. On K3s this can leave the host dir without the - # primary loopback/portmap links or without the kube-ovn delegate binary. - # Use a short-lived hostNetwork pod per node so this repair does not depend - # on the CNI plugin path that it is fixing. - echo "[ovn] verifying K3s CNI binaries after Kube-OVN install" - while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do - local pod_name - pod_name="seedemu-ovn-cni-repair-$(printf '%s' "${name}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')" - pod_name="${pod_name%-}" - echo " repair ${name} with pod/${pod_name}" - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" \ - --ignore-not-found=true --wait=false >/dev/null 2>&1 || true - cat </dev/null || true)" - if [ "${phase}" = "Succeeded" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=80 || true - break - fi - if [ "${phase}" = "Failed" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=120 || true - return 1 - fi - sleep 2 - done - local final_phase - final_phase="$(kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "${final_phase}" != "Succeeded" ]; then - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" describe pod "${pod_name}" || true - return 1 - fi - kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" --wait=false >/dev/null 2>&1 || true - done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) -} - -main() { - requireCommand curl - requireCommand kubectl - requireCommand python3 - requireCommand scp - requireCommand ssh - requireCommand docker - requireCommand tar - - if [ ! -s "${outputKubeconfig}" ]; then - echo "Kubeconfig not found: ${outputKubeconfig}" >&2 - echo "Run buildK3sCluster.sh before installing Kube-OVN." >&2 - exit 1 - fi - - local helm_bin - helm_bin="$(ensureHelm)" - preloadKubeOvnImage - prepareKubeOvnCniBinDir - installKubeOvn "${helm_bin}" - waitKubeOvnReady - repairK3sCniBinDirAfterKubeOvnInstall - echo "Kube-OVN non-primary CNI is ready." -} - -main "$@" diff --git a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py index 0e7dd68c7..5b6ea0346 100755 --- a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py @@ -1,21 +1,62 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateKubeOvnFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateKubeOvnFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that Kube-OVN non-primary CNI is ready for SeedEMU NADs. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# None. This script only reads cluster state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN validation." + exit 0 +fi + +if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + exit 1 +fi + +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pods -o wide +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status deployment/kube-ovn-controller --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/kube-ovn-cni --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/ovs-ovn --timeout=300s +echo "Kube-OVN fabric validation passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh deleted file mode 100755 index f83175151..000000000 --- a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Validate that Kube-OVN non-primary CNI is ready for SeedEMU NADs. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. -# -# Inputs: -# configK3s.yaml and the kubeconfig generated by buildK3sCluster.sh. -# -# Side effects: -# None. This script only reads cluster state. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "${CONFIG_PATH}" ]; then - echo "Missing config file: ${CONFIG_PATH}" >&2 - exit 1 -fi - -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" -eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" - -if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then - echo "No Kube-OVN fabric configured; skipping OVN validation." - exit 0 -fi - -if [ ! -s "${outputKubeconfig}" ]; then - echo "Kubeconfig not found: ${outputKubeconfig}" >&2 - exit 1 -fi - -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pods -o wide -kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s -kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status deployment/kube-ovn-controller --timeout=300s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/kube-ovn-cni --timeout=300s -kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/ovs-ovn --timeout=300s -echo "Kube-OVN fabric validation passed." diff --git a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py index a1c744731..99981ac62 100755 --- a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py +++ b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py @@ -1,21 +1,86 @@ #!/usr/bin/env python3 -"""Python entrypoint for preparePhysicalNodes.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for preparePhysicalNodes with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate physical nodes before building a SeedEMU K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Checks: +# - Each node can be reached with the configured connection method. +# - sudo -n works for the configured user. +# - iproute2 and curl are available. +# - linux-vxlan fabric nodes expose their configured underlay interface. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + if [ "$connection" = "local" ]; then + bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "$command" /dev/null && command -v curl >/dev/null" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" +if [ "${fabricType}" = "linux-vxlan" ]; then + echo "Validating linux-vxlan underlay interfaces..." + while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + echo " ${name}: underlay=${underlay}" + runNodeCommand "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "ip link show $(printf '%q' "$underlay") >/dev/null" + done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +fi + +echo "Physical node preflight passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh deleted file mode 100755 index 0697de0cf..000000000 --- a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Validate physical nodes before building a SeedEMU K3s cluster. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Checks: -# - Each node can be reached with the configured connection method. -# - sudo -n works for the configured user. -# - iproute2 and curl are available. -# - linux-vxlan fabric nodes expose their configured underlay interface. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_PATH="${1:-./configK3s.yaml}" -HELPER="${SCRIPT_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeCommand() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - - if [ "$connection" = "local" ]; then - bash -lc "$command" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "$command" /dev/null && command -v curl >/dev/null" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" -if [ "${fabricType}" = "linux-vxlan" ]; then - echo "Validating linux-vxlan underlay interfaces..." - while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - echo " ${name}: underlay=${underlay}" - runNodeCommand "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "ip link show $(printf '%q' "$underlay") >/dev/null" - done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -fi - -echo "Physical node preflight passed." diff --git a/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py index 212f3006c..0e0b17d8f 100644 --- a/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py +++ b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py @@ -1,19 +1,21 @@ -"""Run adjacent shell resources for k8sTools Python entrypoints.""" +"""Run embedded shell bodies for k8sTools Python entrypoints.""" from __future__ import annotations +import os import subprocess from pathlib import Path -def runAdjacentShell(script_path: Path, argv: list[str]) -> int: - """Execute the .sh file next to a Python entrypoint. +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. Args: script_path: Path to the Python entrypoint being executed. argv: Arguments passed by the caller. + shell_body: Embedded bash program. """ - shell_path = script_path.resolve().with_suffix(".sh") - if not shell_path.is_file(): - raise FileNotFoundError(f"Missing shell resource: {shell_path}") - completed = subprocess.run(["bash", str(shell_path), *argv]) + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py index 053381455..9288effe6 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py @@ -1,21 +1,99 @@ #!/usr/bin/env python3 -"""Python entrypoint for cleanLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for cleanLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove the Linux VXLAN bridge fabric described by configK3s.yaml. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Deletes the configured test macvlan interface, VXLAN interface, and bridge +# on every configured node. It does not uninstall K3s or delete workload pods. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +cleanup_script=' +set -euo pipefail +macvlan_name="$1" +vxlan_name="$2" +bridge_name="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link del "$vxlan_name" 2>/dev/null || true +ip link del "$bridge_name" 2>/dev/null || true +' + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$cleanup_script" \ + "$fabricMacvlanTestName" "$fabricVxlanName" "$fabricBridgeName" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Linux VXLAN fabric cleaned." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh deleted file mode 100755 index c192707bb..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Remove the Linux VXLAN bridge fabric described by configK3s.yaml. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Side effects: -# Deletes the configured test macvlan interface, VXLAN interface, and bridge -# on every configured node. It does not uninstall K3s or delete workload pods. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; nothing to clean." - exit 0 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-clean] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -cleanup_script=' -set -euo pipefail -macvlan_name="$1" -vxlan_name="$2" -bridge_name="$3" - -ip link del "$macvlan_name" 2>/dev/null || true -ip link del "$vxlan_name" 2>/dev/null || true -ip link del "$bridge_name" 2>/dev/null || true -' - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$cleanup_script" \ - "$fabricMacvlanTestName" "$fabricVxlanName" "$fabricBridgeName" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -echo "Linux VXLAN fabric cleaned." diff --git a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py index dd350f622..040f6d024 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py @@ -1,21 +1,144 @@ #!/usr/bin/env python3 -"""Python entrypoint for configureLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for configureLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create a two-node Linux VXLAN bridge fabric for SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Expected config: +# fabric.type: linux-vxlan +# fabric.nodes..underlayInterface: management NIC used as VXLAN underlay. +# +# Side effects: +# Creates a bridge such as br-seedemu and a VXLAN device such as vxseed0 on +# each node. On failure, it calls cleanLinuxVxlanFabric.py to roll back all +# configured fabric interfaces. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric setup." + exit 0 +fi + +rollbackOnError() { + local rc=$? + if [ "$rc" -ne 0 ]; then + echo "[fabric] setup failed; rolling back configured fabric interfaces" >&2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap rollbackOnError EXIT + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-configure] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +configure_script=' +set -euo pipefail +bridge_name="$1" +vxlan_name="$2" +vni="$3" +local_ip="$4" +peer_ip="$5" +underlay="$6" +dst_port="$7" +mtu="$8" + +if [ "${#bridge_name}" -gt 15 ] || [ "${#vxlan_name}" -gt 15 ]; then + echo "Linux interface names must be at most 15 characters" >&2 + exit 1 +fi + +ip link show "$underlay" >/dev/null + +if ip link show "$vxlan_name" >/dev/null 2>&1; then + ip link del "$vxlan_name" +fi + +if ! ip link show "$bridge_name" >/dev/null 2>&1; then + ip link add "$bridge_name" type bridge +fi + +ip link set "$bridge_name" up +ip link add "$vxlan_name" type vxlan id "$vni" local "$local_ip" remote "$peer_ip" dev "$underlay" dstport "$dst_port" +ip link set "$vxlan_name" mtu "$mtu" || true +ip link set "$vxlan_name" master "$bridge_name" +ip link set "$vxlan_name" up +' + +echo "Configuring Linux VXLAN fabric:" +echo " bridge=${fabricBridgeName}" +echo " vxlan=${fabricVxlanName}" +echo " vni=${fabricVni}" +echo " dstPort=${fabricDstPort}" + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$configure_script" \ + "$fabricBridgeName" "$fabricVxlanName" "$fabricVni" "$ip" "$peer_ip" \ + "$underlay" "$fabricDstPort" "$fabricMtu" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +trap - EXIT +echo "Linux VXLAN fabric configured." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh deleted file mode 100755 index add805819..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -# Create a two-node Linux VXLAN bridge fabric for SeedEMU Multus/macvlan. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Expected config: -# fabric.type: linux-vxlan -# fabric.nodes..underlayInterface: management NIC used as VXLAN underlay. -# -# Side effects: -# Creates a bridge such as br-seedemu and a VXLAN device such as vxseed0 on -# each node. On failure, it calls cleanLinuxVxlanFabric.py to roll back all -# configured fabric interfaces. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; skipping fabric setup." - exit 0 -fi - -rollbackOnError() { - local rc=$? - if [ "$rc" -ne 0 ]; then - echo "[fabric] setup failed; rolling back configured fabric interfaces" >&2 - python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - fi - exit "$rc" -} -trap rollbackOnError EXIT - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-configure] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -configure_script=' -set -euo pipefail -bridge_name="$1" -vxlan_name="$2" -vni="$3" -local_ip="$4" -peer_ip="$5" -underlay="$6" -dst_port="$7" -mtu="$8" - -if [ "${#bridge_name}" -gt 15 ] || [ "${#vxlan_name}" -gt 15 ]; then - echo "Linux interface names must be at most 15 characters" >&2 - exit 1 -fi - -ip link show "$underlay" >/dev/null - -if ip link show "$vxlan_name" >/dev/null 2>&1; then - ip link del "$vxlan_name" -fi - -if ! ip link show "$bridge_name" >/dev/null 2>&1; then - ip link add "$bridge_name" type bridge -fi - -ip link set "$bridge_name" up -ip link add "$vxlan_name" type vxlan id "$vni" local "$local_ip" remote "$peer_ip" dev "$underlay" dstport "$dst_port" -ip link set "$vxlan_name" mtu "$mtu" || true -ip link set "$vxlan_name" master "$bridge_name" -ip link set "$vxlan_name" up -' - -echo "Configuring Linux VXLAN fabric:" -echo " bridge=${fabricBridgeName}" -echo " vxlan=${fabricVxlanName}" -echo " vni=${fabricVni}" -echo " dstPort=${fabricDstPort}" - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$configure_script" \ - "$fabricBridgeName" "$fabricVxlanName" "$fabricVni" "$ip" "$peer_ip" \ - "$underlay" "$fabricDstPort" "$fabricMtu" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -trap - EXIT -echo "Linux VXLAN fabric configured." diff --git a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py index 037d32d26..889d212ce 100755 --- a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py +++ b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py @@ -1,21 +1,194 @@ #!/usr/bin/env python3 -"""Python entrypoint for validateLinuxVxlanFabric.sh. - -The shell command body lives in the adjacent .sh resource so changes remain -reviewable and can be validated with bash -n. This wrapper keeps the Python -entrypoint used by k8sTools while preserving the script directory seen by bash. -""" +"""Python entrypoint for validateLinuxVxlanFabric with its shell body embedded.""" from __future__ import annotations import sys from pathlib import Path -from _embeddedShell import runAdjacentShell +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate the Linux VXLAN bridge fabric used by SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Validation: +# 1. Adds temporary /30 IPs to the configured bridge and pings both ways. +# 2. Creates temporary macvlan interfaces on the bridge and pings both ways. +# +# Side effects: +# Temporary test IPs and macvlan interfaces are always removed. If validation +# fails, the configured VXLAN fabric is also removed to avoid stale state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric validation." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +runNodeRootCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -lc $(printf '%q' "$command")" &2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap cleanupOnExit EXIT + +setup_bridge_ip=' +set -euo pipefail +bridge_name="$1" +bridge_test_ip="$2" + +ip link show "$bridge_name" >/dev/null +ip addr del "$bridge_test_ip" dev "$bridge_name" 2>/dev/null || true +ip addr add "$bridge_test_ip" dev "$bridge_name" +ip link set "$bridge_name" up +' + +setup_macvlan=' +set -euo pipefail +bridge_name="$1" +macvlan_name="$2" +macvlan_test_ip="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link add "$macvlan_name" link "$bridge_name" type macvlan mode bridge +ip addr add "$macvlan_test_ip" dev "$macvlan_name" +ip link set "$macvlan_name" up +' + +echo "Validating Linux VXLAN bridge reachability..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_bridge_ip" \ + "$fabricBridgeName" "$bridge_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 1 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_bridge_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $8; exit}')" + peer_bridge_ip="${peer_bridge_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -c 3 -W 2 $(printf '%q' "$peer_bridge_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Removing bridge test IPs before macvlan validation..." +cleanupTestArtifacts +sleep 1 + +echo "Validating macvlan reachability over Linux VXLAN bridge..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_macvlan" \ + "$fabricBridgeName" "$fabricMacvlanTestName" "$macvlan_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 2 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_macvlan_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $9; exit}')" + peer_macvlan_ip="${peer_macvlan_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -I $(printf '%q' "$fabricMacvlanTestName") -c 3 -W 2 $(printf '%q' "$peer_macvlan_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +cleanupTestArtifacts +trap - EXIT +echo "Linux VXLAN fabric validation passed." +''' def main(argv: list[str] | None = None) -> int: """Run this entrypoint with optional argv override for tests.""" - return runAdjacentShell(Path(__file__), list(sys.argv[1:] if argv is None else argv)) + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) if __name__ == "__main__": diff --git a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh deleted file mode 100755 index 5f7321362..000000000 --- a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash -# Validate the Linux VXLAN bridge fabric used by SeedEMU Multus/macvlan. -# -# Args: -# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. -# -# Validation: -# 1. Adds temporary /30 IPs to the configured bridge and pings both ways. -# 2. Creates temporary macvlan interfaces on the bridge and pings both ways. -# -# Side effects: -# Temporary test IPs and macvlan interfaces are always removed. If validation -# fails, the configured VXLAN fabric is also removed to avoid stale state. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" -HELPER="${SETUP_DIR}/manageK3sConfig.py" - -if [ ! -s "$CONFIG_PATH" ]; then - echo "Missing config file: $CONFIG_PATH" >&2 - exit 1 -fi - -eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" - -if [ "${fabricType}" != "linux-vxlan" ]; then - echo "No linux-vxlan fabric configured; skipping fabric validation." - exit 0 -fi - -sshOptions() { - local ssh_key="$1" - printf '%s\n' \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=10 \ - -i "$ssh_key" -} - -runNodeRootScript() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local script="$6" - shift 6 - - echo "[fabric-validate] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -s -- "$@" <<<"$script" - return - fi - - local quoted_args="" - local arg - for arg in "$@"; do - quoted_args+=" $(printf '%q' "$arg")" - done - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" -} - -runNodeRootCommand() { - local name="$1" - local ip="$2" - local connection="$3" - local ssh_user="$4" - local ssh_key="$5" - local command="$6" - - echo "[fabric-validate] ${name} (${ip})" - if [ "$connection" = "local" ]; then - sudo -n bash -lc "$command" - return - fi - - # shellcheck disable=SC2046 - ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -lc $(printf '%q' "$command")" &2 - python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true - fi - exit "$rc" -} -trap cleanupOnExit EXIT - -setup_bridge_ip=' -set -euo pipefail -bridge_name="$1" -bridge_test_ip="$2" - -ip link show "$bridge_name" >/dev/null -ip addr del "$bridge_test_ip" dev "$bridge_name" 2>/dev/null || true -ip addr add "$bridge_test_ip" dev "$bridge_name" -ip link set "$bridge_name" up -' - -setup_macvlan=' -set -euo pipefail -bridge_name="$1" -macvlan_name="$2" -macvlan_test_ip="$3" - -ip link del "$macvlan_name" 2>/dev/null || true -ip link add "$macvlan_name" link "$bridge_name" type macvlan mode bridge -ip addr add "$macvlan_test_ip" dev "$macvlan_name" -ip link set "$macvlan_name" up -' - -echo "Validating Linux VXLAN bridge reachability..." -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_bridge_ip" \ - "$fabricBridgeName" "$bridge_test_ip" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -sleep 1 - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - peer_bridge_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $8; exit}')" - peer_bridge_ip="${peer_bridge_test_ip%/*}" - runNodeRootCommand \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ - "ping -c 3 -W 2 $(printf '%q' "$peer_bridge_ip")" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -echo "Removing bridge test IPs before macvlan validation..." -cleanupTestArtifacts -sleep 1 - -echo "Validating macvlan reachability over Linux VXLAN bridge..." -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - runNodeRootScript \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_macvlan" \ - "$fabricBridgeName" "$fabricMacvlanTestName" "$macvlan_test_ip" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) -sleep 2 - -while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do - peer_macvlan_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $9; exit}')" - peer_macvlan_ip="${peer_macvlan_test_ip%/*}" - runNodeRootCommand \ - "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ - "ping -I $(printf '%q' "$fabricMacvlanTestName") -c 3 -W 2 $(printf '%q' "$peer_macvlan_ip")" -done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) - -cleanupTestArtifacts -trap - EXIT -echo "Linux VXLAN fabric validation passed." diff --git a/seedemu/k8sTools/runner.py b/seedemu/k8sTools/runner.py index e6d6facfa..597750267 100644 --- a/seedemu/k8sTools/runner.py +++ b/seedemu/k8sTools/runner.py @@ -74,6 +74,7 @@ def buildCluster( config_k3s: str | Path, kubeconfig: str | Path, *, + inventory: str | Path | None = None, keep_temp: bool = False, ) -> None: """Build KVM/physical infrastructure and write final config outputs. @@ -82,6 +83,7 @@ def buildCluster( input_config: User YAML with explicit `kind`. config_k3s: Final configK3s.yaml path requested by the user. kubeconfig: Final kubeconfig path requested by the user. + inventory: Optional final cluster inventory YAML path requested by the user. keep_temp: Keep temporary setup resources for debugging. The first-phase implementation expands bundled setup resources into a @@ -92,6 +94,7 @@ def buildCluster( input_path = resolvePath(input_config) config_path = resolvePath(config_k3s) kubeconfig_path = resolvePath(kubeconfig) + inventory_path = resolvePath(inventory) if inventory is not None else None source = loadYaml(input_path) kind = detectKind(source, input_path) @@ -99,11 +102,11 @@ def buildCluster( setup_dir = copyTree("setup", root / "setup", overwrite=True) chmodScripts(setup_dir) if kind == "kvmOvn": - _buildKvmOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) elif kind == "multiHostKvmOvn": - _buildMultiHostKvmOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildMultiHostKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) elif kind == "physicalOvn": - _buildPhysicalOvn(setup_dir, input_path, config_path, kubeconfig_path) + _buildPhysicalOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) else: raise ValueError(f"unsupported kind: {kind}") @@ -253,14 +256,20 @@ def destroyCluster(config_k3s: str | Path, *, keep_temp: bool = False) -> None: raise ValueError(f"unsupported destroy type: {destroy_type}") -def _buildKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a single-hypervisor KVM cluster with Kube-OVN.""" kvm_config = makeKvmConfig( config=input_path, setup_dir=setup_dir, k3s_config_path=setup_dir / "configK3s.yaml", kubeconfig_path=kubeconfig_path, - inventory_path=setup_dir / "cluster.inventory.yaml", + inventory_path=inventory_path or setup_dir / "cluster.inventory.yaml", tmp_dir=setup_dir / "tmp", ) kvm_config.setdefault("fabric", {})["type"] = "ovn" @@ -283,7 +292,13 @@ def _buildKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfi writeYaml(config_path, final_config) -def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildMultiHostKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a multi-hypervisor KVM cluster with routed VM subnets and Kube-OVN.""" source = loadYaml(input_path) source.setdefault("fabric", {})["type"] = "ovn" @@ -291,7 +306,12 @@ def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, outputs = source.setdefault("outputs", {}) outputs["k3sConfig"] = str(setup_dir / "configK3s.yaml") outputs["multiHostKvmState"] = str(setup_dir / "multiHostKvmState.yaml") - setOutputPaths(source, kubeconfig=kubeconfig_path, tmp_dir=setup_dir / "tmp", inventory=setup_dir / "cluster.inventory.yaml") + setOutputPaths( + source, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) temp_input = setup_dir / "multiHostInput.yaml" writeYaml(temp_input, source) kvm_config = makeMultiHostKvmConfig(config=temp_input, setup_dir=setup_dir) @@ -313,13 +333,24 @@ def _buildMultiHostKvmOvn(setup_dir: Path, input_path: Path, config_path: Path, writeYaml(config_path, final_config) -def _buildPhysicalOvn(setup_dir: Path, input_path: Path, config_path: Path, kubeconfig_path: Path) -> None: +def _buildPhysicalOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: """Build a K3s + Kube-OVN cluster on existing physical nodes.""" config = loadYaml(input_path) config["kind"] = "physicalOvn" config.setdefault("fabric", {})["type"] = "ovn" _applyOvnK3sDefaults(config) - setOutputPaths(config, kubeconfig=kubeconfig_path, tmp_dir=setup_dir / "tmp", inventory=setup_dir / "cluster.inventory.yaml") + setOutputPaths( + config, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) writeYaml(setup_dir / "configK3s.yaml", config) runCommand(["python3", str(setup_dir / "preparePhysicalNodes.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) runCommand(["python3", str(setup_dir / "applyK3sCluster.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) diff --git a/seedemu/k8sTools/utils.py b/seedemu/k8sTools/utils.py index 09cde405b..6b30f3d2b 100644 --- a/seedemu/k8sTools/utils.py +++ b/seedemu/k8sTools/utils.py @@ -69,13 +69,13 @@ def copyResourceItems( def chmodScripts(path: str | Path) -> None: - """Make every bundled script entrypoint below path executable. + """Make every bundled Python script entrypoint below path executable. Args: path: Root directory copied from package resources. """ root = Path(path).expanduser() - for script in [*root.rglob("*.py"), *root.rglob("*.sh")]: + for script in root.rglob("*.py"): mode = script.stat().st_mode script.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) diff --git a/seedemu/layers/Ebgp.py b/seedemu/layers/Ebgp.py index 66cad84b6..0f128d4fb 100644 --- a/seedemu/layers/Ebgp.py +++ b/seedemu/layers/Ebgp.py @@ -7,6 +7,9 @@ EbgpFileTemplates: Dict[str, str] = {} +# Keep the senior Ebgp_* protocol names stable: the K3s phased startup helper +# enables eBGP sessions by matching this prefix at runtime. + EbgpFileTemplates["bgp_commons"] = """\ define LOCAL_COMM = ({localAsn}, 0, 0); define CUSTOMER_COMM = ({localAsn}, 1, 0); @@ -15,6 +18,7 @@ """ EbgpFileTemplates["rs_bird_peer"] = """ + #debug {{states}}; ipv4 {{ import all; export all; @@ -25,6 +29,7 @@ """ EbgpFileTemplates["rnode_bird_peer"] = """ + #debug {{states}}; ipv4 {{ table t_bgp; import filter {{ @@ -106,20 +111,39 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel # pipe direct routes to bgp, set LOCAL community, set pref 40 node.addTablePipe('t_direct', 't_bgp', exportFilter = 'filter { bgp_large_community.add(LOCAL_COMM); bgp_local_pref = 40; accept; }') + # # Also export intra-AS OSPF link prefixes into t_bgp so edge routers + # # can advertise all net_* link subnets, not only their own directly + # # connected ones. The generated topologies use /24 for point-to-point + # # net_* links, while ix prefixes are /16 and loopbacks are /32. + # node.addTablePipe( + # 'master4', + # 't_bgp', + # exportFilter = ( + # 'filter { ' + # 'if source = RTS_OSPF && net.len = 24 then { ' + # 'bgp_large_community.add(LOCAL_COMM); ' + # 'bgp_local_pref = 40; ' + # 'accept; ' + # '} ' + # 'reject; ' + # '}' + # ) + # ) + assert routerA != None, 'both nodes are RS node. cannot setup peering.' assert routerA != routerB, 'cannot peer with oneself.' if rsNode != None: - rsNode.addProtocol('bgp', 'p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rs_bird_peer"].format( + rsNode.addProtocol('bgp', 'Ebgp_p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rs_bird_peer"].format( localAddress = addrA, localAsn = rsNode.getAsn(), peerAddress = addrB, peerAsn = routerA.getAsn() )) - routerA.addProtocol('bgp', 'p_rs{}'.format(rsNode.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerA.addProtocol('bgp', 'Ebgp_p_rs{}'.format(rsNode.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrB, localAsn = routerA.getAsn(), peerAddress = addrA, @@ -132,7 +156,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel return if rel == PeerRelationship.Peer: - routerA.addProtocol('bgp', 'p_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerA.addProtocol('bgp', 'Ebgp_p_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrA, localAsn = routerA.getAsn(), peerAddress = addrB, @@ -142,7 +166,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel bgpPref = 20 )) - routerB.addProtocol('bgp', 'p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerB.addProtocol('bgp', 'Ebgp_p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrB, localAsn = routerB.getAsn(), peerAddress = addrA, @@ -153,7 +177,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel )) if rel == PeerRelationship.Provider: - routerA.addProtocol('bgp', 'c_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerA.addProtocol('bgp', 'Ebgp_c_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrA, localAsn = routerA.getAsn(), peerAddress = addrB, @@ -163,7 +187,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel bgpPref = 30 )) - routerB.addProtocol('bgp', 'u_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerB.addProtocol('bgp', 'Ebgp_u_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrB, localAsn = routerB.getAsn(), peerAddress = addrA, @@ -174,7 +198,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel )) if rel == PeerRelationship.Unfiltered: - routerA.addProtocol('bgp', 'x_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerA.addProtocol('bgp', 'Ebgp_x_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrA, localAsn = routerA.getAsn(), peerAddress = addrB, @@ -184,7 +208,7 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel bgpPref = 30 )) - routerB.addProtocol('bgp', 'x_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( + routerB.addProtocol('bgp', 'Ebgp_x_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( localAddress = addrB, localAsn = routerB.getAsn(), peerAddress = addrA, diff --git a/seedemu/layers/Ibgp.py b/seedemu/layers/Ibgp.py index 2f8b9d438..f26e737ac 100644 --- a/seedemu/layers/Ibgp.py +++ b/seedemu/layers/Ibgp.py @@ -2,11 +2,14 @@ from seedemu.core.enums import NetworkType, NodeRole from .Base import Base from seedemu.core import ScopedRegistry, Node, Graphable, Emulator, Layer -from typing import List, Set, Dict +from typing import List, Set, Dict, Tuple IbgpFileTemplates: Dict[str, str] = {} IbgpFileTemplates['ibgp_peer'] = ''' + # debug {{states,events}}; + # hold time 36000; + # keepalive time 60; ipv4 {{ table t_bgp; import all; @@ -17,11 +20,46 @@ neighbor {peerAddress} as {asn}; ''' +# Standard iBGP peer template used for clients, RR mesh peers, and full mesh. +IbgpFileTemplates['ibgp_client'] = ''' + # debug {{states,events}}; + # hold time 36000; + # keepalive time 60; + ipv4 {{ + table t_bgp; + import all; + export all; + igp table t_ospf; + next hop self; + }}; + local {localAddress} as {asn}; + neighbor {peerAddress} as {asn}; +''' + +# Route Reflector server-side template used for RR-to-client sessions. +IbgpFileTemplates['ibgp_rr_server'] = ''' + # debug {{states,events}}; + # hold time 36000; + # keepalive time 60; + passive yes; + ipv4 {{ + table t_bgp; + import all; + export all; + igp table t_ospf; + }}; + local {localAddress} as {asn}; + neighbor {peerAddress} as {asn}; + rr client; + rr cluster id {clusterId}; +''' + class Ibgp(Layer, Graphable): """! @brief The Ibgp (iBGP) layer. - This layer automatically setup full mesh peering between routers within AS. + This layer automatically sets up full-mesh iBGP or Route Reflector based + iBGP sessions between routers within each AS. """ __masked: Set[int] @@ -97,31 +135,120 @@ def render(self, emulator: Emulator): self._log('setting up IBGP peering for as{}...'.format(asn)) routers: List[Node] = ScopedRegistry(str(asn), reg).getByType('rnode') + routers_map: Dict[str, Node] = {r.getName(): r for r in routers} + + as_obj = base.getAutonomousSystem(asn) + clusters = as_obj._aggregateBgpClusters() + has_rr = True + if len(clusters) == 1: + cid, (rrs, clients) = list(clusters.items())[0] + if len(rrs) == 0: + has_rr = False + if has_rr: + self._render_rr_mode(asn, clusters, routers_map) + else: + self._render_full_mesh_mode(asn, routers) + + def _render_rr_mode(self, asn: int, clusters: Dict[str, Tuple[List[str], List[str]]], routers_map: Dict[str, Node]): + """! + @brief Render Route Reflector based iBGP sessions for one AS. + + @param asn AS number being rendered. + @param clusters mapping from cluster ID to RR names and client names. + @param routers_map mapping from router name to router node. + """ + self._log(f'setting up IBGP (Route Reflector) for as{asn}...') + + # Collect every RR so they can be connected through an RR full mesh. + all_rr_names: Set[str] = set() + + # Configure each cluster as a hub-and-spoke RR topology. + for cluster_id, (rr_names, client_names) in clusters.items(): + all_rr_names.update(rr_names) + + for rr_name in rr_names: + if rr_name not in routers_map: continue + rr_node = routers_map[rr_name] + rr_node.addTable('t_bgp') + rr_node.addTablePipe('t_bgp') + rr_node.addTablePipe('t_direct', 't_bgp') + laddr = rr_node.getLoopbackAddress() + + for client_name in client_names: + if client_name not in routers_map: continue + client_node = routers_map[client_name] + + client_node.addTable('t_bgp') + client_node.addTablePipe('t_bgp') + client_node.addTablePipe('t_direct', 't_bgp') + raddr = client_node.getLoopbackAddress() + + # The RR side enables route reflection for this client. + rr_node.addProtocol('bgp', f'Ibgp_to_cli_{client_name}', IbgpFileTemplates['ibgp_rr_server'].format( + localAddress=laddr, peerAddress=raddr, asn=asn, clusterId=cluster_id + )) - for local in routers: - self._log('setting up IBGP peering on as{}/{}...'.format(asn, local.getName())) - - remotes = [] - self.__dfs(local, remotes) - - n = 1 - for remote in remotes: - if local == remote: continue - - laddr = local.getLoopbackAddress() - raddr = remote.getLoopbackAddress() - local.addTable('t_bgp') - local.addTablePipe('t_bgp') - local.addTablePipe('t_direct', 't_bgp') - local.addProtocol('bgp', 'ibgp{}'.format(n), IbgpFileTemplates['ibgp_peer'].format( - localAddress = laddr, - peerAddress = raddr, - asn = asn + # The client side uses a normal iBGP peer session to the RR. + client_node.addProtocol('bgp', f'Ibgp_to_rr_{rr_name}', IbgpFileTemplates['ibgp_client'].format( + localAddress=raddr, peerAddress=laddr, asn=asn )) + + self._log(f'adding RR peering: {rr_name}(RR) <-> {client_name}(Client) cluster {cluster_id}') - n += 1 + # Connect all RRs with a normal full mesh so reflected routes propagate. + sorted_rrs = sorted([routers_map[name] for name in all_rr_names if name in routers_map], key=lambda x: x.getName()) - self._log('adding peering: {} <-> {} (ibgp, as{})'.format(laddr, raddr, asn)) + for i in range(len(sorted_rrs)): + for j in range(i + 1, len(sorted_rrs)): + node_a = sorted_rrs[i] + node_b = sorted_rrs[j] + + node_a.addTable('t_bgp') + node_a.addTablePipe('t_bgp') + node_a.addTablePipe('t_direct', 't_bgp') + + node_b.addTable('t_bgp') + node_b.addTablePipe('t_bgp') + node_b.addTablePipe('t_direct', 't_bgp') + + # RR-to-RR sessions are normal bidirectional iBGP peer sessions. + node_a.addProtocol('bgp', f'Ibgp_mesh_{node_b.getName()}', IbgpFileTemplates['ibgp_peer'].format( + localAddress=node_a.getLoopbackAddress(), peerAddress=node_b.getLoopbackAddress(), asn=asn + )) + node_b.addProtocol('bgp', f'Ibgp_mesh_{node_a.getName()}', IbgpFileTemplates['ibgp_peer'].format( + localAddress=node_b.getLoopbackAddress(), peerAddress=node_a.getLoopbackAddress(), asn=asn + )) + self._log(f'adding RR Mesh: {node_a.getName()} <-> {node_b.getName()}') + + def _render_full_mesh_mode(self, asn: int, routers_list: List[Node]): + """! + @brief Render the legacy full-mesh iBGP sessions for one AS. + + @param asn AS number being rendered. + @param routers_list routers participating in the legacy iBGP mesh. + """ + self._log(f'setting up IBGP (Full Mesh) for as{asn}...') + + for local in routers_list: + remotes = [] + self.__dfs(local, remotes) + + n = 1 + for remote in remotes: + if local == remote: continue + + laddr = local.getLoopbackAddress() + raddr = remote.getLoopbackAddress() + local.addTable('t_bgp') + local.addTablePipe('t_bgp') + local.addTablePipe('t_direct', 't_bgp') + local.addProtocol('bgp', 'Ibgp{}'.format(n), IbgpFileTemplates['ibgp_peer'].format( + localAddress = laddr, + peerAddress = raddr, + asn = asn + )) + n += 1 + self._log('adding peering: {} <-> {} (ibgp, as{})'.format(laddr, raddr, asn)) def _doCreateGraphs(self, emulator: Emulator): base: Base = emulator.getRegistry().get('seedemu', 'layer', 'Base') @@ -157,4 +284,3 @@ def print(self, indent: int) -> str: out += '{}\n'.format(asn) return out - diff --git a/seedemu/layers/Ospf.py b/seedemu/layers/Ospf.py index b25de7e74..8106f4c08 100644 --- a/seedemu/layers/Ospf.py +++ b/seedemu/layers/Ospf.py @@ -11,15 +11,21 @@ import all; export all; }}; + tick 1; + area 0 {{ {interfaces} }}; """ +# OspfFileTemplates['ospf_interface'] = """\ +# interface "{interfaceName}" {{ hello 10; dead 40; type pointopoint; retransmit 20;}}; +# """ OspfFileTemplates['ospf_interface'] = """\ interface "{interfaceName}" {{ hello 1; dead count 2; }}; """ + OspfFileTemplates['ospf_stub_interface'] = """\ interface "{interfaceName}" {{ stub; }}; """ diff --git a/seedemu/layers/Routing.py b/seedemu/layers/Routing.py index 8cdbda534..430f2ef5e 100644 --- a/seedemu/layers/Routing.py +++ b/seedemu/layers/Routing.py @@ -4,7 +4,7 @@ from seedemu.core.enums import NetworkType from typing import List, Dict from ipaddress import IPv4Network - +import random RoutingFileTemplates: Dict[str, str] = {} RoutingFileTemplates["rs_bird"] = """\ @@ -18,17 +18,43 @@ interface "{interfaceName}"; """ -RoutingFileTemplates["rnode_bird"] = """\ -router id {routerId}; -ipv4 table t_direct; -protocol device {{ +RoutingFileTemplates["kernel1"] = """ +protocol kernel {{ + merge paths on; + persist; + scan time {interval}; + ipv4 {{ + import none; + # Core modification here: add a filter + export filter {{ + # Allow direct routes to be installed into the kernel (ensure basic connectivity) + if source = RTS_DEVICE then accept; + # Allow OSPF routes to be installed into the kernel (ensure iBGP Loopback reachability) + if source = RTS_OSPF then accept; + # Reject all other routes (including BGP routes) from being installed into the kernel! + reject; + }}; + }}; }} +""" + +RoutingFileTemplates["kernel2"] = """ protocol kernel {{ + merge paths on; + persist; + scan time {interval}; ipv4 {{ - import all; + import none; export all; }}; - learn; +}} +""" + + +RoutingFileTemplates["rnode_bird"] = """\ +router id {routerId}; +ipv4 table t_direct; +protocol device {{ }} """ @@ -89,8 +115,8 @@ def _installBird(self, node: Node): node.setBaseSystem(BaseSystem.SEEDEMU_ROUTER) def _configure_rs(self, rs_node: Node): - rs_node.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird') - rs_node.appendStartCommand('bird -d', True) + rs_node.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird ') + #rs_node.appendStartCommand('bird', True) self._log("Bootstrapping bird.conf for RS {}...".format(rs_node.getName())) rs_ifaces = rs_node.getInterfaces() @@ -117,8 +143,8 @@ def _configure_bird_router(self, rnode: Router): rnode.setFile("/etc/bird/bird.conf", RoutingFileTemplates["rnode_bird"].format( routerId = rnode.getLoopbackAddress())) - rnode.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird') - rnode.appendStartCommand('bird -d', True) + rnode.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird ') + #rnode.appendStartCommand('bird', True) if has_localnet: rnode.addProtocol('direct', 'local_nets', RoutingFileTemplates['rnode_bird_direct'].format(interfaces = ifaces)) @@ -186,6 +212,13 @@ def render(self, emulator: Emulator): if type == 'rs' or type == 'rnode': assert issubclass(obj.__class__, Router), 'routing: render: adding new RS/Router after routing layer configured is not currently supported.' + if type == 'rs' or type == 'rnode': + rnode: Router = obj + content1 = '\ninclude "/etc/bird/conf/*.conf";\n' + rnode.appendFile('/etc/bird/bird.conf',content1) + t=60000+random.randint(0, 12000) + rnode.setFile("/etc/bird/conf/kernel.conf",RoutingFileTemplates["kernel1"].format(interval=t)) + if type == 'rnode': rnode: Router = obj if rnode.hasExtension('RealWorldRouter'): # could also be ScionRouter which needs RealWorldAccess @@ -230,4 +263,4 @@ def print(self, indent: int) -> str: out = ' ' * indent out += 'RoutingLayer: BIRD 2.0.x\n' - return out \ No newline at end of file + return out From 3be8954aebff1fadca4392e293d145965c262940 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Wed, 17 Jun 2026 11:32:20 +0800 Subject: [PATCH 3/6] update --- seedemu/k8sTools/README.md | 13 +- seedemu/k8sTools/k8sTools.py | 17 + .../resources/running/buildRegistryImages.py | 1 + .../resources/running/manageK8sManifest.py | 109 +++- .../resources/running/manageRunningStage.py | 1 + .../setup/ovn/installKubeOvnFabric.py | 28 +- seedemu/k8sTools/runner.py | 32 +- tests/ci/README.md | 8 +- tests/ci/feature_manifest.json | 159 +++--- tests/ci/run_ci.py | 12 +- .../test_bgp_control_plane_extensions.py | 496 ------------------ .../control_plane/test_bgp_runtime_matrix.py | 413 --------------- 12 files changed, 244 insertions(+), 1045 deletions(-) delete mode 100644 tests/control_plane/test_bgp_control_plane_extensions.py delete mode 100644 tests/control_plane/test_bgp_runtime_matrix.py diff --git a/seedemu/k8sTools/README.md b/seedemu/k8sTools/README.md index a781e495e..0d6204e0b 100644 --- a/seedemu/k8sTools/README.md +++ b/seedemu/k8sTools/README.md @@ -23,16 +23,19 @@ Deploy workload: python k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml ``` -`up` infers the logical compiler image prefix from `images.yaml`. Pass +`up` infers the logical compiler image prefix from `images.yaml` or +`images.txt`. Pass `--image-registry-prefix ` only when the compile output intentionally uses a non-standard or mixed prefix. Clean workload only: ```bash -python k8sTools.py down -f ./output -k kubeconfig.yaml +python k8sTools.py clean -f ./output -k kubeconfig.yaml ``` +`down` remains accepted as a compatibility alias for `clean`. + Destroy infrastructure: ```bash @@ -61,9 +64,9 @@ Python entrypoints there: `up` copies bundled `resources/running/` into a temporary directory and invokes `manageRunningStage.py preflight`, `build`, and `up`. The running stage renders -`kustomization.yaml`, rewrites OVN manifests when needed, builds/pushes images -to the registry resolved from `configK3s.yaml`, applies the manifest, and waits -for readiness. +`kustomization.yaml`, rewrites OVN manifests when needed, reads image metadata +from `images.yaml` or `images.txt`, builds/pushes images to the registry +resolved from `configK3s.yaml`, applies the manifest, and waits for readiness. No persistent setup/running working directory is created by default. Persistent outputs are limited to the user-requested `configK3s.yaml`, `kubeconfig.yaml`, diff --git a/seedemu/k8sTools/k8sTools.py b/seedemu/k8sTools/k8sTools.py index 79492a2a6..8a2d64014 100644 --- a/seedemu/k8sTools/k8sTools.py +++ b/seedemu/k8sTools/k8sTools.py @@ -73,6 +73,16 @@ def down(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool """ cleanWorkload(outputDir, kubeconfig, keep_temp=keepTemp) + def clean(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool = False) -> None: + """Delete workload resources while keeping cluster infrastructure. + + Args: + outputDir: Compile output directory. + kubeconfig: Kubeconfig path. + keepTemp: Keep temporary cleanup resources for debugging. + """ + self.down(outputDir, kubeconfig, keepTemp=keepTemp) + def destroy(self, configK3s: str | Path, *, keepTemp: bool = False) -> None: """Destroy K3s/OVN and optional KVM infrastructure. @@ -113,6 +123,11 @@ def runCli(self, argv: list[str] | None = None) -> int: down.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") down.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + clean = sub.add_parser("clean", help="delete workload resources only") + clean.add_argument("-f", "--folder", required=True, help="compile output directory") + clean.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") + clean.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + destroy = sub.add_parser("destroy", help="destroy cluster and optional KVM resources") destroy.add_argument("-d", "--config-k3s", required=True, help="configK3s.yaml") destroy.add_argument("--keep-temp", action="store_true", help="keep temporary destroy resources") @@ -130,6 +145,8 @@ def runCli(self, argv: list[str] | None = None) -> int: ) elif args.command == "down": self.down(args.folder, args.kubeconfig, keepTemp=args.keep_temp) + elif args.command == "clean": + self.clean(args.folder, args.kubeconfig, keepTemp=args.keep_temp) elif args.command == "destroy": self.destroy(args.config_k3s, keepTemp=args.keep_temp) else: diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.py b/seedemu/k8sTools/resources/running/buildRegistryImages.py index 470ce8b68..921081de5 100755 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.py +++ b/seedemu/k8sTools/resources/running/buildRegistryImages.py @@ -71,6 +71,7 @@ fi IMAGES_YAML="${OUTPUT_DIR}/images.yaml" +[ -s "${IMAGES_YAML}" ] || IMAGES_YAML="${OUTPUT_DIR}/images.txt" OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" cd "${OUTPUT_DIR}" diff --git a/seedemu/k8sTools/resources/running/manageK8sManifest.py b/seedemu/k8sTools/resources/running/manageK8sManifest.py index 105cd43f5..bc4c9cbd6 100755 --- a/seedemu/k8sTools/resources/running/manageK8sManifest.py +++ b/seedemu/k8sTools/resources/running/manageK8sManifest.py @@ -278,7 +278,7 @@ def running_context(config_path: str) -> dict[str, str]: "setupConfig": str(setup_config_path), "outputDir": str(output_dir), "manifest": str(manifest_path), - "imagesYaml": str(output_dir / "images.yaml"), + "imagesYaml": str(resolveImagesPath(output_dir)), "kustomization": str(output_dir / "kustomization.yaml"), "imageRegistryPrefix": str(running.get("imageRegistryPrefix") or "seedemu"), "registryPrefix": f"{registry_host}:{registry_port}", @@ -329,6 +329,18 @@ def resolveManifestPath(running: dict[str, Any], output_dir: Path, network_backe return default_manifest +def resolveImagesPath(output_dir: Path) -> Path: + """Return the generated image metadata file for the compile output. + + Args: + output_dir: Compile output directory. + """ + images_yaml = output_dir / "images.yaml" + if images_yaml.exists(): + return images_yaml + return output_dir / "images.txt" + + def config_value(args: argparse.Namespace) -> None: """Print one resolved value from configRunning.yaml.""" values = running_context(args.config) @@ -379,8 +391,48 @@ def strip_prefix(image: str, prefix: str) -> str: def load_images(path: str) -> list[dict[str, str]]: - data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} - return data.get("images", []) + """Load image metadata from images.yaml or legacy images.txt. + + Args: + path: Compiler image metadata file. + """ + image_path = Path(path) + text = image_path.read_text(encoding="utf-8") + if image_path.name == "images.txt": + return imagesFromText(text) + data = yaml.safe_load(text) or {} + if isinstance(data, dict): + images = data.get("images", []) + if isinstance(images, list): + return images + return imagesFromText(text) + + +def imagesFromText(text: str) -> list[dict[str, str]]: + """Parse one-image-reference-per-line metadata from NativeKubernetesCompiler. + + Args: + text: Contents of images.txt. + """ + images: list[dict[str, str]] = [] + for line in text.splitlines(): + image = line.strip() + if not image or image.startswith("#"): + continue + images.append({"name": image, "context": contextFromImage(image)}) + return images + + +def contextFromImage(image: str) -> str: + """Infer a Docker build context directory from an image reference. + + Args: + image: Full image reference generated by the compiler. + """ + repo = image.rsplit("/", 1)[-1] + if ":" in repo: + repo = repo.rsplit(":", 1)[0] + return repo def mapped_images(args: argparse.Namespace) -> None: @@ -426,7 +478,7 @@ def render_kustomization(args: argparse.Namespace) -> None: ) payload = {"resources": [rendered_manifest.name], "images": images} else: - payload = {"resources": ["k8s.yaml"], "images": images} + payload = {"resources": namespaceResources(manifest_path, output_path.parent) + ["k8s.yaml"], "images": images} patches = networkAttachmentPatches(args.manifest, args.cni_master_interface) if patches: payload["patches"] = patches @@ -455,8 +507,7 @@ def renderKubeOvnManifest( output. In macvlan mode, Kube-OVN acts as the centralized IPAM plugin and avoids creating thousands of OVN logical switches/routes for scale tests. """ - with open(manifest_path, "r", encoding="utf-8") as fh: - docs = [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] + docs = loadManifestDocs(Path(manifest_path)) namespace_name = findNamespaceName(docs) attached_cni = resolveAttachedCniType(attached_cni_type) @@ -464,6 +515,9 @@ def renderKubeOvnManifest( vpc_name = kubeOvnResourceName("vpc", namespace_name) rendered: list[dict[str, Any]] = [] vpc_written = False + namespace_written = any(doc.get("kind") == "Namespace" for doc in docs) + if not namespace_written: + rendered.append(namespaceResource(namespace_name)) for doc in docs: if use_ovn_attached and doc.get("kind") == "Namespace" and not vpc_written: @@ -485,11 +539,33 @@ def renderKubeOvnManifest( rendered.append(doc) if use_ovn_attached and not vpc_written: - rendered.insert(0, kubeOvnVpc(namespace_name, vpc_name)) + insert_at = 1 if rendered and rendered[0].get("kind") == "Namespace" else 0 + rendered.insert(insert_at, kubeOvnVpc(namespace_name, vpc_name)) output_path.write_text(yaml.safe_dump_all(rendered, sort_keys=False), encoding="utf-8") +def namespaceResource(namespace_name: str) -> dict[str, Any]: + """Return a Kubernetes Namespace resource for compiler outputs without one.""" + return {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace_name}} + + +def namespaceResources(manifest_path: Path, output_dir: Path) -> list[str]: + """Write namespace.yaml when the manifest lacks an explicit Namespace. + + Args: + manifest_path: Source manifest path. + output_dir: Directory that will contain kustomization.yaml. + """ + docs = loadManifestDocs(manifest_path) + if any(doc.get("kind") == "Namespace" for doc in docs): + return [] + namespace_name = findNamespaceName(docs) + namespace_path = output_dir / "namespace.yaml" + namespace_path.write_text(yaml.safe_dump(namespaceResource(namespace_name), sort_keys=False), encoding="utf-8") + return [namespace_path.name] + + def resolveAttachedCniType(attached_cni_type: str | None = None) -> str: """Return the configured secondary CNI type for Kube-OVN rendering.""" value = ( @@ -501,6 +577,16 @@ def resolveAttachedCniType(attached_cni_type: str | None = None) -> str: return str(value).strip().lower().replace("_", "-") +def loadManifestDocs(manifest_path: Path) -> list[dict[str, Any]]: + """Load Kubernetes documents from a manifest path. + + Args: + manifest_path: YAML manifest path. + """ + with open(manifest_path, "r", encoding="utf-8") as fh: + return [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] + + def isKubeOvnAttachedCni(attached_cni_type: str) -> bool: """Return true when secondary NICs should use the Kube-OVN CNI directly.""" return attached_cni_type in {"kube-ovn", "ovn"} @@ -826,14 +912,7 @@ def deployment_names(args: argparse.Namespace) -> None: def namespace(args: argparse.Namespace) -> None: - with open(args.manifest, "r", encoding="utf-8") as fh: - for doc in yaml.safe_load_all(fh): - if isinstance(doc, dict) and doc.get("kind") == "Namespace": - name = (doc.get("metadata") or {}).get("name") - if name: - print(name) - return - raise SystemExit(f"No Namespace object found in {args.manifest}") + print(findNamespaceName(loadManifestDocs(Path(args.manifest)))) def validate_manifest(args: argparse.Namespace) -> None: diff --git a/seedemu/k8sTools/resources/running/manageRunningStage.py b/seedemu/k8sTools/resources/running/manageRunningStage.py index 5fbd4addc..6328d25e9 100755 --- a/seedemu/k8sTools/resources/running/manageRunningStage.py +++ b/seedemu/k8sTools/resources/running/manageRunningStage.py @@ -70,6 +70,7 @@ def context(config: Path) -> dict[str, str]: "masterConnection", "cniMasterInterface", "networkBackend", + "attachedCniType", "rolloutTimeoutSeconds", ] return {key: helperOutput(["config-value", "--config", str(config), "--key", key]) for key in keys} diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py index db591c5bc..160806a04 100755 --- a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py +++ b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py @@ -152,25 +152,47 @@ ensureHelm() { # Print a helm binary path. If helm is not installed on the host, download a # private copy under ovn.helmCacheDir so no system package is modified. - if command -v helm >/dev/null 2>&1; then + helmUsable() { + # Args: + # $1: helm binary candidate. + # A partial curl/tar extraction can leave an executable that segfaults. + # Validate by running `helm version` before trusting any cached binary. + local candidate="$1" + [ -x "${candidate}" ] || return 1 + "${candidate}" version --short >/dev/null 2>&1 + } + + if command -v helm >/dev/null 2>&1 && helmUsable "$(command -v helm)"; then command -v helm return fi local helm_dir="${ovnHelmCacheDir}/bin" local helm_bin="${helm_dir}/helm" - if [ -x "${helm_bin}" ]; then + if helmUsable "${helm_bin}"; then echo "${helm_bin}" return fi mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" + rm -f "${helm_bin}" local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" + local archive_tmp="${archive}.tmp" echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 - curl -fsSL https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive}" + rm -f "${archive_tmp}" + curl --fail --location --show-error --retry 5 --retry-delay 2 --retry-all-errors \ + https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive_tmp}" + tar -tzf "${archive_tmp}" >/dev/null + mv "${archive_tmp}" "${archive}" + rm -rf "${ovnHelmCacheDir}/download/linux-amd64" tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" chmod +x "${helm_bin}" + if ! helmUsable "${helm_bin}"; then + rm -f "${helm_bin}" + echo "Downloaded helm binary is not usable: ${helm_bin}" >&2 + exit 1 + fi echo "${helm_bin}" } diff --git a/seedemu/k8sTools/runner.py b/seedemu/k8sTools/runner.py index 597750267..7f5891b19 100644 --- a/seedemu/k8sTools/runner.py +++ b/seedemu/k8sTools/runner.py @@ -50,22 +50,32 @@ def temporaryWorkDir(prefix: str, keep_temp: bool = False) -> Iterator[Path]: def inferImageRegistryPrefix(output_dir: Path) -> str: - """Infer the compiler image prefix from images.yaml. + """Infer the compiler image prefix from generated image metadata. Args: - output_dir: Compile output directory containing images.yaml. + output_dir: Compile output directory containing images.yaml or images.txt. """ images_path = output_dir / "images.yaml" - if not images_path.exists(): + if images_path.exists(): + data = loadYaml(images_path) + images = data.get("images") if isinstance(data.get("images"), list) else [] + for item in images: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + if "/" in name: + return name.rsplit("/", 1)[0] return "seedemu" - data = loadYaml(images_path) - images = data.get("images") if isinstance(data.get("images"), list) else [] - for item in images: - if not isinstance(item, dict): - continue - name = str(item.get("name") or "").strip() - if "/" in name: - return name.rsplit("/", 1)[0] + + images_txt = output_dir / "images.txt" + if images_txt.exists(): + for line in images_txt.read_text(encoding="utf-8").splitlines(): + name = line.strip() + if not name or name.startswith("#"): + continue + if "/" in name: + return name.rsplit("/", 1)[0] + return "seedemu" return "seedemu" diff --git a/tests/ci/README.md b/tests/ci/README.md index cfcdfd551..35053a67d 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -11,7 +11,8 @@ The CI runner writes both human-readable logs and machine-readable artifacts: - `junit.xml` records the same stage in a format GitHub and review tooling can ingest. - `feature-coverage.json` records the manifest-derived coverage state, including - declared gaps such as IPv6 work that has not landed on this integration line. + covered features and declared gaps that have not landed on this integration + line. The static stage compiles importable Python source plus representative examples. It intentionally excludes embedded payload templates under @@ -34,3 +35,8 @@ points, but they are not default pull-request gates in this PR: python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build python3 tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration ``` + +The runtime integration stage is kept as an explicit entry point for future +Docker runtime probes. The current manifest does not enable any runtime group on +this branch, so the stage is a reserved hook unless new groups are added to +`feature_manifest.json`. diff --git a/tests/ci/feature_manifest.json b/tests/ci/feature_manifest.json index ad33eed55..38667cb12 100644 --- a/tests/ci/feature_manifest.json +++ b/tests/ci/feature_manifest.json @@ -5,141 +5,106 @@ "status": "covered", "description": "Legacy IPv4 examples continue to compile and build without requiring any new control-plane capability.", "unit_groups": [], - "compile_examples": ["basic-a00-simple-as"], - "build_examples": ["basic-a00-simple-as"], + "compile_examples": [ + "basic-a00-simple-as" + ], + "build_examples": [ + "basic-a00-simple-as" + ], "runtime_groups": [] }, "ipv6-core": { "status": "declared-gap", - "description": "Repository-level IPv6 readiness is tracked here, but this integration branch does not yet carry the IPv6 readiness examples or runtime probes.", + "description": "Repository-level IPv6 readiness is tracked here, but this K8s integration branch does not currently carry a representative IPv6 example or runtime probe.", "unit_groups": [], "compile_examples": [], "build_examples": [], "runtime_groups": [], - "notes": "Do not mark IPv6 as covered on this branch until the IPv6 readiness work is merged into this integration line." + "notes": "Keep declared as a gap until native IPv6 topology/API coverage or another representative IPv6 example is added." }, "routing-bird-frr": { - "status": "covered", - "description": "Router-selected BIRD/FRR backends are rendered by Routing from protocol-layer intent.", - "unit_groups": ["control-plane-unit"], - "compile_examples": ["basic-a12-bgp-mixed-backend"], - "build_examples": ["basic-a12-bgp-mixed-backend"], - "runtime_groups": ["control-plane-runtime"] + "status": "declared-gap", + "description": "BIRD/FRR interoperability coverage from the source CI framework is not currently carried by this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the branch includes a maintained BIRD/FRR example and matching runtime probe." }, "exabgp-service": { - "status": "covered", - "description": "ExaBGP is exercised only as ExaBgpService plus Binding, including the IX speaker example.", - "unit_groups": ["control-plane-unit"], - "compile_examples": [ - "basic-a13-exabgp-control-plane", - "internet-b30-mini-internet-exabgp-ix" - ], - "build_examples": ["basic-a13-exabgp-control-plane"], - "runtime_groups": ["control-plane-runtime"] + "status": "declared-gap", + "description": "ExaBGP coverage exists on the source CI framework branch, but this K8s integration branch does not carry ExaBgpService or its examples.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until ExaBGP service support is present on this branch." }, "looking-glass": { - "status": "covered", - "description": "Looking Glass remains a route-state observer and is tested separately from ExaBGP event streams.", - "unit_groups": ["control-plane-unit"], - "compile_examples": ["basic-a14-bgp-event-looking-glass"], - "build_examples": ["basic-a14-bgp-event-looking-glass"], - "runtime_groups": ["control-plane-runtime"] + "status": "declared-gap", + "description": "Looking Glass service support is present, but the source CI framework examples are not present on this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the representative Looking Glass example is merged here." }, "dns-endpoint": { "status": "covered", "description": "DNS endpoint behavior is represented by the mini Internet plus DNS example.", "unit_groups": [], - "compile_examples": ["internet-b02-mini-internet-with-dns"], - "build_examples": ["internet-b02-mini-internet-with-dns"], + "compile_examples": [ + "internet-b02-mini-internet-with-dns" + ], + "build_examples": [ + "internet-b02-mini-internet-with-dns" + ], "runtime_groups": [] }, "legacy-guard": { - "status": "covered", - "description": "Removed legacy control-plane APIs stay removed through explicit guard tests.", - "unit_groups": ["control-plane-unit"], + "status": "declared-gap", + "description": "Removed legacy control-plane API guards exist on the source CI framework branch, but those tests are not present on this K8s integration branch.", + "unit_groups": [], "compile_examples": [], "build_examples": [], - "runtime_groups": [] - } - }, - "unit_groups": { - "control-plane-unit": { - "description": "Control-plane API, rendering, service, and legacy guard checks.", - "pytest_args": ["tests/control_plane/test_bgp_control_plane_extensions.py"] - } - }, - "runtime_groups": { - "control-plane-runtime": { - "description": "Docker runtime probes for selected BIRD/FRR, ExaBGP, and Looking Glass examples.", - "pytest_args": ["-m", "integration", "tests/control_plane/test_bgp_runtime_matrix.py"], - "default": false, - "notes": "Manual workflow-dispatch entry for this PR. PR3 will make runtime evidence richer and artifact-indexed." + "runtime_groups": [], + "notes": "Keep declared as a gap until control-plane guard tests are merged here." } }, + "unit_groups": {}, + "runtime_groups": {}, "examples": { "basic-a00-simple-as": { "description": "Small IPv4 default behavior smoke example.", "script": "examples/basic/A00_simple_as/simple_as.py", - "args": ["amd"], - "clean": ["output"], - "expected": ["output/docker-compose.yml"], - "compile": true, - "build": true - }, - "basic-a12-bgp-mixed-backend": { - "description": "Mixed BIRD/FRR routing backend example.", - "script": "examples/basic/A12_bgp_mixed_backend/bgp_mixed_backend.py", - "args": ["amd"], - "clean": ["output"], - "expected": ["output/docker-compose.yml"], - "compile": true, - "build": true - }, - "basic-a13-exabgp-control-plane": { - "description": "ExaBGP service plus Binding control-plane speaker example.", - "script": "examples/basic/A13_exabgp_control_plane/exabgp_control_plane.py", - "args": ["amd"], - "env": { - "SEED_A13_EXABGP_PORT": "5101" - }, - "clean": ["output"], - "expected": ["output/docker-compose.yml"], - "compile": true, - "build": true - }, - "basic-a14-bgp-event-looking-glass": { - "description": "Looking Glass route-state observer plus separate ExaBGP event dashboard example.", - "script": "examples/basic/A14_bgp_event_looking_glass/bgp_event_looking_glass.py", - "args": ["amd"], - "env": { - "SEED_A14_LG_PORT": "5002", - "SEED_A14_EVENT_PORT": "5003" - }, - "clean": ["output"], - "expected": ["output/docker-compose.yml"], + "args": [ + "amd" + ], + "clean": [ + "output" + ], + "expected": [ + "output/docker-compose.yml" + ], "compile": true, "build": true }, "internet-b02-mini-internet-with-dns": { "description": "Mini Internet with DNS component and local DNS endpoints.", "script": "examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py", - "args": ["amd"], - "clean": ["output", "base_internet.bin", "dns_component.bin"], - "expected": ["output/docker-compose.yml"], + "args": [ + "amd" + ], + "clean": [ + "output", + "base_internet.bin", + "dns_component.bin" + ], + "expected": [ + "output/docker-compose.yml" + ], "compile": true, "build": true - }, - "internet-b30-mini-internet-exabgp-ix": { - "description": "Internet-scale ExaBGP IX speaker compile coverage.", - "script": "examples/internet/B30_mini_internet_exabgp_ix/mini_internet_exabgp_ix.py", - "args": ["amd"], - "env": { - "SEED_B30_EXABGP_PORT": "5130" - }, - "clean": ["output", "base_internet.bin"], - "expected": ["output/docker-compose.yml"], - "compile": true, - "build": false } } } diff --git a/tests/ci/run_ci.py b/tests/ci/run_ci.py index 9d74d4ade..0594b0b49 100644 --- a/tests/ci/run_ci.py +++ b/tests/ci/run_ci.py @@ -352,9 +352,13 @@ def run_static(artifact_dir: Path) -> int: checks.append(_run_command("whitespace", _git_diff_check_command(), artifact_dir)) compile_targets = [ - "seedemu", - "tests/control_plane", - "tests/ci", + target + for target in [ + "seedemu", + "tests/control_plane", + "tests/ci", + ] + if (REPO_ROOT / target).exists() ] example_dirs = sorted( { @@ -382,7 +386,7 @@ def run_static(artifact_dir: Path) -> int: smoke = ( "import seedemu; " "from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf; " - "from seedemu.services import ExaBgpService, BgpLookingGlassService; " + "from seedemu.services import BgpLookingGlassService; " "from seedemu.compiler import Docker; " "import tests.ci.run_ci" ) diff --git a/tests/control_plane/test_bgp_control_plane_extensions.py b/tests/control_plane/test_bgp_control_plane_extensions.py deleted file mode 100644 index b271c2cd0..000000000 --- a/tests/control_plane/test_bgp_control_plane_extensions.py +++ /dev/null @@ -1,496 +0,0 @@ -from __future__ import annotations - -import importlib -import os -from pathlib import Path -import subprocess -import sys - -import pytest - -from seedemu.compiler import Docker, Platform -from seedemu.core import Binding, Emulator, Filter -from seedemu.core.Node import DEFAULT_SOFTWARE -from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing -from seedemu.layers._bgp_metadata import set_bgp_backend -from seedemu.services import BgpLookingGlassService, ExaBgpService - - -def _file_content(node, path: str) -> str: - for file in node.getFiles(): - file_path, content = file.get() - if file_path == path: - return content - return "" - - -def _attach_exabgp_peer(server, router_name: str, *, router_asn: int): - for method_name in ("addPeer", "addRouterPeer", "attachToRouter"): - if not hasattr(server, method_name): - continue - method = getattr(server, method_name) - try: - return method(router_name, router_asn=router_asn) - except TypeError: - return method(router_name, router_asn) - raise AssertionError("ExaBGP server does not expose a router peer attachment API") - - -def _compiled_output_text(output_dir: Path) -> str: - chunks = [] - for path in output_dir.rglob("*"): - if path.is_file(): - chunks.append(path.read_text(encoding="utf-8", errors="replace")) - return "\n".join(chunks) - - -def test_default_node_software_uses_installable_netcat_package(): - assert "netcat-openbsd" in DEFAULT_SOFTWARE - assert "netcat" not in DEFAULT_SOFTWARE - - -def test_legacy_frr_bgp_layer_is_removed(): - with pytest.raises(ModuleNotFoundError): - importlib.import_module("seedemu.layers.FrrBgp") - assert not hasattr(importlib.import_module("seedemu.layers"), "FrrBgp") - - -def test_router_rejects_legacy_exabgp_backends(): - as2 = Base().createAutonomousSystem(2) - with pytest.raises(AssertionError, match="unsupported routing backend"): - as2.createRouter("exabgp", routingBackend="exabgp") - with pytest.raises(AssertionError, match="unsupported routing backend"): - as2.createRouter("external", routingBackend="external") - - -def test_route_server_peering_renders_from_intent_in_routing_layer(): - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - - base.createInternetExchange(100) - as150 = base.createAutonomousSystem(150) - as150.createNetwork("net0") - as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") - - ebgp.addRsPeer(100, 150) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.render() - - reg = emu.getRegistry() - route_server = reg.get("ix", "rs", "ix100") - router = reg.get("150", "rnode", "router0") - - rs_conf = _file_content(route_server, "/etc/bird/bird.conf") - router_conf = _file_content(router, "/etc/bird/bird.conf") - - assert "protocol bgp p_as150" in rs_conf - assert "rs client" in rs_conf - assert "neighbor 10.100.0.150 as 150" in rs_conf - assert "protocol bgp p_rs100" in router_conf - assert "neighbor 10.100.0.100 as 100" in router_conf - assert "bgp_large_community.add(PEER_COMM)" in router_conf - - -def test_frr_route_server_backend_is_explicitly_rejected(): - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - - ix = base.createInternetExchange(100) - set_bgp_backend(ix.getRouteServerNode(), "frr") - as150 = base.createAutonomousSystem(150) - as150.createNetwork("net0") - as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") - - ebgp.addRsPeer(100, 150) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - - with pytest.raises(NotImplementedError, match="FRR route-server nodes are not supported yet"): - emu.render() - - -def test_frr_backend_renders_frr_config_for_selected_router(): - emu = Emulator() - base = Base() - routing = Routing() - ospf = Ospf() - ibgp = Ibgp() - ebgp = Ebgp() - - base.createInternetExchange(100) - base.createInternetExchange(101) - - as2 = base.createAutonomousSystem(2) - as2.createNetwork("net0") - as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") - as2.createRouter("r2", routingBackend="frr").joinNetwork("net0").joinNetwork("ix101") - - as151 = base.createAutonomousSystem(151) - as151.createNetwork("net0") - as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") - - as152 = base.createAutonomousSystem(152) - as152.createNetwork("net0") - as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") - - ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) - ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ospf) - emu.addLayer(ibgp) - emu.addLayer(ebgp) - emu.render() - - reg = emu.getRegistry() - r1 = reg.get("2", "rnode", "r1") - r2 = reg.get("2", "rnode", "r2") - - assert any(cmd == "bird -d" for cmd, _ in r1.getStartCommands()) - assert not any(cmd == "bird -d" for cmd, _ in r2.getStartCommands()) - assert _file_content(r2, "/etc/bird/bird.conf") == "" - - frr_conf = _file_content(r2, "/etc/frr/frr.conf") - assert "router bgp 2" in frr_conf - assert "neighbor" in frr_conf - assert "RM_CONNECTED_TO_BGP" in frr_conf - assert "LC_LOCAL_OR_CUSTOMER" in frr_conf - assert "router ospf" in frr_conf - assert "interface net0" in frr_conf - - -def test_exabgp_service_renders_dashboard_and_router_peer(): - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - exabgp = ExaBgpService() - - base.createInternetExchange(100) - - as2 = base.createAutonomousSystem(2) - as2.createNetwork("net0") - as2.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") - - as151 = base.createAutonomousSystem(151) - as151.createNetwork("net0") - as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") - as151.createHost("observer").joinNetwork("net0") - - ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) - - exabgp.install("observer_tool").attachToRouter("router0").setLocalAsn(65010).addAnnouncement("198.51.100.0/24") - emu.addBinding(Binding("observer_tool", filter=Filter(nodeName="observer", asn=151))) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.addLayer(exabgp) - emu.render() - - reg = emu.getRegistry() - observer = reg.get("151", "hnode", "observer") - router = reg.get("151", "rnode", "router0") - - exabgp_conf = _file_content(observer, "/etc/exabgp/exabgp.conf") - assert "198.51.100.0/24" in exabgp_conf - assert "process exabgp_json_sink" in exabgp_conf - assert "peer-as 151" in exabgp_conf - - dashboard = _file_content(observer, "/opt/exabgp/dashboard.py") - assert "/api/events" in dashboard - - bird_conf = _file_content(router, "/etc/bird/bird.conf") - assert "exabgp_65010" in bird_conf - assert "peer table t_bgp" in bird_conf - assert "bgp_large_community.add(CUSTOMER_COMM)" in bird_conf - assert "bgp_local_pref = 30" in bird_conf - - -def test_exabgp_service_renders_frr_peer_without_bird_config(): - emu = Emulator() - base = Base() - routing = Routing() - ospf = Ospf() - ibgp = Ibgp() - ebgp = Ebgp() - exabgp = ExaBgpService() - - base.createInternetExchange(100) - as2 = base.createAutonomousSystem(2) - as2.createNetwork("net0") - as2.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix100") - as2.createHost("observer").joinNetwork("net0") - - exabgp.install("observer_tool").attachToRouter("router0").setLocalAsn(65010).addAnnouncement("198.51.100.0/24") - emu.addBinding(Binding("observer_tool", filter=Filter(nodeName="observer", asn=2))) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ospf) - emu.addLayer(ibgp) - emu.addLayer(ebgp) - emu.addLayer(exabgp) - emu.render() - - reg = emu.getRegistry() - router = reg.get("2", "rnode", "router0") - observer = reg.get("2", "hnode", "observer") - - assert _file_content(router, "/etc/bird/bird.conf") == "" - frr_conf = _file_content(router, "/etc/frr/frr.conf") - assert "neighbor 10.2.0.71 remote-as 65010" in frr_conf - assert "description exabgp_65010" in frr_conf - assert "address-family ipv4 unicast" in frr_conf - - exabgp_conf = _file_content(observer, "/etc/exabgp/exabgp.conf") - assert "198.51.100.0/24" in exabgp_conf - assert "peer-as 2" in exabgp_conf - - -def test_exabgp_service_renders_multi_peer_ix_speaker_config(): - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - exabgp = ExaBgpService() - - base.createInternetExchange(100) - as2 = base.createAutonomousSystem(2) - as2.createRouter("r100").joinNetwork("ix100") - as3 = base.createAutonomousSystem(3) - as3.createRouter("r100").joinNetwork("ix100") - speaker_as = base.createAutonomousSystem(65030) - speaker_as.createHost("route_speaker").joinNetwork("ix100", address="10.100.0.230") - - speaker = exabgp.install("external_route_speaker") - speaker.setLocalAsn(65030).addAnnouncement("203.0.113.0/24").enableDashboard(5000) - _attach_exabgp_peer(speaker, "r100", router_asn=2) - _attach_exabgp_peer(speaker, "r100", router_asn=3) - emu.addBinding(Binding("external_route_speaker", filter=Filter(nodeName="route_speaker", asn=65030))) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.addLayer(exabgp) - emu.render() - - reg = emu.getRegistry() - route_speaker = reg.get("65030", "hnode", "route_speaker") - as2_router = reg.get("2", "rnode", "r100") - as3_router = reg.get("3", "rnode", "r100") - - exabgp_conf = _file_content(route_speaker, "/etc/exabgp/exabgp.conf") - assert exabgp_conf.count("neighbor ") == 2 - assert "peer-as 2" in exabgp_conf - assert "peer-as 3" in exabgp_conf - assert "203.0.113.0/24" in exabgp_conf - assert "process exabgp_json_sink" in exabgp_conf - assert _file_content(route_speaker, "/etc/bird/bird.conf") == "" - assert not any(cmd == "bird -d" for cmd, _ in route_speaker.getStartCommands()) - - as2_conf = _file_content(as2_router, "/etc/bird/bird.conf") - as3_conf = _file_content(as3_router, "/etc/bird/bird.conf") - assert "exabgp_65030" in as2_conf - assert "exabgp_65030" in as3_conf - assert "neighbor 10.100.0.230 as 65030" in as2_conf - assert "neighbor 10.100.0.230 as 65030" in as3_conf - assert "bgp_large_community.add(CUSTOMER_COMM)" in as2_conf - assert "bgp_large_community.add(CUSTOMER_COMM)" in as3_conf - assert "bgp_local_pref = 30" in as2_conf - assert "bgp_local_pref = 30" in as3_conf - - -def test_exabgp_service_renders_ix_speaker_peer(): - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - exabgp = ExaBgpService() - - base.createInternetExchange(100) - as2 = base.createAutonomousSystem(2) - as2.createRouter("r100").joinNetwork("ix100") - speaker_as = base.createAutonomousSystem(65030) - speaker_as.createHost("external_speaker").joinNetwork("ix100", address="10.100.0.230") - - speaker = exabgp.install("external_route_speaker") - speaker.setLocalAsn(65030).addAnnouncement("203.0.113.0/24") - _attach_exabgp_peer(speaker, "r100", router_asn=2) - emu.addBinding(Binding("external_route_speaker", filter=Filter(nodeName="external_speaker", asn=65030))) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.addLayer(exabgp) - emu.render() - - reg = emu.getRegistry() - external_speaker = reg.get("65030", "hnode", "external_speaker") - as2_router = reg.get("2", "rnode", "r100") - - exabgp_conf = _file_content(external_speaker, "/etc/exabgp/exabgp.conf") - assert "local-as 65030" in exabgp_conf - assert "peer-as 2" in exabgp_conf - assert "203.0.113.0/24" in exabgp_conf - assert _file_content(external_speaker, "/etc/bird/bird.conf") == "" - assert not any(cmd == "bird -d" for cmd, _ in external_speaker.getStartCommands()) - - bird_conf = _file_content(as2_router, "/etc/bird/bird.conf") - assert "exabgp_65030" in bird_conf - assert "neighbor 10.100.0.230 as 65030" in bird_conf - - -def test_frr_bgp_respects_ospf_stub_intent(): - emu = Emulator() - base = Base() - routing = Routing() - ospf = Ospf() - - as2 = base.createAutonomousSystem(2) - as2.createNetwork("net0") - as2.createNetwork("net1") - as2.createRouter("r1", routingBackend="frr").joinNetwork("net0").joinNetwork("net1") - - ospf.markAsStub(2, "net1") - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ospf) - emu.render() - - reg = emu.getRegistry() - r1 = reg.get("2", "rnode", "r1") - frr_conf = _file_content(r1, "/etc/frr/frr.conf") - assert "interface net0" in frr_conf - assert "interface net1" in frr_conf - assert "ip ospf passive" in frr_conf - - -def test_bgp_looking_glass_supports_frr_router_route_state(): - emu = Emulator() - base = Base() - routing = Routing() - ospf = Ospf() - ibgp = Ibgp() - looking_glass = BgpLookingGlassService() - - as2 = base.createAutonomousSystem(2) - as2.createNetwork("net0") - as2.createRouter("router0", routingBackend="frr").joinNetwork("net0") - as2.createHost("lg").joinNetwork("net0") - - looking_glass.install("bgp_lg").attach("router0") - emu.addBinding(Binding("bgp_lg", filter=Filter(nodeName="lg", asn=2))) - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ospf) - emu.addLayer(ibgp) - emu.addLayer(looking_glass) - - emu.render() - - reg = emu.getRegistry() - lg = reg.get("2", "hnode", "lg") - router = reg.get("2", "rnode", "router0") - - proxy = _file_content(router, "/opt/seed-lg/proxy.py") - frontend = _file_content(lg, "/opt/seed-lg/frontend.py") - proxy_cmds = [cmd for cmd, _ in router.getStartCommands()] - - assert "show bgp summary" in proxy - assert "show ip ospf neighbor" in proxy - assert "/api/state" in frontend - assert any("SEED_LG_BACKEND=\"frr\"" in cmd for cmd in proxy_cmds) - assert any("waiting for frr" in cmd for cmd in proxy_cmds) - assert not any("waiting for bird" in cmd for cmd in proxy_cmds) - - -def test_new_bgp_examples_compile_outputs_exist(): - repo_root = Path(__file__).resolve().parents[2] - examples = [ - repo_root / "examples" / "basic" / "A12_bgp_mixed_backend" / "bgp_mixed_backend.py", - repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "exabgp_control_plane.py", - repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "bgp_event_looking_glass.py", - ] - for script in examples: - env = dict(**os.environ) - env["PYTHONPATH"] = str(repo_root) - result = subprocess.run( - [sys.executable, str(script), "amd"], - cwd=script.parent, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0, result.stderr - - for output_dir in [ - repo_root / "examples" / "basic" / "A12_bgp_mixed_backend" / "output", - repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "output", - repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output", - ]: - assert output_dir.exists() - a13_compose = (repo_root / "examples" / "basic" / "A13_exabgp_control_plane" / "output" / "docker-compose.yml").read_text(encoding="utf-8") - a14_compose = (repo_root / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output" / "docker-compose.yml").read_text(encoding="utf-8") - assert "5001:5000/tcp" in a13_compose - assert "5002:5000/tcp" in a14_compose - assert "5003:5000/tcp" in a14_compose - - -def test_b30_mini_internet_exabgp_ix_compile_assertions(): - repo_root = Path(__file__).resolve().parents[2] - script = repo_root / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "mini_internet_exabgp_ix.py" - output_dir = script.parent / "output" - - env = dict(**os.environ) - env["PYTHONPATH"] = str(repo_root) - env["SEED_B30_EXABGP_PORT"] = "5130" - result = subprocess.run( - [sys.executable, str(script), "amd"], - cwd=script.parent, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0, result.stderr - assert output_dir.exists() - - compose = (output_dir / "docker-compose.yml").read_text(encoding="utf-8") - output_text = compose + "\n" + _compiled_output_text(output_dir) - assert "5130:5000/tcp" in compose - assert "203.0.113.0/24" in output_text - assert "peer-as 2" in output_text - assert "peer-as 3" in output_text - assert "neighbor 10.100.0.180 as 180" in output_text - assert "bgp_large_community.add(CUSTOMER_COMM)" in output_text - assert "bgp_local_pref = 30" in output_text - assert "process exabgp_json_sink" in output_text - assert "dashboard.py" in output_text - - -def test_b30_mini_internet_exabgp_ix_emulator_compiles_from_builder(): - from examples.internet.B30_mini_internet_exabgp_ix import mini_internet_exabgp_ix - - emu = mini_internet_exabgp_ix.build_emulator() - emu.render() - output_dir = Path("/tmp/seedemu-b30-static-compile") - emu.compile(Docker(selfManagedNetwork=True, platform=Platform.AMD64), str(output_dir), override=True) - - output_text = _compiled_output_text(output_dir) - assert "203.0.113.0/24" in output_text - assert "peer-as 2" in output_text - assert "peer-as 3" in output_text diff --git a/tests/control_plane/test_bgp_runtime_matrix.py b/tests/control_plane/test_bgp_runtime_matrix.py deleted file mode 100644 index ec8def2a7..000000000 --- a/tests/control_plane/test_bgp_runtime_matrix.py +++ /dev/null @@ -1,413 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -import sys -import tempfile -import time -from pathlib import Path -import re - -import pytest - - -REPO_ROOT = Path(__file__).resolve().parents[2] -PYTHON = Path(os.environ.get("PYTHON", sys.executable)) - - -def _run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]: - return subprocess.run( - cmd, - cwd=str(cwd) if cwd is not None else str(REPO_ROOT), - env=env, - text=True, - capture_output=True, - timeout=timeout, - check=False, - ) - - -def _must_run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]: - result = _run(cmd, cwd=cwd, env=env, timeout=timeout) - assert result.returncode == 0, ( - f"command failed: {' '.join(cmd)}\n" - f"stdout:\n{result.stdout}\n" - f"stderr:\n{result.stderr}" - ) - return result - - -def _classic_docker_build_env() -> dict[str, str]: - env = dict(os.environ) - env["COMPOSE_BAKE"] = "false" - env["DOCKER_BUILDKIT"] = "0" - env["COMPOSE_PARALLEL_LIMIT"] = "1" - return env - - -def _docker_exec(container: str, shell_cmd: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]: - return _must_run(["docker", "exec", container, "sh", "-lc", shell_cmd], timeout=timeout) - - -def _docker_exec_maybe(container: str, shell_cmd: str, *, timeout: int = 120) -> str | None: - result = _run(["docker", "exec", container, "sh", "-lc", shell_cmd], timeout=timeout) - if result.returncode != 0: - return None - return result.stdout - - -def _docker_ps_names() -> list[str]: - result = _must_run(["docker", "ps", "--format", "{{.Names}}"]) - return [line.strip() for line in result.stdout.splitlines() if line.strip()] - - -def _wait_container_name(predicate, *, timeout_s: int = 90, error_message: str) -> str: - def _probe(): - for name in _docker_ps_names(): - if predicate(name): - return name - return None - - return _wait_until(_probe, timeout_s=timeout_s, interval_s=2, error_message=error_message) - - -def _wait_http_ok(url: str, *, timeout_s: int = 60) -> None: - deadline = time.time() + timeout_s - while time.time() < deadline: - result = _run(["curl", "--noproxy", "*", "-fsS", "-o", "/dev/null", "-w", "%{http_code}", url], timeout=5) - if result.returncode == 0 and result.stdout.strip() == "200": - return - time.sleep(1) - raise AssertionError(f"HTTP endpoint did not become ready: {url}") - - -def _wait_until(predicate, *, timeout_s: int = 90, interval_s: int = 2, error_message: str = "condition not met"): - deadline = time.time() + timeout_s - last_value = None - while time.time() < deadline: - last_value = predicate() - if last_value: - return last_value - time.sleep(interval_s) - raise AssertionError(f"{error_message}\nlast_value={last_value}") - - -def _wait_docker_output( - container: str, - shell_cmd: str, - *, - contains: list[str], - timeout_s: int = 90, - interval_s: int = 2, - error_message: str, -) -> str: - def _probe(): - text = _docker_exec_maybe(container, shell_cmd) - if text is None: - return None - return text if all(needle in text for needle in contains) else None - - return _wait_until(_probe, timeout_s=timeout_s, interval_s=interval_s, error_message=error_message) - - -def _wait_command_output( - cmd: list[str], - *, - contains: list[str], - timeout_s: int = 90, - interval_s: int = 2, - error_message: str, -) -> str: - def _probe(): - result = _run(cmd) - if result.returncode != 0: - return None - return result.stdout if all(needle in result.stdout for needle in contains) else None - - return _wait_until(_probe, timeout_s=timeout_s, interval_s=interval_s, error_message=error_message) - - -def _assert_runtime_clear() -> None: - result = _must_run(["docker", "ps", "--format", "{{.Names}}"]) - offenders = [ - line.strip() - for line in result.stdout.splitlines() - if line.strip().startswith("as") or line.strip() == "seedemu_internet_map" - ] - assert not offenders, f"runtime validation requires a clean emulator environment, found: {offenders}" - - -def _assert_bridge_nf_disabled() -> None: - result = _must_run( - ["sysctl", "net.bridge.bridge-nf-call-iptables", "net.bridge.bridge-nf-call-ip6tables", "net.bridge.bridge-nf-call-arptables"] - ) - values = {} - for line in result.stdout.splitlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - values[key.strip()] = value.strip() - blocking = {key: value for key, value in values.items() if value != "0"} - assert not blocking, f"bridge netfilter must be disabled before runtime validation:\n{result.stdout}" - - -def _compose_down(compose_file: Path) -> None: - _run(["docker", "compose", "-f", str(compose_file), "down", "--remove-orphans"], timeout=300) - - -def _map_url_from_compose(compose_file: Path) -> str: - text = compose_file.read_text(encoding="utf-8") - match = re.search(r"-\s*(?:\$\{SEED_DEMO_MAP_PORT:-)?([0-9]+)(?:\})?:8080/tcp", text) - host_port = match.group(1) if match else "8080" - return f"http://127.0.0.1:{host_port}/pro/home" - - -@pytest.mark.integration -def test_runtime_a12_mixed_backend_fresh_build(): - _assert_runtime_clear() - _assert_bridge_nf_disabled() - - script = REPO_ROOT / "examples" / "basic" / "A12_bgp_mixed_backend" / "bgp_mixed_backend.py" - compose = REPO_ROOT / "examples" / "basic" / "A12_bgp_mixed_backend" / "output" / "docker-compose.yml" - - env = dict(os.environ) - env["PYTHONPATH"] = str(REPO_ROOT) - env["SEED_DEMO_MAP_PORT"] = "18080" - - try: - _must_run([str(PYTHON), str(script), "amd"], env=env) - _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) - _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) - - ospf = _wait_docker_output( - "as2brd-r2-10.2.0.253", - 'vtysh -c "show ip ospf neighbor"', - contains=["Full"], - error_message="A12 OSPF neighbor did not become visible", - ) - bgp = _wait_docker_output( - "as2brd-r2-10.2.0.253", - 'vtysh -c "show ip bgp summary"', - contains=["10.101.0.152", " 1"], - error_message="A12 BGP summary did not become available", - ) - host_ping = _wait_command_output( - ["docker", "exec", "as151h-web-10.151.0.71", "sh", "-lc", "ping -c 2 -W 1 10.152.0.71"], - contains=["0% packet loss"], - error_message="A12 host-to-host reachability did not converge", - ) - router_ping = _wait_command_output( - ["docker", "exec", "as151brd-router0-10.151.0.254", "sh", "-lc", "ping -c 2 -W 1 10.152.0.254"], - contains=["0% packet loss"], - error_message="A12 router-to-router reachability did not converge", - ) - route = _wait_docker_output( - "as152brd-router0-10.152.0.254", - "birdc show route 10.152.0.0/24 all", - contains=["local_nets", "BGP.large_community: (152, 0, 0)"], - error_message="A12 local route export did not appear", - ) - _wait_http_ok(_map_url_from_compose(compose)) - finally: - _compose_down(compose) - - -@pytest.mark.integration -def test_runtime_a13_exabgp_fresh_build(): - _assert_runtime_clear() - _assert_bridge_nf_disabled() - - script = REPO_ROOT / "examples" / "basic" / "A13_exabgp_control_plane" / "exabgp_control_plane.py" - compose = REPO_ROOT / "examples" / "basic" / "A13_exabgp_control_plane" / "output" / "docker-compose.yml" - - env = dict(os.environ) - env["PYTHONPATH"] = str(REPO_ROOT) - env["SEED_DEMO_MAP_PORT"] = "18080" - env["SEED_A13_EXABGP_PORT"] = "5101" - - try: - _must_run([str(PYTHON), str(script), "amd"], env=env) - _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) - _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) - - _wait_http_ok(_map_url_from_compose(compose)) - _wait_http_ok("http://127.0.0.1:5101/") - procs = _docker_exec( - "as151h-ExaBGP_Control_Plane_Tool-10.151.0.71", - 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep', - ).stdout - assert "dashboard.py" in procs and "exabgp" in procs and "event_sink.py" in procs - route = _wait_docker_output( - "as151brd-router0-10.151.0.254", - "birdc show route 198.51.100.0/24 all", - contains=["198.51.100.0/24", "AS65010"], - error_message="A13 ExaBGP route did not appear on Bird peer", - ) - finally: - _compose_down(compose) - - -@pytest.mark.integration -def test_runtime_a14_observability_fresh_build(): - _assert_runtime_clear() - _assert_bridge_nf_disabled() - - script = REPO_ROOT / "examples" / "basic" / "A14_bgp_event_looking_glass" / "bgp_event_looking_glass.py" - compose = REPO_ROOT / "examples" / "basic" / "A14_bgp_event_looking_glass" / "output" / "docker-compose.yml" - - env = dict(os.environ) - env["PYTHONPATH"] = str(REPO_ROOT) - env["SEED_DEMO_MAP_PORT"] = "18080" - env["SEED_A14_LG_PORT"] = "5002" - env["SEED_A14_EVENT_PORT"] = "5003" - - try: - _must_run([str(PYTHON), str(script), "amd"], env=env) - _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) - _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) - - _wait_http_ok(_map_url_from_compose(compose)) - _wait_http_ok("http://127.0.0.1:5002/summary/router0") - _wait_http_ok("http://127.0.0.1:5003/") - lg = _docker_exec("as2h-looking-glass-10.2.0.71", 'ps -ef | egrep "frontend|proxy" | grep -v grep').stdout - assert "frontend" in lg - exa = _docker_exec("as151h-ExaBGP_Control_Plane_Tool-10.151.0.71", 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep').stdout - assert "dashboard.py" in exa and "exabgp" in exa and "event_sink.py" in exa - lg_ping = _docker_exec("as2h-looking-glass-10.2.0.71", "ping -c 2 -W 1 10.2.0.254").stdout - assert "0% packet loss" in lg_ping - finally: - _compose_down(compose) - - -@pytest.mark.integration -def test_runtime_exabgp_to_frr_peer(): - _assert_runtime_clear() - _assert_bridge_nf_disabled() - - out_dir = Path(tempfile.mkdtemp(prefix="frr-exabgp-runtime-", dir="/tmp")) - compose = out_dir / "docker-compose.yml" - - source = """ -from seedemu.core import Binding, Emulator, Filter -from seedemu.layers import Base, Ebgp, Routing -from seedemu.services import ExaBgpService -from seedemu.compiler import Docker, Platform - -emu = Emulator() -base = Base() -routing = Routing() -ebgp = Ebgp() -exabgp = ExaBgpService() - -base.createInternetExchange(100) -as151 = base.createAutonomousSystem(151) -as151.createNetwork('net0') -as151.createRouter('router0', routingBackend='frr').joinNetwork('net0').joinNetwork('ix100') -as151.createHost('observer').joinNetwork('net0').addPortForwarding(5105, 5000) - -exabgp.install('observer_tool').attachToRouter('router0').setLocalAsn(65010).addAnnouncement('198.51.100.0/24').enableDashboard(5000) -emu.addBinding(Binding('observer_tool', filter=Filter(nodeName='observer', asn=151))) - -emu.addLayer(base) -emu.addLayer(routing) -emu.addLayer(ebgp) -emu.addLayer(exabgp) -emu.render() -emu.compile(Docker(platform=Platform.AMD64, internetMapEnabled=False), OUTPUT_DIR, override=True) -""" - script = out_dir / "generate.py" - script.write_text(source.replace("OUTPUT_DIR", repr(str(out_dir))), encoding="utf-8") - - env = dict(os.environ) - env["PYTHONPATH"] = str(REPO_ROOT) - - try: - _must_run([str(PYTHON), str(script)], env=env) - _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) - _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) - - _wait_http_ok("http://127.0.0.1:5105/") - summary = _wait_docker_output( - "as151brd-router0-10.151.0.254", - 'vtysh -c "show ip bgp summary"', - contains=["65010", " 1"], - error_message="FRR peer summary did not converge", - ) - route = _wait_docker_output( - "as151brd-router0-10.151.0.254", - 'vtysh -c "show ip bgp 198.51.100.0/24"', - contains=["198.51.100.0/24", "valid"], - error_message="FRR peer route did not appear", - ) - finally: - _compose_down(compose) - - -@pytest.mark.integration -def test_runtime_b30_mini_internet_exabgp_ix_fresh_build(): - _assert_runtime_clear() - _assert_bridge_nf_disabled() - - script = REPO_ROOT / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "mini_internet_exabgp_ix.py" - compose = REPO_ROOT / "examples" / "internet" / "B30_mini_internet_exabgp_ix" / "output" / "docker-compose.yml" - - env = dict(os.environ) - env["PYTHONPATH"] = str(REPO_ROOT) - env["SEED_DEMO_MAP_PORT"] = "18080" - env["SEED_B30_EXABGP_PORT"] = "5130" - - try: - _must_run([str(PYTHON), str(script), "amd"], env=env, timeout=900) - _must_run(["docker", "compose", "-f", str(compose), "build"], env=_classic_docker_build_env(), timeout=1800) - _must_run(["docker", "compose", "-f", str(compose), "up", "-d", "--remove-orphans"], timeout=600) - - _wait_http_ok(_map_url_from_compose(compose), timeout_s=120) - _wait_http_ok("http://127.0.0.1:5130/", timeout_s=120) - - exabgp_container = _wait_container_name( - lambda name: "ExaBGP_Control_Plane_Tool" in name or "external" in name.lower() and "speaker" in name.lower(), - error_message="B30 ExaBGP external speaker container did not start", - ) - as2_peer = _wait_container_name( - lambda name: name.startswith("as2brd-r100-"), - error_message="B30 AS2 r100 peer container did not start", - ) - as3_peer = _wait_container_name( - lambda name: name.startswith("as3brd-r100-"), - error_message="B30 AS3 r100 peer container did not start", - ) - - procs = _docker_exec( - exabgp_container, - 'ps -ef | egrep "exabgp|dashboard|event_sink" | grep -v grep', - ).stdout - assert "dashboard.py" in procs and "exabgp" in procs and "event_sink.py" in procs - - version = _docker_exec(exabgp_container, "exabgp --version 2>&1").stdout - assert "ExaBGP" in version or "exabgp" in version.lower() - - config = _docker_exec(exabgp_container, "cat /etc/exabgp/exabgp.conf").stdout - assert "peer-as 2" in config - assert "peer-as 3" in config - assert "203.0.113.0/24" in config - assert config.count("neighbor ") >= 2 - - _wait_docker_output( - as2_peer, - "birdc show route 203.0.113.0/24 all", - contains=["203.0.113.0/24"], - timeout_s=180, - error_message="B30 AS2 peer did not learn the ExaBGP IX tool route", - ) - _wait_docker_output( - as3_peer, - "birdc show route 203.0.113.0/24 all", - contains=["203.0.113.0/24"], - timeout_s=180, - error_message="B30 AS3 peer did not learn the ExaBGP IX tool route", - ) - - event_log = _docker_exec(exabgp_container, "test -r /var/log/exabgp/events.jsonl && wc -l /var/log/exabgp/events.jsonl").stdout - assert "/var/log/exabgp/events.jsonl" in event_log - finally: - _compose_down(compose) From 3963ef6b6236f62dd43f715883365d707de5da02 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Wed, 17 Jun 2026 11:48:10 +0800 Subject: [PATCH 4/6] Prefer upstream development conflict resolutions --- .github/workflows/k8s-check.yaml | 136 ++ .github/workflows/on-pull-request.yaml | 519 ++++++-- seedemu/compiler/kubernetes.py | 1616 +++++++----------------- seedemu/core/AutonomousSystem.py | 11 - seedemu/layers/Ebgp.py | 3 +- 5 files changed, 1004 insertions(+), 1281 deletions(-) create mode 100644 .github/workflows/k8s-check.yaml diff --git a/.github/workflows/k8s-check.yaml b/.github/workflows/k8s-check.yaml new file mode 100644 index 000000000..d2954887d --- /dev/null +++ b/.github/workflows/k8s-check.yaml @@ -0,0 +1,136 @@ +name: Check Native Kubernetes + +on: + pull_request: + branches: + - master + - development + - k8s-new + paths: + - ".github/workflows/k8s-check.yaml" + - "MANIFEST.in" + - "development.env" + - "requirements.txt" + - "setup.py" + - "docker_images/multiarch/**" + - "seedemu/compiler/Docker.py" + - "seedemu/compiler/kubernetes.py" + - "seedemu/compiler/__init__.py" + - "seedemu/k8sTools/**" + - "examples/internet/b61_k8s_compile/**" + - "tests/k8s/**" + workflow_dispatch: + inputs: + build_images: + description: "Build and push native K8s workload images to a local registry" + type: boolean + required: false + default: true + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.10" + K8S_NAMESPACE: seedemu-k8s-b61 + K8S_IMAGE_REGISTRY_PREFIX: seedemu + K8S_OUTPUT_DIR: /tmp/seedemu-k8s-output + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + k8s-static: + name: K8s Static + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out the source repository + uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install static-check dependencies + run: python -m pip install PyYAML + - name: Validate K8s source files + run: python3 tests/k8s/validateK8sStatic.py + + k8s-compile: + name: K8s Compile + runs-on: ubuntu-latest + needs: k8s-static + timeout-minutes: 20 + steps: + - name: Check out the source repository + uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "requirements.txt" + - name: Install compile dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - name: Compile native K8s example + run: | + source development.env + python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ + --output-dir "${K8S_OUTPUT_DIR}" + - name: Validate native K8s output + run: | + python3 tests/k8s/validateNativeK8sOutput.py \ + --output-dir "${K8S_OUTPUT_DIR}" \ + --expected-namespace "${K8S_NAMESPACE}" \ + --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" + - name: Archive native K8s compile output + uses: actions/upload-artifact@v6 + with: + name: native-k8s-output + path: ${{ env.K8S_OUTPUT_DIR }} + if-no-files-found: error + + k8s-build-images: + name: K8s Build Images + runs-on: ubuntu-latest + needs: k8s-compile + timeout-minutes: 45 + if: ${{ github.event_name == 'pull_request' || github.event.inputs.build_images == 'true' }} + steps: + - name: Check out the source repository + uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "requirements.txt" + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - name: Check Docker buildx + run: | + docker version + docker buildx version + - name: Compile native K8s example for image build + run: | + rm -rf "${K8S_OUTPUT_DIR}" + source development.env + python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ + --output-dir "${K8S_OUTPUT_DIR}" + - name: Validate native K8s build output + run: | + python3 tests/k8s/validateNativeK8sOutput.py \ + --output-dir "${K8S_OUTPUT_DIR}" \ + --expected-namespace "${K8S_NAMESPACE}" \ + --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" + - name: Build and push images to local registry + run: | + bash tests/k8s/buildNativeK8sImages.sh \ + --output-dir "${K8S_OUTPUT_DIR}" \ + --registry-prefix 127.0.0.1:5000 \ + --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" diff --git a/.github/workflows/on-pull-request.yaml b/.github/workflows/on-pull-request.yaml index ac6bc177e..8cac7fed8 100644 --- a/.github/workflows/on-pull-request.yaml +++ b/.github/workflows/on-pull-request.yaml @@ -5,147 +5,456 @@ on: branches: - master - development - - development2 - development-ipv6 workflow_dispatch: inputs: - run-example-build: - description: "Build selected representative examples" + selected-example: + description: "Run one selected example code, or all" + type: choice + required: true + default: all + options: + - all + - A00 + - A01 + - A02a + - A03a + - A03b + - A05 + - A06 + - A63 + - A62 + - B00 + - B02 + - B31 + - D01 + - D10 + - D20 + - D21 + - D22 + - D23 + run-basic-examples: + description: "Run selected basic examples" type: boolean required: false - default: false - run-runtime-integration: - description: "Run Docker runtime integration probes" + default: true + run-internet-examples: + description: "Run selected internet examples" type: boolean required: false - default: false + default: true + run-blockchain-examples: + description: "Run selected blockchain examples" + type: boolean + required: false + default: true + run-full-lifecycle: + description: "Run Docker build/up/probe/test/down lifecycle" + type: boolean + required: false + default: true -permissions: - contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: - PIP_DISABLE_PIP_VERSION_CHECK: "1" + SELECTED_EXAMPLE: ${{ github.event.inputs['selected-example'] || 'all' }} + RUN_BASIC_EXAMPLES: ${{ github.event.inputs['run-basic-examples'] || 'true' }} + RUN_INTERNET_EXAMPLES: ${{ github.event.inputs['run-internet-examples'] || 'true' }} + RUN_BLOCKCHAIN_EXAMPLES: ${{ github.event.inputs['run-blockchain-examples'] || 'true' }} + RUN_FULL_LIFECYCLE: ${{ github.event.inputs['run-full-lifecycle'] || 'true' }} jobs: - static: - name: Static + static-checks: + name: Static Checks runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Check out the source repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "**/*requirements.txt" - - name: Install Python dependencies - run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Run static checks - run: python tests/ci/run_ci.py static --artifact-dir ci-artifacts/static - - name: Upload static artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: static-ci-artifacts - path: ci-artifacts/static + uses: actions/checkout@v5 - unit: - name: Unit - needs: static - runs-on: ubuntu-latest - steps: - - name: Check out the source repository - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: "pip" + cache: pip cache-dependency-path: "**/*requirements.txt" - - name: Install Python dependencies - run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Run unit checks - run: python tests/ci/run_ci.py unit --artifact-dir ci-artifacts/unit - - name: Upload unit artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: unit-ci-artifacts - path: ci-artifacts/unit - example-compile: - name: Example Compile - needs: static - runs-on: ubuntu-latest - steps: - - name: Check out the source repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "**/*requirements.txt" - - name: Install Python dependencies - run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Compile representative examples - run: python tests/ci/run_ci.py example-compile --artifact-dir ci-artifacts/example-compile - - name: Upload example compile artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: example-compile-ci-artifacts - path: ci-artifacts/example-compile + - name: Install dependencies + run: | + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - example-build: - name: Example Build - needs: example-compile - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs['run-example-build'] == 'true' }} + - name: Compile testing framework and selected example tests + run: | + python -m py_compile \ + seedemu/testing/base.py \ + seedemu/testing/internet.py \ + seedemu/testing/blockchain.py \ + seedemu/testing/satellite.py \ + seedemu/testing/registry.py \ + seedemu/testing/runtime.py \ + seedemu/testing/cli.py \ + examples/basic/A00_simple_as/simple_as.py \ + examples/basic/A00_simple_as/test_runtime.py \ + examples/basic/A01_transit_as/transit_as.py \ + examples/basic/A01_transit_as/test_runtime.py \ + examples/basic/A02a_transit_as_mpls/transit_as_mpls.py \ + examples/basic/A02a_transit_as_mpls/test_runtime.py \ + examples/basic/A03a_out_to_real_world/out_to_real_world.py \ + examples/basic/A03a_out_to_real_world/test_runtime.py \ + examples/basic/A03b_from_real_world/from_real_world.py \ + examples/basic/A03b_from_real_world/test_runtime.py \ + examples/basic/A05_components/components.py \ + examples/basic/A05_components/test_runtime.py \ + examples/basic/A06_merge_emulation/merge_emulation.py \ + examples/basic/A06_merge_emulation/test_runtime.py \ + examples/basic/A63_control_plane_regression/control_plane_regression.py \ + examples/basic/A63_control_plane_regression/test_runtime.py \ + examples/basic/A62_route_reflector/route_reflector.py \ + examples/basic/A62_route_reflector/test_runtime.py \ + examples/internet/B00_mini_internet/mini_internet.py \ + examples/internet/B00_mini_internet/test_runtime.py \ + examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py \ + examples/internet/B02_mini_internet_with_dns/test_runtime.py \ + examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py \ + examples/internet/B31_mini_internet_mpls/test_runtime.py \ + examples/blockchain/D01_ethereum_pos/ethereum_pos.py \ + examples/blockchain/D01_ethereum_pos/test_runtime.py \ + examples/blockchain/D10_eth_explorer/ethereum_pos.py \ + examples/blockchain/D10_eth_explorer/test_runtime.py \ + examples/blockchain/D20_faucet/faucet.py \ + examples/blockchain/D20_faucet/test_runtime.py \ + examples/blockchain/D21_deploy_contract/deploy_contract.py \ + examples/blockchain/D21_deploy_contract/test_runtime.py \ + examples/blockchain/D22_oracle/simple_oracle.py \ + examples/blockchain/D22_oracle/test_runtime.py \ + examples/blockchain/D23_validator/validator.py \ + examples/blockchain/D23_validator/test_runtime.py \ + examples/blockchain/D23_validator/test_vc_at_running.py + + - name: Validate selected example manifests + run: | + python seedemu/testing/cli.py clean examples/basic/A00_simple_as/example.yaml + python seedemu/testing/cli.py clean examples/basic/A01_transit_as/example.yaml + python seedemu/testing/cli.py clean examples/basic/A02a_transit_as_mpls/example.yaml + python seedemu/testing/cli.py clean examples/basic/A03a_out_to_real_world/example.yaml + python seedemu/testing/cli.py clean examples/basic/A03b_from_real_world/example.yaml + python seedemu/testing/cli.py clean examples/basic/A05_components/example.yaml + python seedemu/testing/cli.py clean examples/basic/A06_merge_emulation/example.yaml + python seedemu/testing/cli.py clean examples/basic/A63_control_plane_regression/example.yaml + python seedemu/testing/cli.py clean examples/basic/A62_route_reflector/example.yaml + python seedemu/testing/cli.py clean examples/internet/B00_mini_internet/example.yaml + python seedemu/testing/cli.py clean examples/internet/B02_mini_internet_with_dns/example.yaml + python seedemu/testing/cli.py clean examples/internet/B31_mini_internet_mpls/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D01_ethereum_pos/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D10_eth_explorer/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D20_faucet/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D21_deploy_contract/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D22_oracle/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D23_validator/example.yaml + + compile-selected-examples: + name: Compile ${{ matrix.example.code }} + needs: static-checks runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + example: + - code: A00 + category: basic + requires_mpls: false + manifest: examples/basic/A00_simple_as/example.yaml + - code: A01 + category: basic + requires_mpls: false + manifest: examples/basic/A01_transit_as/example.yaml + - code: A02a + category: basic + requires_mpls: true + manifest: examples/basic/A02a_transit_as_mpls/example.yaml + - code: A03a + category: basic + requires_mpls: false + manifest: examples/basic/A03a_out_to_real_world/example.yaml + - code: A03b + category: basic + requires_mpls: false + manifest: examples/basic/A03b_from_real_world/example.yaml + - code: A05 + category: basic + requires_mpls: false + manifest: examples/basic/A05_components/example.yaml + - code: A06 + category: basic + requires_mpls: false + manifest: examples/basic/A06_merge_emulation/example.yaml + - code: A63 + category: basic + requires_mpls: false + manifest: examples/basic/A63_control_plane_regression/example.yaml + - code: A62 + category: basic + requires_mpls: false + manifest: examples/basic/A62_route_reflector/example.yaml + - code: B00 + category: internet + requires_mpls: false + manifest: examples/internet/B00_mini_internet/example.yaml + - code: B02 + category: internet + requires_mpls: false + manifest: examples/internet/B02_mini_internet_with_dns/example.yaml + - code: B31 + category: internet + requires_mpls: true + manifest: examples/internet/B31_mini_internet_mpls/example.yaml + - code: D01 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D01_ethereum_pos/example.yaml + - code: D10 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D10_eth_explorer/example.yaml + - code: D20 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D20_faucet/example.yaml + - code: D21 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D21_deploy_contract/example.yaml + - code: D22 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D22_oracle/example.yaml + - code: D23 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D23_validator/example.yaml steps: - name: Check out the source repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: "pip" + cache: pip cache-dependency-path: "**/*requirements.txt" - - name: Install Python dependencies - run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Build selected examples - run: python tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build - - name: Upload example build artifacts - if: always() - uses: actions/upload-artifact@v4 + + - name: Install dependencies + run: | + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + + - name: Compile selected example + if: >- + ${{ + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + source development.env + python seedemu/testing/cli.py compile \ + ${{ matrix.example.manifest }} \ + --artifact-dir ci-artifacts/${{ matrix.example.code }}-compile + + - name: Upload compile artifacts + if: >- + ${{ + always() && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + uses: actions/upload-artifact@v6 with: - name: example-build-ci-artifacts - path: ci-artifacts/example-build + name: example-${{ matrix.example.code }}-compile-artifacts + path: ci-artifacts/${{ matrix.example.code }}-compile - runtime-integration: - name: Runtime Integration - needs: example-compile - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs['run-runtime-integration'] == 'true' }} + run-selected-examples: + name: Run ${{ matrix.example.code }} + needs: compile-selected-examples runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + example: + - code: A00 + category: basic + requires_mpls: false + manifest: examples/basic/A00_simple_as/example.yaml + - code: A01 + category: basic + requires_mpls: false + manifest: examples/basic/A01_transit_as/example.yaml + - code: A02a + category: basic + requires_mpls: true + manifest: examples/basic/A02a_transit_as_mpls/example.yaml + - code: A03a + category: basic + requires_mpls: false + manifest: examples/basic/A03a_out_to_real_world/example.yaml + - code: A03b + category: basic + requires_mpls: false + manifest: examples/basic/A03b_from_real_world/example.yaml + - code: A05 + category: basic + requires_mpls: false + manifest: examples/basic/A05_components/example.yaml + - code: A06 + category: basic + requires_mpls: false + manifest: examples/basic/A06_merge_emulation/example.yaml + - code: A63 + category: basic + requires_mpls: false + manifest: examples/basic/A63_control_plane_regression/example.yaml + - code: A62 + category: basic + requires_mpls: false + manifest: examples/basic/A62_route_reflector/example.yaml + - code: B00 + category: internet + requires_mpls: false + manifest: examples/internet/B00_mini_internet/example.yaml + - code: B02 + category: internet + requires_mpls: false + manifest: examples/internet/B02_mini_internet_with_dns/example.yaml + - code: B31 + category: internet + requires_mpls: true + manifest: examples/internet/B31_mini_internet_mpls/example.yaml + - code: D01 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D01_ethereum_pos/example.yaml + - code: D10 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D10_eth_explorer/example.yaml + - code: D20 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D20_faucet/example.yaml + - code: D21 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D21_deploy_contract/example.yaml + - code: D22 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D22_oracle/example.yaml + - code: D23 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D23_validator/example.yaml steps: - name: Check out the source repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: "pip" + cache: pip cache-dependency-path: "**/*requirements.txt" - - name: Install Python dependencies - run: python -m pip install "setuptools<81" -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Run runtime integration probes - run: python tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration - - name: Upload runtime integration artifacts - if: always() - uses: actions/upload-artifact@v4 + + - name: Install dependencies + run: | + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + + - name: Load MPLS kernel modules + id: mpls + if: >- + ${{ + matrix.example.requires_mpls && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + set +e + sudo modprobe mpls_router + sudo modprobe mpls_iptunnel + sudo modprobe mpls_gso + if test -d /proc/sys/net/mpls && lsmod | grep -q '^mpls_router'; then + echo "available=true" >> "$GITHUB_OUTPUT" + lsmod | grep '^mpls_' + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::warning::MPLS kernel modules are not available on this runner. Skipping the MPLS runtime lifecycle for ${{ matrix.example.code }}. Use a self-hosted runner with mpls_router, mpls_iptunnel, and mpls_gso for full MPLS runtime testing." + fi + + - name: Report skipped MPLS lifecycle + if: >- + ${{ + matrix.example.requires_mpls && + steps.mpls.outputs.available == 'false' && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + echo "Skipping ${{ matrix.example.code }} full lifecycle because this GitHub runner does not provide MPLS kernel modules." + + - name: Run selected example lifecycle + if: >- + ${{ + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + (!matrix.example.requires_mpls || steps.mpls.outputs.available == 'true') && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + source development.env + python seedemu/testing/cli.py all \ + ${{ matrix.example.manifest }} \ + --artifact-dir ci-artifacts/${{ matrix.example.code }} + + - name: Upload lifecycle artifacts + if: >- + ${{ + always() && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + (!matrix.example.requires_mpls || steps.mpls.outputs.available == 'true') && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + uses: actions/upload-artifact@v6 with: - name: runtime-integration-ci-artifacts - path: ci-artifacts/runtime-integration + name: example-${{ matrix.example.code }}-lifecycle-artifacts + path: ci-artifacts/${{ matrix.example.code }} diff --git a/seedemu/compiler/kubernetes.py b/seedemu/compiler/kubernetes.py index 3505197e9..d5ad8d962 100644 --- a/seedemu/compiler/kubernetes.py +++ b/seedemu/compiler/kubernetes.py @@ -1,1279 +1,569 @@ from __future__ import annotations -from seedemu.core.Emulator import Emulator -from seedemu.core import Node, Network, Compiler, Scope, OptionHandling, BaseVolume -from seedemu.core.enums import NodeRole, NetworkType -from .Docker import Docker, DockerCompilerFileTemplates -from .DockerImage import DockerImage -from typing import Dict, Generator, List, Set, Tuple, Optional, Any -from hashlib import md5 -from os import mkdir, chdir + import json import os +import ipaddress +import re import shutil -import shlex +from hashlib import md5, sha1 +from os import chdir, mkdir +from pathlib import Path +from typing import Any, Dict, List + import yaml +from seedemu.compiler.Docker import Docker +from seedemu.core import Network, Node + -class SchedulingStrategy: - """Scheduling strategy constants for Kubernetes node placement.""" - NONE = "none" # No scheduling constraints - AUTO = "auto" # Automatic soft grouping + soft spreading - BY_AS = "by_as" # Schedule pods by AS number (soft affinity) - BY_AS_HARD = "by_as_hard" # Same ASN must land on one explicit Kubernetes node - BY_ROLE = "by_role" # Schedule pods by node role (router, host, etc.) - CUSTOM = "custom" # Use custom labels provided by user +INTERNET_MAP_META_PREFIX = "org.seedsecuritylabs.seedemu.meta." +# Border routers are already represented through the router node path in SEED's +# registry. Compiling "brdnode" again creates duplicate Deployment names. +SEEDEMU_NODE_TYPES = ["rnode", "csnode", "hnode", "rs", "snode"] -class KubernetesCompiler(Docker): - """! - @brief The Kubernetes compiler class. +class NativeKubernetesCompiler(Docker): + """Minimal Kubernetes compiler for a k8s-native baseline. - Compiles the emulation to Kubernetes manifests with support for: - - Multi-node scheduling (nodeSelector, nodeAffinity) - - Resource management (requests/limits) - - Service generation (ClusterIP, NodePort) - - Configurable CNI types for cross-node networking + Design goals: + - no inventory dependency during compile + - no nodeSelector / affinity / topology spreading + - no node-aware preload planning + - generated orchestration artifacts are limited to one manifest file and + images.yaml; build/deploy orchestration lives in seedemu.k8sTools. """ - __registry_prefix: str __namespace: str - __use_multus: bool - __manifests: List[str] - __build_commands: List[str] - _current_node: Node - - # New fields for multi-node support - __scheduling_strategy: str - __node_labels: Dict[str, Dict[str, str]] - __default_resources: Dict[str, Dict[str, str]] __cni_type: str - __local_link_cni_type: Optional[str] __cni_master_interface: str - __generate_services: bool - __service_type: str __image_pull_policy: str + __image_registry_prefix: str + __manifests: List[Dict[str, Any]] + __image_entries: List[Dict[str, str]] def __init__( self, - registry_prefix: str = "localhost:5000", - namespace: str = "seedemu", - use_multus: bool = True, - internetMapEnabled: bool = True, - # New parameters for multi-node deployment - scheduling_strategy: str = SchedulingStrategy.NONE, - node_labels: Dict[str, Dict[str, str]] = None, - default_resources: Dict[str, Dict[str, str]] = None, - cni_type: str = "bridge", - local_link_cni_type: Optional[str] = None, - cni_master_interface: str = "eth0", - generate_services: bool = False, - service_type: str = "ClusterIP", + image_registry_prefix: str = "seedemu", + registry_prefix: str | None = None, + namespace: str = "seedemu-k8s-b61", + cni_type: str = "kube-ovn", + cni_master_interface: str = "ens2", image_pull_policy: str = "Always", - **kwargs - ): - """! - @brief Kubernetes compiler constructor. - - @param registry_prefix (optional) Registry prefix for docker images. Default "localhost:5000". - @param namespace (optional) Kubernetes namespace. Default "seedemu". - @param use_multus (optional) Use Multus CNI for multiple interfaces. Default True. - @param internetMapEnabled (optional) Enable Internet Map visualization service. Default True. - @param scheduling_strategy (optional) How to schedule pods across nodes. Options: - - "none": No scheduling constraints (default) - - "auto": Let scheduler auto-balance with soft AS/role affinity - - "by_as": Prefer pods with same AS on same node (no node pre-labeling needed) - - "by_role": Prefer same role on same node (no node pre-labeling needed) - - "custom": Use custom node_labels mapping - @param node_labels (optional) Custom node labels for scheduling. Dict mapping AS numbers or - node names to label dicts. Example: {"150": {"kubernetes.io/hostname": "node1"}} - @param default_resources (optional) Default resource requests/limits for all pods. - Example: {"requests": {"cpu": "100m", "memory": "128Mi"}, - "limits": {"cpu": "500m", "memory": "512Mi"}} - @param cni_type (optional) CNI plugin type for NetworkAttachmentDefinition. Options: - - "bridge": Local bridge (default, single-node only) - - "macvlan": macvlan for cross-node with L2 access - - "ipvlan": ipvlan for cross-node networking - - "host-local": Use host-local IPAM - @param local_link_cni_type (optional) Override CNI type for node-local internal links. - When left unset, `by_as_hard` + `macvlan/ipvlan` automatically falls back to - `bridge` for `Local` / `CrossConnect` networks so same-AS internal links stay - isolated on the selected Kubernetes node. - @param cni_master_interface (optional) Master interface for macvlan/ipvlan. Default "eth0". - @param generate_services (optional) Generate K8s Service resources for nodes. Default False. - @param service_type (optional) Service type when generate_services is True. - Options: "ClusterIP", "NodePort". Default "ClusterIP". - @param image_pull_policy (optional) K8s ImagePullPolicy. Default "Always". - """ - # Force selfManagedNetwork=True so that parent logic generates /replace_address.sh call in start.sh - kwargs['selfManagedNetwork'] = True + # use_multus: bool = True, + # create_namespace: bool = True, + **kwargs: Any, + ) -> None: + kwargs["selfManagedNetwork"] = True super().__init__(**kwargs) - self.__registry_prefix = registry_prefix - self.__namespace = namespace - self.__use_multus = use_multus - self.__internet_map_enabled = internetMapEnabled + if registry_prefix is not None: + image_registry_prefix = registry_prefix + self.__image_registry_prefix = image_registry_prefix.strip().strip("/") + self.__namespace = namespace.strip() or "seedemu" + self.__cni_type = cni_type.strip().lower() or "bridge" + self.__cni_master_interface = cni_master_interface.strip() or "eth0" + self.__image_pull_policy = image_pull_policy.strip() or "Always" self.__manifests = [] - self.__build_commands = [] - self._current_node = None - - # Multi-node deployment settings - strategy = (scheduling_strategy or SchedulingStrategy.NONE).strip().lower() - valid = { - SchedulingStrategy.NONE, - SchedulingStrategy.AUTO, - SchedulingStrategy.BY_AS, - SchedulingStrategy.BY_AS_HARD, - SchedulingStrategy.BY_ROLE, - SchedulingStrategy.CUSTOM, - } - self.__scheduling_strategy = strategy if strategy in valid else SchedulingStrategy.NONE - self.__node_labels = node_labels or {} - self.__default_resources = default_resources or {} - self.__cni_type = cni_type - self.__local_link_cni_type = local_link_cni_type.strip().lower() if isinstance(local_link_cni_type, str) and local_link_cni_type.strip() else None - self.__cni_master_interface = cni_master_interface - self.__generate_services = generate_services - self.__service_type = service_type - self.__image_pull_policy = image_pull_policy + self.__image_entries = [] def getName(self) -> str: - return "Kubernetes" - - def _resolveInternetMapImages(self) -> Tuple[str, str]: - """Resolve source and deployment image refs for the optional Internet Map service.""" - source_image = os.environ.get( - "SEED_INTERNET_MAP_SOURCE_IMAGE", - "handsonsecurity/seedemu-multiarch-map:buildx-latest", - ).strip() or "handsonsecurity/seedemu-multiarch-map:buildx-latest" - - default_target = source_image - if self.__registry_prefix: - default_target = f"{self.__registry_prefix}/seedemu-internet-map:buildx-latest" - - target_image = os.environ.get("SEED_INTERNET_MAP_IMAGE", default_target).strip() or default_target - return source_image, target_image + return "NativeKubernetes" + def getManifestName(self) -> str: + """Return the manifest filename generated by this compiler.""" + if self._usesKubeOvn(): + return "k8s.kube-ovn.yaml" + return "k8s.yaml" @staticmethod def _safeBridgeName(name: str) -> str: - """Generate a Linux bridge name that fits the 15-char IFNAMSIZ limit. - - Uses 'br-' prefix (3 chars) + md5 hash truncated to 12 chars = 15 chars total. - This ensures uniqueness while staying within the kernel limit. - """ - h = md5(name.encode()).hexdigest()[:12] - return f"br-{h}" + return f"br-{md5(name.encode()).hexdigest()[:12]}" + + def _usesKubeOvn(self) -> bool: + """Return true when the compiler should emit Kube-OVN resources.""" + return self.__cni_type in {"kube-ovn", "kube_ovn", "ovn"} - def _doCompile(self, emulator: Emulator): + def _doCompile(self, emulator) -> None: registry = emulator.getRegistry() self._groupSoftware(emulator) - # Implementation overview: - # (1) networks -> NetworkAttachmentDefinition (Multus), - # (2) nodes -> Deployment/VirtualMachine, - # (3) output artifacts (k8s.yaml, build_images.sh, .env). - # 1. Generate NetworkAttachmentDefinitions (if Multus) - if self.__use_multus: - for ((scope, type, name), obj) in registry.getAll().items(): - if type == 'net': - self.__manifests.append(self._compileNetK8s(obj)) - - # 2. Compile Nodes - for ((scope, type, name), obj) in registry.getAll().items(): - if type in ['rnode', 'csnode', 'hnode', 'rs', 'snode']: + self.__manifests.append( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": self.__namespace}, + } + ) + + for ((_, obj_type, _), obj) in registry.getAll().items(): + if obj_type == "net": + self.__manifests.append(self._compileNetK8s(obj)) + + for ((_, obj_type, _), obj) in registry.getAll().items(): + if obj_type in SEEDEMU_NODE_TYPES: self.__manifests.append(self._compileNodeK8s(obj)) - # 3. Generate Output Files + manifests = ( + self._renderKubeOvnManifests(self.__manifests) + if self._usesKubeOvn() + else self.__manifests + ) + with open(self.getManifestName(), "w", encoding="utf-8") as handle: + yaml.safe_dump_all(manifests, handle, sort_keys=False) - # manifests.yaml - with open('k8s.yaml', 'w') as f: - f.write("\n---\n".join(self.__manifests)) + self._stage_base_image_contexts() + with open("images.yaml", "w", encoding="utf-8") as handle: + yaml.safe_dump({"images": self.__image_entries}, handle, sort_keys=False) - # Stage local Dockerfile contexts for built-in base images (to avoid relying on Docker Hub mirrors - # that may not whitelist custom images like handsonsecurity/*). - used_images = sorted(getattr(self, "_used_images", set())) - try: - repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) - base_image_sources = { - # These match DockerImageConstant defaults. - "handsonsecurity/seedemu-multiarch-base:buildx-latest": os.path.join(repo_root, "docker_images", "multiarch", "seedemu-base"), - "handsonsecurity/seedemu-multiarch-router:buildx-latest": os.path.join(repo_root, "docker_images", "multiarch", "seedemu-router"), - } - staged_any = False - for image in used_images: - src = base_image_sources.get(image) - if not src or not os.path.isdir(src): - continue - digest = md5(image.encode("utf-8")).hexdigest() - dst_root = os.path.join("base_images", digest) - os.makedirs(os.path.dirname(dst_root), exist_ok=True) - if os.path.exists(dst_root): - shutil.rmtree(dst_root) - shutil.copytree(src, dst_root) - staged_any = True - if not staged_any and os.path.isdir("base_images"): - shutil.rmtree("base_images") - except Exception: - # Best-effort only: compilation should still succeed even if we can't stage base images. - pass - - # build_images.sh - with open('build_images.sh', 'w') as f: - f.write("#!/usr/bin/env bash\n") - f.write("set -euo pipefail\n") - # Allow callers to override BuildKit/parallelism without editing generated artifacts. - f.write('export DOCKER_BUILDKIT="${SEED_DOCKER_BUILDKIT:-0}"\n') - f.write('PARALLELISM="${SEED_BUILD_PARALLELISM:-1}"\n') - f.write('if ! [[ "${PARALLELISM}" =~ ^[0-9]+$ ]]; then PARALLELISM=1; fi\n') - f.write('export REGISTRY_PUSH_RETRIES="${SEED_REGISTRY_PUSH_RETRIES:-5}"\n') - f.write('if ! [[ "${REGISTRY_PUSH_RETRIES}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_RETRIES=5; fi\n') - f.write('export REGISTRY_PUSH_BACKOFF_SECONDS="${SEED_REGISTRY_PUSH_BACKOFF_SECONDS:-5}"\n') - f.write('if ! [[ "${REGISTRY_PUSH_BACKOFF_SECONDS}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_BACKOFF_SECONDS=5; fi\n') - f.write('export REGISTRY_PUSH_TIMEOUT_SECONDS="${SEED_REGISTRY_PUSH_TIMEOUT_SECONDS:-180}"\n') - f.write('if ! [[ "${REGISTRY_PUSH_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then REGISTRY_PUSH_TIMEOUT_SECONDS=180; fi\n') - f.write('export SEED_IMAGE_DISTRIBUTION_MODE="${SEED_IMAGE_DISTRIBUTION_MODE:-registry}"\n') - f.write('if [[ "${SEED_IMAGE_DISTRIBUTION_MODE}" != "registry" && "${SEED_IMAGE_DISTRIBUTION_MODE}" != "preload" ]]; then export SEED_IMAGE_DISTRIBUTION_MODE="registry"; fi\n') - f.write('export SEED_DOCKER_MAX_CONCURRENT_UPLOADS="${SEED_DOCKER_MAX_CONCURRENT_UPLOADS:-1}"\n') - f.write('if ! [[ "${SEED_DOCKER_MAX_CONCURRENT_UPLOADS}" =~ ^[0-9]+$ ]]; then export SEED_DOCKER_MAX_CONCURRENT_UPLOADS=1; fi\n') - # Export so the inline Python snippet (daemon.json edit) can see it. - f.write('export SEED_DOCKER_IO_MIRROR_ENDPOINT="${SEED_DOCKER_IO_MIRROR_ENDPOINT:-https://docker.m.daocloud.io}"\n') - f.write('MIRROR_HOST="${SEED_DOCKER_IO_MIRROR_ENDPOINT#http://}"\n') - f.write('MIRROR_HOST="${MIRROR_HOST#https://}"\n') - # Export so the inline Python snippet (daemon.json edit) can see it. - f.write(f'export REGISTRY_PREFIX="{self.__registry_prefix}"\n') - f.write('export REGISTRY_LOCAL_ENDPOINT="${SEED_REGISTRY_LOCAL_ENDPOINT:-}"\n') - f.write("\n") - f.write("docker_pull() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if command -v timeout >/dev/null 2>&1; then\n") - f.write(" timeout 180s docker pull \"$image\"\n") - f.write(" else\n") - f.write(" docker pull \"$image\"\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write("mirror_image_name() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if [[ -z \"${MIRROR_HOST}\" ]]; then\n") - f.write(" echo \"$image\"\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" if [[ \"$image\" == *\"/\"* ]]; then\n") - f.write(" echo \"${MIRROR_HOST}/${image}\"\n") - f.write(" else\n") - f.write(" echo \"${MIRROR_HOST}/library/${image}\"\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write("ensure_image_present() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if docker image inspect \"$image\" >/dev/null 2>&1; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" local mirror\n") - f.write(" mirror=\"$(mirror_image_name \"$image\")\"\n") - # In environments where Docker Hub is blocked, try the mirror first. - f.write(" if [[ \"$mirror\" != \"$image\" ]]; then\n") - f.write(" if docker_pull \"$mirror\" >/dev/null 2>&1; then\n") - f.write(" docker tag \"$mirror\" \"$image\" >/dev/null 2>&1 || true\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" fi\n") - f.write(" if docker_pull \"$image\" >/dev/null 2>&1; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" echo \"[build_images] ERROR: cannot pull base image: $image\" >&2\n") - f.write(" echo \"[build_images] Hint: set SEED_DOCKER_IO_MIRROR_ENDPOINT to a reachable mirror\" >&2\n") - f.write(" return 1\n") - f.write("}\n") - f.write("\n") - f.write("ensure_docker_daemon_config() {\n") - f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then return 0; fi\n") - f.write(" if [[ -z \"${REGISTRY_PREFIX}\" ]]; then return 0; fi\n") - f.write(" if [[ \"$(id -u)\" != \"0\" ]]; then return 0; fi\n") - f.write(" if ! command -v systemctl >/dev/null 2>&1; then return 0; fi\n") - f.write(" mkdir -p /etc/docker\n") - f.write(" local result\n") - f.write(" result=\"$(python3 - <<'PY'\n") - f.write("import json\n") - f.write("from pathlib import Path\n") - f.write("import os\n") - f.write("\n") - f.write("registry = os.environ.get('REGISTRY_PREFIX', '')\n") - f.write("mirror = os.environ.get('SEED_DOCKER_IO_MIRROR_ENDPOINT', '')\n") - f.write("max_uploads = os.environ.get('SEED_DOCKER_MAX_CONCURRENT_UPLOADS', '1')\n") - f.write("try:\n") - f.write(" max_uploads_value = max(1, int(max_uploads))\n") - f.write("except Exception:\n") - f.write(" max_uploads_value = 1\n") - f.write("p = Path('/etc/docker/daemon.json')\n") - f.write("data = {}\n") - f.write("if p.exists():\n") - f.write(" try:\n") - f.write(" data = json.loads(p.read_text(encoding='utf-8'))\n") - f.write(" except Exception:\n") - f.write(" data = {}\n") - f.write("\n") - f.write("before = json.dumps(data, sort_keys=True)\n") - f.write("insec = set(data.get('insecure-registries', []) or [])\n") - f.write("if registry and '/' not in registry:\n") - f.write(" insec.add(registry)\n") - f.write(" if ':' in registry:\n") - f.write(" port = registry.rsplit(':', 1)[1]\n") - f.write(" insec.add(f'127.0.0.1:{port}')\n") - f.write(" insec.add(f'localhost:{port}')\n") - f.write("data['insecure-registries'] = sorted(insec)\n") - f.write("\n") - f.write("mirrors = list(data.get('registry-mirrors', []) or [])\n") - f.write("if mirror and mirror not in mirrors:\n") - f.write(" mirrors.append(mirror)\n") - f.write("data['registry-mirrors'] = mirrors\n") - f.write("data['max-concurrent-uploads'] = max_uploads_value\n") - f.write("\n") - f.write("after = json.dumps(data, sort_keys=True)\n") - f.write("if after != before:\n") - f.write(" p.write_text(json.dumps(data, indent=2) + '\\n', encoding='utf-8')\n") - f.write(" print('changed')\n") - f.write("else:\n") - f.write(" print('unchanged')\n") - f.write("PY\n") - f.write(")\"\n") - f.write(" if [[ \"${result}\" == \"changed\" ]]; then\n") - f.write(" systemctl restart docker >/dev/null 2>&1 || true\n") - f.write(" sleep 2\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write('export REGISTRY_ENDPOINT="${REGISTRY_PREFIX%%/*}"\n') - f.write('if [[ -z "${REGISTRY_LOCAL_ENDPOINT}" ]] && [[ "${REGISTRY_ENDPOINT}" == *:* ]] && [[ "${REGISTRY_ENDPOINT}" != 127.0.0.1:* ]] && [[ "${REGISTRY_ENDPOINT}" != localhost:* ]]; then REGISTRY_LOCAL_ENDPOINT="127.0.0.1:${REGISTRY_ENDPOINT##*:}"; fi\n') - f.write('export REGISTRY_PROBE_ENDPOINT="${REGISTRY_LOCAL_ENDPOINT:-${REGISTRY_ENDPOINT}}"\n') - f.write("registry_probe() {\n") - f.write(" if [[ -z \"${REGISTRY_PROBE_ENDPOINT}\" ]]; then return 0; fi\n") - f.write(" if command -v curl >/dev/null 2>&1; then\n") - f.write(" curl -m 5 -fsS \"http://${REGISTRY_PROBE_ENDPOINT}/v2/\" >/dev/null\n") - f.write(" elif command -v wget >/dev/null 2>&1; then\n") - f.write(" wget -q -T 5 -O /dev/null \"http://${REGISTRY_PROBE_ENDPOINT}/v2/\"\n") - f.write(" else\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write("wait_for_registry() {\n") - f.write(" local retries=\"${1:-6}\"\n") - f.write(" local attempt=1\n") - f.write(" while [ \"${attempt}\" -le \"${retries}\" ]; do\n") - f.write(" if registry_probe; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" sleep \"${attempt}\"\n") - f.write(" attempt=$((attempt + 1))\n") - f.write(" done\n") - f.write(" return 1\n") - f.write("}\n") - f.write("\n") - f.write("local_push_image_ref() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if [[ -z \"${REGISTRY_LOCAL_ENDPOINT}\" ]] || [[ -z \"${REGISTRY_PREFIX}\" ]]; then\n") - f.write(" echo \"${image}\"\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" local suffix=\"${image#${REGISTRY_PREFIX}/}\"\n") - f.write(" if [[ \"${suffix}\" == \"${image}\" ]]; then\n") - f.write(" echo \"${image}\"\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" echo \"${REGISTRY_LOCAL_ENDPOINT}/${suffix}\"\n") - f.write("}\n") - f.write("\n") - f.write("docker_push_with_timeout() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if command -v timeout >/dev/null 2>&1; then\n") - f.write(" timeout \"${REGISTRY_PUSH_TIMEOUT_SECONDS}\" docker push \"${image}\"\n") - f.write(" else\n") - f.write(" docker push \"${image}\"\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write("retry_push() {\n") - f.write(" local image=\"$1\"\n") - f.write(" if [[ -z \"${REGISTRY_PREFIX}\" ]]; then return 0; fi\n") - f.write(" local attempt=1\n") - f.write(" while [ \"${attempt}\" -le \"${REGISTRY_PUSH_RETRIES}\" ]; do\n") - f.write(" if ! wait_for_registry 3; then\n") - f.write(" echo \"[build_images] registry probe failed before push attempt ${attempt}/${REGISTRY_PUSH_RETRIES}: ${REGISTRY_PROBE_ENDPOINT}\" >&2\n") - f.write(" fi\n") - f.write(" if docker_push_with_timeout \"${image}\"; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" if [ \"${attempt}\" -ge \"${REGISTRY_PUSH_RETRIES}\" ]; then\n") - f.write(" break\n") - f.write(" fi\n") - f.write(" local sleep_seconds=$((REGISTRY_PUSH_BACKOFF_SECONDS * attempt))\n") - f.write(" echo \"[build_images] retrying push ${image} in ${sleep_seconds}s (${attempt}/${REGISTRY_PUSH_RETRIES})\" >&2\n") - f.write(" sleep \"${sleep_seconds}\"\n") - f.write(" attempt=$((attempt + 1))\n") - f.write(" done\n") - f.write(" echo \"[build_images] ERROR: push failed after ${REGISTRY_PUSH_RETRIES} attempts: ${image}\" >&2\n") - f.write(" return 1\n") - f.write("}\n") - f.write("\n") - f.write("seedemu_build_and_push() {\n") - f.write(" local image=\"$1\"\n") - f.write(" local context_dir=\"$2\"\n") - f.write(" docker build -t \"${image}\" \"${context_dir}\"\n") - f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" if [[ -n \"${REGISTRY_PREFIX}\" ]]; then\n") - f.write(" local push_image\n") - f.write(" push_image=\"$(local_push_image_ref \"${image}\")\"\n") - f.write(" if [[ \"${push_image}\" != \"${image}\" ]]; then\n") - f.write(" docker tag \"${image}\" \"${push_image}\"\n") - f.write(" fi\n") - f.write(" retry_push \"${push_image}\"\n") - f.write(" fi\n") - f.write("}\n") - f.write("\n") - f.write("seedemu_copy_and_push_image() {\n") - f.write(" local source_image=\"$1\"\n") - f.write(" local target_image=\"$2\"\n") - f.write(" ensure_image_present \"${source_image}\"\n") - f.write(" docker tag \"${source_image}\" \"${target_image}\"\n") - f.write(" if [[ \"${SEED_IMAGE_DISTRIBUTION_MODE}\" == \"preload\" ]]; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" if [[ -n \"${REGISTRY_PREFIX}\" ]]; then\n") - f.write(" local push_image\n") - f.write(" push_image=\"$(local_push_image_ref \"${target_image}\")\"\n") - f.write(" if [[ \"${push_image}\" != \"${target_image}\" ]]; then\n") - f.write(" docker tag \"${target_image}\" \"${push_image}\"\n") - f.write(" fi\n") - f.write(" retry_push \"${push_image}\"\n") - f.write(" fi\n") - f.write("}\n") - f.write("export -f registry_probe wait_for_registry local_push_image_ref docker_push_with_timeout retry_push seedemu_build_and_push seedemu_copy_and_push_image\n") - f.write("\n") - f.write("prepare_dummy_image() {\n") - f.write(" local base_image=\"$1\"\n") - f.write(" local dummy_tag=\"$2\"\n") - f.write(" if docker image inspect \"$dummy_tag\" >/dev/null 2>&1; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" if ! docker image inspect \"$base_image\" >/dev/null 2>&1; then\n") - f.write(" local ctx=\"base_images/${dummy_tag}\"\n") - f.write(" if [ -f \"${ctx}/Dockerfile\" ]; then\n") - f.write(" echo \"[build_images] building base image: ${base_image} (from ${ctx})\" >&2\n") - f.write(" # Pre-pull common upstream bases via mirror+tag to avoid Docker Hub outages.\n") - f.write(" ensure_image_present \"ubuntu:20.04\"\n") - f.write(" docker build -t \"${base_image}\" \"${ctx}\"\n") - f.write(" else\n") - f.write(" ensure_image_present \"$base_image\"\n") - f.write(" fi\n") - f.write(" fi\n") - f.write(" mkdir -p dummies\n") - f.write(" local df=\"dummies/${dummy_tag}.Dockerfile\"\n") - f.write(" printf 'FROM %s\\n' \"$base_image\" > \"$df\"\n") - f.write(" docker build -t \"$dummy_tag\" -f \"$df\" dummies >/dev/null\n") - f.write("}\n") - f.write("\n") - f.write("load_prefetched_images() {\n") - f.write(" if [ ! -d prefetched_images ]; then\n") - f.write(" return 0\n") - f.write(" fi\n") - f.write(" local tarball\n") - f.write(" shopt -s nullglob\n") - f.write(" for tarball in prefetched_images/*.tar; do\n") - f.write(" echo \"[build_images] loading prefetched image: ${tarball}\" >&2\n") - f.write(" docker load -i \"${tarball}\" >/dev/null\n") - f.write(" done\n") - f.write(" shopt -u nullglob\n") - f.write("}\n") - f.write("\n") - f.write("ensure_docker_daemon_config\n") - f.write("load_prefetched_images\n") - - if used_images: - f.write('echo "[build_images] preparing base-image dummies"\n') - for image in used_images: - digest = md5(image.encode("utf-8")).hexdigest() - f.write(f'prepare_dummy_image "{image}" "{digest}"\n') - f.write("\n") - f.write('JOBS_FILE="$(mktemp)"\n') - f.write('cleanup() { rm -f "${JOBS_FILE}"; }\n') - f.write('trap cleanup EXIT\n') - f.write("cat > \"${JOBS_FILE}\" <<'JOBS'\n") - f.write("\n".join(self.__build_commands)) - f.write("\nJOBS\n") - f.write('if [ "${PARALLELISM}" -le 1 ]; then\n') - f.write(' while IFS= read -r cmd; do\n') - f.write(' [ -z "${cmd}" ] && continue\n') - f.write(' echo "+ ${cmd}"\n') - f.write(' eval "${cmd}"\n') - f.write(' done < "${JOBS_FILE}"\n') - f.write('else\n') - # Run each full line as one job (preserve spaces in the docker commands). - f.write(" awk 'NF' \"${JOBS_FILE}\" | xargs -P \"${PARALLELISM}\" -d '\\n' -I {} bash -lc 'set -euo pipefail; cmd=\"{}\"; echo \"+ ${cmd}\"; eval \"${cmd}\"'\n") - f.write('fi\n') - os.chmod('build_images.sh', 0o755) - - image_refs = [] - for command in self.__build_commands: - parts = shlex.split(command) - if len(parts) >= 2 and parts[0] == 'seedemu_build_and_push': - image_refs.append(parts[1]) - elif len(parts) >= 3 and parts[0] == 'seedemu_copy_and_push_image': - image_refs.append(parts[2]) - with open('images.txt', 'w') as f: - if image_refs: - f.write("\n".join(image_refs) + "\n") - - # Generate .env file - self.generateEnvFile(Scope(0), '') # Simplified scope handling - - def _compileNetK8s(self, net: Network) -> str: - """Generates NetworkAttachmentDefinition for a network. - - Supports multiple CNI types: - - bridge: Local bridge (single-node) - - macvlan: For cross-node with L2 access - - ipvlan: For cross-node networking - - kube-ovn/ovn: Kube-OVN attached networks through OVS - """ - name = self._getRealNetName(net).replace('_', '-').lower() + def _compileNetK8s(self, net: Network) -> Dict[str, Any]: + name = self._getRealNetName(net).replace("_", "-").lower() prefix = str(net.getPrefix()) - cni_type = self._resolveNetworkCniType(net) - - # Build CNI config based on cni_type - if cni_type == "macvlan": + if self.__cni_type == "macvlan": config = { "cniVersion": "0.3.1", "type": "macvlan", "master": self.__cni_master_interface, "mode": "bridge", - "ipam": { - "type": "static" # We manage IPs inside the container - } + "ipam": {"type": "static"}, } - elif cni_type == "ipvlan": + elif self.__cni_type == "ipvlan": config = { "cniVersion": "0.3.1", "type": "ipvlan", "master": self.__cni_master_interface, "mode": "l2", - "ipam": { - "type": "static" - } - } - elif cni_type in {"kube-ovn", "ovn"}: - config = { - "cniVersion": "0.3.1", - "type": "kube-ovn", - "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", - "provider": f"{name}.{self.__namespace}.ovn", + "ipam": {"type": "static"}, } - elif cni_type == "host-local": + elif self.__cni_type == "host-local": config = { "cniVersion": "0.3.1", "type": "bridge", - # The Linux bridge device lives at node scope (not K8s namespace scope). - # If we only hash `name`, different namespaces compiling the same topology - # (e.g., multiple smoke runs) will share the same L2 segment and collide on - # IPs/MACs, causing flaky BGP/OSPF behavior. Salt with namespace to isolate. "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), "isGateway": False, - "ipam": { - "type": "host-local", - "subnet": prefix - } + "ipam": {"type": "host-local", "subnet": prefix}, } - else: # Default: bridge (single-node) + else: config = { "cniVersion": "0.3.1", "type": "bridge", - # Same rationale as host-local branch: isolate bridge names per namespace. "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "ipam": {} # No IPAM, we manage IPs inside + "ipam": {"type": "static"}, } - scope, _, registry_name = net.getRegistryInfo() - annotations = { - "org.seedsecuritylabs.seedemu.meta.type": "global" if scope == "ix" else "local", - "org.seedsecuritylabs.seedemu.meta.scope": str(scope), - "org.seedsecuritylabs.seedemu.meta.name": str(registry_name), - "org.seedsecuritylabs.seedemu.meta.prefix": prefix, - } - if net.getDisplayName() is not None: - annotations["org.seedsecuritylabs.seedemu.meta.displayname"] = str(net.getDisplayName()) - if net.getDescription() is not None: - annotations["org.seedsecuritylabs.seedemu.meta.description"] = str(net.getDescription()) - - manifest = { + return { "apiVersion": "k8s.cni.cncf.io/v1", "kind": "NetworkAttachmentDefinition", "metadata": { "name": name, "namespace": self.__namespace, - "annotations": annotations, - }, - "spec": { - "config": json.dumps(config), + "annotations": self._getInternetMapNetMeta(net), }, + "spec": {"config": json.dumps(config)}, } - return yaml.safe_dump(manifest, sort_keys=False) - - def _resolveNetworkCniType(self, net: Network) -> str: - net_type = net.getType() - if net_type in {NetworkType.Local, NetworkType.CrossConnect}: - if self.__local_link_cni_type: - return self.__local_link_cni_type - if self.__scheduling_strategy == SchedulingStrategy.BY_AS_HARD and self.__cni_type in {"macvlan", "ipvlan"}: - return "bridge" - return self.__cni_type - - def _compileNodeK8s(self, node: Node) -> str: - """Compile a node to Kubernetes Deployment manifest or KubeVirt VirtualMachine. - - Includes: - - Scheduling constraints (nodeSelector based on strategy) - - Resource requests/limits - - Enhanced labels for AS and role identification - - Optional Service generation - """ - self._current_node = node - - # Check virtualization mode - # Virtualization split point: "KubeVirt" -> VirtualMachine, otherwise Deployment. - virtualization_mode = getattr(node, "getVirtualizationMode", lambda: "Container")() - if virtualization_mode == "KubeVirt": - result = self._compileNodeKubeVirt(node) - - if self.__generate_services: - node_name = self._getComposeNodeName(node).replace('_', '-').lower() - asn = str(node.getAsn()) - role = self._nodeRoleToString(node.getRole()) - labels = { - "app": node_name, - "seedemu.io/asn": asn, - "seedemu.io/role": role, - "seedemu.io/name": node.getName(), - "kubevirt.io/domain": node_name - } - service = self._compileServiceK8s(node, node_name, labels) - if service: - result += "\n---\n" + service - - return result - - # 1. Prepare Docker build context (reuse logic from Docker compiler) - real_nodename = self._getRealNodeName(node) - # Ensure the directory exists - if not os.path.exists(real_nodename): - mkdir(real_nodename) + def _renderKubeOvnManifests(self, manifests: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert compiler manifests to Kube-OVN secondary-network resources. - # We need to change directory to generate the Dockerfile in the right place - cwd = os.getcwd() - chdir(real_nodename) + Args: + manifests: Namespace, NAD, and workload manifests generated by the + compiler before fabric-specific conversion. - # Generate Dockerfile - # This will call _addFile, which we intercept to generate the address script - image, _ = self._selectImageFor(node) - dockerfile = self._computeDockerfile(node) - with open('Dockerfile', 'w') as f: - f.write(dockerfile) - - chdir(cwd) - - # 2. Add build command - # If registry_prefix is empty, don't add slash - if self.__registry_prefix: - full_image_name = f"{self.__registry_prefix}/{real_nodename}:latest" - else: - full_image_name = f"{real_nodename}:latest" - - build_context = shlex.quote(f"./{real_nodename}") - build_image = shlex.quote(full_image_name) - build_cmd = f"seedemu_build_and_push {build_image} {build_context}" - self.__build_commands.append(build_cmd) - - # 3. Generate Deployment Manifest - node_name = self._getComposeNodeName(node).replace('_', '-').lower() # K8s names must be DNS compliant - asn = str(node.getAsn()) - role = self._nodeRoleToString(node.getRole()) - - # Compute networks annotation - annotations = {} - if self.__use_multus: - nets = [] - net_specs = [] - needs_json_annotation = False - for iface in node.getInterfaces(): - net = iface.getNet() - net_name = self._getRealNetName(net).replace('_', '-').lower() - nets.append(net_name) - resolved_cni_type = self._resolveNetworkCniType(net) - if resolved_cni_type in {"macvlan", "ipvlan"}: - prefix_len = net.getPrefix().prefixlen - net_specs.append({ - "name": net_name, - "ips": [f"{iface.getAddress()}/{prefix_len}"] - }) - needs_json_annotation = True - else: - net_specs.append({ - "name": net_name, - }) - if resolved_cni_type != self.__cni_type: - needs_json_annotation = True - - if needs_json_annotation and net_specs: - # Static IPAM requires IPs in Multus runtime config. Use JSON form annotation - # so each secondary interface gets deterministic seed-emulator addresses. - annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(net_specs) - elif nets: - annotations["k8s.v1.cni.cncf.io/networks"] = ", ".join(nets) - - # Compute Envs - envs = [{"name": "CONTAINER_NAME", "value": node_name}] - - for o, s in node.getScopedRuntimeOptions(): - envs.append({"name": o.name.upper(), "value": str(o.value)}) - - # Build enhanced labels for identification and service discovery - labels = { - "app": node_name, - "seedemu.io/asn": asn, - "seedemu.io/role": role, - "seedemu.io/name": node.getName(), - "seedemu.io/workload": "seedemu", - } - - # Build pod spec - pod_spec = { - "containers": [{ - "name": "main", - "image": full_image_name, - "imagePullPolicy": self.__image_pull_policy, - "securityContext": {"privileged": True, "capabilities": {"add": ["ALL"]}}, - "command": ["/start.sh"], - "env": envs, - "volumeMounts": [] - }] + Returns: + Manifest list containing a Kube-OVN Vpc, one Subnet per SeedEMU + network, kube-ovn NADs, and workloads with provider IP annotations. + """ + namespace_name = self._findManifestNamespace(manifests) + vpc_name = self._kubeOvnResourceName("vpc", namespace_name) + rendered: List[Dict[str, Any]] = [] + vpc_written = False + + for manifest in manifests: + if manifest.get("kind") == "Namespace" and not vpc_written: + rendered.append(manifest) + rendered.append(self._kubeOvnVpc(namespace_name, vpc_name)) + vpc_written = True + continue + if manifest.get("kind") == "NetworkAttachmentDefinition": + rendered.extend(self._convertNetworkAttachmentToKubeOvn(manifest, namespace_name, vpc_name)) + continue + self._convertWorkloadAnnotationsToKubeOvn(manifest, namespace_name) + rendered.append(manifest) + + if not vpc_written: + rendered.insert(0, self._kubeOvnVpc(namespace_name, vpc_name)) + return rendered + + def _findManifestNamespace(self, manifests: List[Dict[str, Any]]) -> str: + """Return the namespace used by workload manifests.""" + for manifest in manifests: + if manifest.get("kind") == "Namespace": + name = (manifest.get("metadata") or {}).get("name") + if name: + return str(name) + for manifest in manifests: + metadata = manifest.get("metadata") or {} + name = metadata.get("namespace") + if name: + return str(name) + return self.__namespace + + def _kubeOvnResourceName(self, prefix: str, value: str) -> str: + """Return a DNS-safe Kube-OVN cluster-scoped resource name.""" + safe = "".join(ch if ch.isalnum() or ch == "-" else "-" for ch in value.lower()).strip("-") + digest = sha1(value.encode("utf-8")).hexdigest()[:8] + return f"{prefix}-{digest}-{safe}"[:63].rstrip("-") + + def _kubeOvnVpc(self, namespace_name: str, vpc_name: str) -> Dict[str, Any]: + """Return a Kube-OVN Vpc that isolates one SeedEMU namespace.""" + return { + "apiVersion": "kubeovn.io/v1", + "kind": "Vpc", + "metadata": {"name": vpc_name}, + "spec": {"namespaces": [namespace_name]}, } - # Add resource requests/limits if configured - if self.__default_resources: - pod_spec["containers"][0]["resources"] = self.__default_resources - - # Add scheduling constraints based on strategy - self._applySchedulingConstraints(pod_spec, node, asn, role, labels) - - manifest = { - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": node_name, - "namespace": self.__namespace, - "labels": labels - }, + def _convertNetworkAttachmentToKubeOvn( + self, + manifest: Dict[str, Any], + default_namespace: str, + vpc_name: str, + ) -> List[Dict[str, Any]]: + """Return [Subnet, NAD] for one SeedEMU network.""" + metadata = manifest.get("metadata") or {} + nad_name = str(metadata.get("name") or "") + namespace_name = str(metadata.get("namespace") or default_namespace) + if not nad_name: + return [manifest] + annotations = metadata.get("annotations") or {} + prefix = annotations.get(self._metaKey("prefix")) + if not prefix: + raise ValueError(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") + + provider = f"{nad_name}.{namespace_name}.ovn" + subnet_name = self._kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") + subnet = { + "apiVersion": "kubeovn.io/v1", + "kind": "Subnet", + "metadata": {"name": subnet_name}, "spec": { - "replicas": 1, - "selector": {"matchLabels": {"app": node_name}}, - "template": { - "metadata": { - "labels": labels, - "annotations": annotations - }, - "spec": pod_spec - } - } + "protocol": "IPv4", + "provider": provider, + "vpc": vpc_name, + "cidrBlock": str(prefix), + "gateway": self._firstUsableIp(str(prefix)), + "gatewayType": "distributed", + "natOutgoing": False, + "private": False, + }, } - result = json.dumps(manifest) - - # Generate Service if enabled - if self.__generate_services: - service = self._compileServiceK8s(node, node_name, labels) - if service: - result += "\n---\n" + service - - return result - - def _computeNodeSelector(self, node: Node, asn: str, role: str) -> Dict[str, str]: - """Compute nodeSelector based on scheduling strategy.""" - if self.__scheduling_strategy in { - SchedulingStrategy.NONE, - SchedulingStrategy.AUTO, - SchedulingStrategy.BY_AS, - SchedulingStrategy.BY_ROLE, - }: - return {} - - node_key = f"{asn}_{node.getName()}" - if node_key in self.__node_labels: - return self.__node_labels[node_key] - if asn in self.__node_labels: - return self.__node_labels[asn] - - if self.__scheduling_strategy == SchedulingStrategy.BY_AS_HARD: - raise AssertionError( - f"SchedulingStrategy.BY_AS_HARD requires an explicit nodeSelector mapping for ASN {asn}. " - f"Pass SEED_NODE_LABELS_JSON with an entry for ASN {asn}, for example: " - f'{{"{asn}": {{"kubernetes.io/hostname": "node-name"}}}}' + converted = dict(manifest) + converted["spec"] = { + "config": json.dumps( + { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + }, + separators=(",", ":"), ) - - if self.__scheduling_strategy == SchedulingStrategy.CUSTOM: - return {} - - return {} - - def _computePodAffinity(self, asn: str, role: str) -> Dict[str, Any]: - """Compute pod affinity rules for soft grouping strategies.""" - preferred_terms: List[Dict[str, Any]] = [] - - if self.__scheduling_strategy in {SchedulingStrategy.AUTO, SchedulingStrategy.BY_AS}: - preferred_terms.append({ - "weight": 90, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [{ - "key": "seedemu.io/asn", - "operator": "In", - "values": [asn], - }] - }, - "topologyKey": "kubernetes.io/hostname", - } - }) - - if self.__scheduling_strategy in {SchedulingStrategy.AUTO, SchedulingStrategy.BY_ROLE}: - preferred_terms.append({ - "weight": 40, - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [{ - "key": "seedemu.io/role", - "operator": "In", - "values": [role], - }] - }, - "topologyKey": "kubernetes.io/hostname", - } - }) - - if not preferred_terms: - return {} - - return { - "podAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": preferred_terms - } } + return [subnet, converted] - def _computeTopologySpreadConstraints(self) -> List[Dict[str, Any]]: - """Compute topology spread constraints for automatic balancing.""" - if self.__scheduling_strategy != SchedulingStrategy.AUTO: - return [] - - return [{ - "maxSkew": 1, - "topologyKey": "kubernetes.io/hostname", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": { - "matchLabels": { - "seedemu.io/workload": "seedemu" - } - } - }] - - def _applySchedulingConstraints( + def _convertWorkloadAnnotationsToKubeOvn( self, - pod_spec: Dict[str, Any], - node: Node, - asn: str, - role: str, - labels: Dict[str, str], + manifest: Dict[str, Any], + default_namespace: str, ) -> None: - """Apply nodeSelector/affinity/topology spread to a pod or VMI spec.""" - node_selector = self._computeNodeSelector(node, asn, role) - if node_selector: - pod_spec["nodeSelector"] = node_selector - - affinity = self._computePodAffinity(asn, role) - if affinity: - pod_spec["affinity"] = affinity - - spread = self._computeTopologySpreadConstraints() - if spread: - pod_spec["topologySpreadConstraints"] = spread - - def _compileServiceK8s(self, node: Node, node_name: str, labels: Dict[str, str]) -> Optional[str]: - """Generate a Kubernetes Service for a node if needed.""" - ports = node.getPorts() - - # Only generate service if there are exposed ports or for routers - if not ports and node.getRole() != NodeRole.Router: - return None - - service_ports = [] - - # Add ports defined on the node - for (host_port, node_port, proto) in ports: - service_ports.append({ - "name": f"port-{node_port}-{proto}", - "port": node_port, - "targetPort": node_port, - "protocol": proto.upper() - }) - - # If no ports but it's a router, add a default SSH-like port for management - if not service_ports: - return None - - service = { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": f"{node_name}-svc", - "namespace": self.__namespace, - "labels": labels - }, - "spec": { - "type": self.__service_type, - "selector": {"app": node_name}, - "ports": service_ports + """Rewrite Multus static IP annotations for Kube-OVN attached NICs.""" + annotations = self._getPodTemplateAnnotations(manifest) + if not annotations: + return + raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") + if not isinstance(raw_networks, str): + return + try: + networks = json.loads(raw_networks) + except json.JSONDecodeError: + return + if not isinstance(networks, list): + return + + changed = False + for item in networks: + if not isinstance(item, dict): + continue + ips = item.pop("ips", None) + if not ips: + continue + nad_name, nad_namespace = self._parseNetworkSelection(item, default_namespace) + if not nad_name: + continue + ip_values = [str(ip_value).strip().split("/", 1)[0] for ip_value in ips if str(ip_value).strip()] + if not ip_values: + continue + annotations[f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address"] = ",".join(ip_values) + changed = True + + if changed: + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) + + def _getPodTemplateAnnotations(self, manifest: Dict[str, Any]) -> Dict[str, str] | None: + """Return pod-level annotations from a workload or Pod manifest.""" + kind = manifest.get("kind") + if kind == "Pod": + metadata = manifest.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + if kind in {"Deployment", "DaemonSet", "StatefulSet", "Job"}: + template = manifest.setdefault("spec", {}).setdefault("template", {}) + metadata = template.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + return None + + def _parseNetworkSelection(self, item: Dict[str, Any], default_namespace: str) -> tuple[str, str]: + """Return (nad_name, namespace) from one Multus network selection item.""" + raw_name = str(item.get("name") or "") + namespace = str(item.get("namespace") or default_namespace) + if "/" in raw_name: + namespace, raw_name = raw_name.split("/", 1) + return raw_name, namespace + + def _firstUsableIp(self, cidr: str) -> str: + """Return the first usable IPv4 address in a CIDR block.""" + network = ipaddress.ip_network(cidr, strict=False) + if network.version != 4: + raise ValueError(f"Kube-OVN manifest generation currently supports IPv4 only: {cidr}") + hosts = network.hosts() + try: + return str(next(hosts)) + except StopIteration: + return str(network.network_address) + + def _compileNodeK8s(self, node: Node) -> Dict[str, Any]: + real_nodename = self._getRealNodeName(node) + if not os.path.exists(real_nodename): + mkdir(real_nodename) + + cwd = os.getcwd() + chdir(real_nodename) + dockerfile = self._computeDockerfile(node) + with open("Dockerfile", "w", encoding="utf-8") as handle: + handle.write(dockerfile) + self._patch_interface_setup_context(Path("Dockerfile")) + chdir(cwd) + + full_image_name = f"{self.__image_registry_prefix}/{real_nodename}:latest" + self.__image_entries.append( + { + "name": full_image_name, + "context": f"./{real_nodename}", } - } - - return json.dumps(service) + ) - def _compileNodeKubeVirt(self, node: Node) -> str: - """Compile a node to KubeVirt VirtualMachine manifest.""" - node_name = self._getComposeNodeName(node).replace('_', '-').lower() + node_name = self._getComposeNodeName(node).replace("_", "-").lower() asn = str(node.getAsn()) role = self._nodeRoleToString(node.getRole()) - # Fail-fast boundary for unsupported container-only features. - self._assertKubeVirtCompatibility(node) + annotations = self._getInternetMapNodeMeta(node) + net_specs = [] + for iface in node.getInterfaces(): + net = iface.getNet() + net_name = self._getRealNetName(net).replace("_", "-").lower() + prefix_len = net.getPrefix().prefixlen + net_specs.append({"name": net_name, "ips": [f"{iface.getAddress()}/{prefix_len}"]}) + if net_specs: + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(net_specs) - # 1. Build Cloud-Init User Data - user_data = { - "packages": list(node.getSoftware()), - "write_files": [], - "runcmd": [] - } + envs = [{"name": "CONTAINER_NAME", "value": node_name}] + for opt, _scope in node.getScopedRuntimeOptions(): + envs.append({"name": opt.name.upper(), "value": str(opt.value)}) - # Add files - for f in node.getFiles(): - path, content = f.get() - user_data["write_files"].append({ - "path": path, - "content": content, - "permissions": "0644" - }) - - # Generate and add replace_address.sh to configure static IPs on Multus interfaces. - # For VMs, interface names are not always eth1/eth2 (predictable naming varies), - # so the generated script falls back to auto-detection when needed. - address_script = self._generateK8sAddressScript(interface_prefix="eth") - user_data["write_files"].append({ - "path": "/usr/local/bin/replace_address.sh", - "content": address_script, - "permissions": "0755" - }) - user_data["runcmd"].append(["/usr/local/bin/replace_address.sh"]) - - # Routers need forwarding and relaxed rp_filter for asymmetric paths. - user_data["runcmd"].extend([ - ["sh", "-c", "sysctl -w net.ipv4.ip_forward=1"], - ["sh", "-c", "sysctl -w net.ipv4.conf.all.rp_filter=0"], - ["sh", "-c", "sysctl -w net.ipv4.conf.default.rp_filter=0"], - ]) - - for command in node.getBuildCommands(): - user_data["runcmd"].append(self._toCloudInitRunCommand(command, False)) - - for command in node.getBuildCommandsAtEnd(): - user_data["runcmd"].append(self._toCloudInitRunCommand(command, False)) - - # Add start commands - for cmd, fork in node.getStartCommands(): - user_data["runcmd"].append(self._toCloudInitRunCommand(cmd, fork)) - - for cmd, fork in node.getPostConfigCommands(): - user_data["runcmd"].append(self._toCloudInitRunCommand(cmd, fork)) - - cloud_init_yaml = "#cloud-config\n" + yaml.dump(user_data) - - # 2. Network Configuration - networks = [{"name": "default", "pod": {}}] - interfaces = [{"name": "default", "masquerade": {}}] - - for i, iface in enumerate(node.getInterfaces()): - net = iface.getNet() - net_name = self._getRealNetName(net).replace('_', '-').lower() - - networks.append({ - "name": f"net{i+1}", - "multus": {"networkName": net_name} - }) - interfaces.append({ - "name": f"net{i+1}", - "bridge": {} - }) - - # 3. Build Manifest labels = { "app": node_name, "seedemu.io/asn": asn, "seedemu.io/role": role, "seedemu.io/name": node.getName(), "seedemu.io/workload": "seedemu", - "kubevirt.io/domain": node_name } - # Default base image (Ubuntu 22.04) - # TODO: Make this configurable via options - container_disk_image = "quay.io/containerdisks/ubuntu:22.04" - cloud_init_secret_name = f"{node_name.replace('.', '-')}-cloudinit" - - cloud_init_secret = { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": cloud_init_secret_name, - "namespace": self.__namespace, - "labels": labels - }, - "type": "Opaque", - "stringData": { - "userdata": cloud_init_yaml - } - } - - manifest = { - "apiVersion": "kubevirt.io/v1", - "kind": "VirtualMachine", + return { + "apiVersion": "apps/v1", + "kind": "Deployment", "metadata": { "name": node_name, "namespace": self.__namespace, - "labels": labels + "labels": labels, }, "spec": { - "runStrategy": "Always", + "replicas": 1, + "selector": {"matchLabels": {"app": node_name}}, "template": { - "metadata": { - "labels": labels - }, + "metadata": {"labels": labels, "annotations": annotations}, "spec": { - "domain": { - "devices": { - "disks": [ - {"name": "containerdisk", "disk": {"bus": "virtio"}}, - {"name": "cloudinitdisk", "disk": {"bus": "virtio"}} - ], - "interfaces": interfaces - }, - "resources": { - "requests": {"memory": "512M"}, - "limits": {"memory": "1024M"} - } - }, - "networks": networks, - "volumes": [ + "containers": [ { - "name": "containerdisk", - "containerDisk": {"image": container_disk_image} - }, - { - "name": "cloudinitdisk", - "cloudInitNoCloud": { - "secretRef": { - "name": cloud_init_secret_name - } - } + "name": "main", + "image": full_image_name, + "imagePullPolicy": self.__image_pull_policy, + "securityContext": { + "privileged": True, + "capabilities": {"add": ["ALL"]}, + }, + "command": ["/start.sh"], + "env": envs, + "volumeMounts": [], } ] - } - } - } + }, + }, + }, } - # Add scheduling constraints - self._applySchedulingConstraints(manifest["spec"]["template"]["spec"], node, asn, role, labels) - - return json.dumps(cloud_init_secret) + "\n---\n" + json.dumps(manifest) - - def _assertKubeVirtCompatibility(self, node: Node) -> None: - unsupported_features: List[str] = [] + def _metaKey(self, key: str) -> str: + return f"{INTERNET_MAP_META_PREFIX}{key}" + + def _nodeInternetMapRole(self, node: Node) -> str: + _scope, obj_type, _name = node.getRegistryInfo() + if obj_type == "hnode": + return "Host" + if obj_type == "rnode": + return "Router" + if obj_type == "brdnode": + return "BorderRouter" + if obj_type == "csnode": + return "SCION Control Service" + if obj_type == "snode": + return "Emulator Service Worker" + if obj_type == "rs": + return "Route Server" + return obj_type + + def _getInternetMapNodeMeta(self, node: Node) -> Dict[str, str]: + _scope, _obj_type, name = node.getRegistryInfo() + meta = { + self._metaKey("asn"): str(node.getAsn()), + self._metaKey("nodename"): str(name), + self._metaKey("role"): self._nodeInternetMapRole(node), + } - if node.getImportedFiles(): - unsupported_features.append("imported files") - if node.getDockerCommands(): - unsupported_features.append("docker commands") - if node.getDockerVolumes(): - unsupported_features.append("docker volumes") + if node.getDisplayName() is not None: + meta[self._metaKey("displayname")] = str(node.getDisplayName()) + if node.getDescription() is not None: + meta[self._metaKey("description")] = str(node.getDescription()) + if len(node.getClasses()) > 0: + meta[self._metaKey("class")] = json.dumps(node.getClasses()) - runtime_options = list(node.getScopedRuntimeOptions()) - if runtime_options: - unsupported_features.append("runtime options") + for key, value in node.getLabel().items(): + meta[self._metaKey(key)] = str(value) - if unsupported_features: - unsupported_list = ", ".join(unsupported_features) - raise AssertionError( - f"Node as{node.getAsn()}/{node.getName()} in KubeVirt mode has unsupported features: {unsupported_list}" + for index, iface in enumerate(node.getInterfaces()): + net = iface.getNet() + meta[self._metaKey(f"net.{index}.name")] = str(net.getName()) + meta[self._metaKey(f"net.{index}.address")] = ( + f"{iface.getAddress()}/{net.getPrefix().prefixlen}" ) - def _toCloudInitRunCommand(self, command: str, fork: bool) -> List[str]: - command_value = f"{command} &" if fork else command - return ["sh", "-c", command_value] - - def _addFile(self, path: str, content: str) -> str: - """Override _addFile to intercept the replacement script generation.""" - if path == '/replace_address.sh' and self.__use_multus and self._current_node: - # Generate K8s specific address replacement script - content = self._generateK8sAddressScript() - return super()._addFile(path, content) - - def _generateK8sAddressScript(self, interface_prefix: str = "net") -> str: - """ - Generate a script to configure static IPs on Multus interfaces. + return meta - - In containers, Multus interfaces are typically named net1/net2/... - - In VMs (KubeVirt), interface names depend on predictable naming rules and - are not reliably eth1/eth2/... + def _getInternetMapNetMeta(self, net: Network) -> Dict[str, str]: + scope, _obj_type, name = net.getRegistryInfo() + meta = { + self._metaKey("type"): "global" if scope == "ix" else "local", + self._metaKey("scope"): str(scope), + self._metaKey("name"): str(name), + self._metaKey("prefix"): str(net.getPrefix()), + } - The generated script prefers deterministic `${prefix}1..N` when present, - otherwise it auto-detects non-default interfaces by ifindex order. - """ - node = self._current_node - iface_count = len(node.getInterfaces()) - - script = "#!/bin/bash\n" - script += "set -euo pipefail\n" - script += "# K8s Address Replacement (Multus)\n\n" - script += f"iface_prefix=\"{interface_prefix}\"\n" - script += f"expected_ifaces={iface_count}\n\n" - - script += "declare -a seed_ifaces=()\n" - script += "if [ -n \"${iface_prefix}\" ] && ip link show \"${iface_prefix}1\" >/dev/null 2>&1; then\n" - script += " for i in $(seq 1 ${expected_ifaces}); do\n" - script += " seed_ifaces+=(\"${iface_prefix}${i}\")\n" - script += " done\n" - script += "else\n" - script += " default_if=\"$(ip route show default 2>/dev/null | awk '/default/ {print $5; exit}')\"\n" - script += " [ -z \"${default_if}\" ] && default_if=\"eth0\"\n" - script += " mapfile -t seed_ifaces < <(\n" - script += " for dev in /sys/class/net/*; do\n" - script += " name=\"$(basename \"$dev\")\"\n" - script += " [ \"$name\" = \"lo\" ] && continue\n" - script += " [ \"$name\" = \"$default_if\" ] && continue\n" - script += " idx=\"$(cat \"$dev/ifindex\" 2>/dev/null || echo 9999)\"\n" - script += " echo \"$idx $name\"\n" - script += " done | sort -n | awk '{print $2}'\n" - script += " )\n" - script += "fi\n\n" - - script += "configure_iface() {\n" - script += " local slot=\"$1\"\n" - script += " local addr=\"$2\"\n" - script += " local iface=\"${seed_ifaces[$slot]:-}\"\n" - script += " if [ -z \"$iface\" ]; then\n" - script += " echo \"Missing data interface index $slot for $addr\" >&2\n" - script += " return 1\n" - script += " fi\n" - script += " echo \"Configuring $iface with $addr\"\n" - script += " ip link set \"$iface\" up\n" - script += " ip addr flush dev \"$iface\" || true\n" - script += " ip addr add \"$addr\" dev \"$iface\"\n" - script += "}\n\n" - - for i, iface in enumerate(node.getInterfaces()): - addr = iface.getAddress() - prefix_len = iface.getNet().getPrefix().prefixlen - script += f"configure_iface {i} \"{addr}/{prefix_len}\"\n" - - return script - - def attachInternetMap(self, asn: int = -1, net: str = '', ip_address: str = '', - port_forwarding: str = '') -> 'KubernetesCompiler': - """! - @brief Add the Internet Map visualization service to the Kubernetes deployment. - - @param asn the autonomous system number (not used in K8s deployment) - @param net the network name (not used in K8s deployment) - @param ip_address the IP address (not used in K8s deployment) - @param port_forwarding the port forwarding (not used in K8s deployment) - - @returns self, for chaining API calls. - """ - if not self.__internet_map_enabled: - return self - - self._log('attaching the Internet Map service to Kubernetes deployment') - source_image, target_image = self._resolveInternetMapImages() - if target_image != source_image: - self.__build_commands.append( - f"seedemu_copy_and_push_image {source_image} {target_image}" - ) - - # Generate Internet Map Kubernetes manifests - internet_map_manifest = f""" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: seedemu-internet-map - namespace: {self.__namespace} - labels: - app: seedemu-internet-map -spec: - replicas: 1 - selector: - matchLabels: - app: seedemu-internet-map - template: - metadata: - labels: - app: seedemu-internet-map - spec: - containers: - - name: internet-map - image: {target_image} - ports: - - containerPort: 8080 - imagePullPolicy: {self.__image_pull_policy} - securityContext: - privileged: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock ---- -apiVersion: v1 -kind: Service -metadata: - name: seedemu-internet-map-service - namespace: {self.__namespace} -spec: - type: NodePort - selector: - app: seedemu-internet-map - ports: - - port: 8080 - targetPort: 8080 -""" - - self.__manifests.append(internet_map_manifest) - self.__internet_map_enabled = False # Prevent duplicate additions - - return self - - -class NativeKubernetesCompiler(KubernetesCompiler): - """Compatibility name kept for callers using the native compiler import.""" - - pass + if net.getDisplayName() is not None: + meta[self._metaKey("displayname")] = str(net.getDisplayName()) + if net.getDescription() is not None: + meta[self._metaKey("description")] = str(net.getDescription()) + + return meta + + def _patch_interface_setup_context(self, dockerfile_path: Path) -> None: + dockerfile = dockerfile_path.read_text(encoding="utf-8") + match = re.search(r"^COPY\s+(\S+)\s+/interface_setup\s*$", dockerfile, re.MULTILINE) + if not match: + return + + script_path = dockerfile_path.parent / match.group(1) + if not script_path.exists(): + return + + script_path.write_text( + """#!/bin/bash +set -euo pipefail + +cidr_to_net() { + ipcalc -n "$1" | sed -E -n 's/^Network: +([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\/[0-9]{1,2}) +.*/\\1/p' +} + +tmpfile="$(mktemp)" +trap 'rm -f "$tmpfile"' EXIT + +ip -j addr | jq -cr '.[]' | while read -r iface; do + ifname="$(jq -cr '.ifname' <<< "$iface")" + jq -cr '.addr_info[]?' <<< "$iface" | while read -r iaddr; do + addr="$(jq -cr '"\\(.local)/\\(.prefixlen)"' <<< "$iaddr")" + net="$(cidr_to_net "$addr")" + [ -z "$net" ] && continue + line="$(grep -F "$net" ifinfo.txt || true)" + [ -z "$line" ] && continue + new_ifname="$(cut -d: -f1 <<< "$line")" + latency="$(cut -d: -f3 <<< "$line")" + bw="$(cut -d: -f4 <<< "$line")" + loss="$(cut -d: -f5 <<< "$line")" + [ -z "$new_ifname" ] && continue + [ "$bw" = 0 ] && bw=1000000000000 + printf '%s|%s|%s|%s|%s\\n' "$ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "$tmpfile" + done +done + +i=0 +while IFS='|' read -r ifname new_ifname latency bw loss; do + [ -z "$ifname" ] && continue + tmp_ifname="seedtmp${i}" + if [ "$ifname" != "$new_ifname" ]; then + ip link set "$ifname" down + ip link set "$ifname" name "$tmp_ifname" + else + tmp_ifname="$ifname" + fi + printf '%s|%s|%s|%s|%s\\n' "$tmp_ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "${tmpfile}.stage2" + i=$((i + 1)) +done < "$tmpfile" + +while IFS='|' read -r tmp_ifname new_ifname latency bw loss; do + [ -z "$tmp_ifname" ] && continue + if [ "$tmp_ifname" != "$new_ifname" ]; then + ip link set "$tmp_ifname" name "$new_ifname" + fi + ip link set "$new_ifname" up + [ -z "$loss" ] && loss=0 + tc qdisc add dev "$new_ifname" root handle 1:0 tbf rate "${bw}bit" buffer 1000000 limit 1000 + tc qdisc add dev "$new_ifname" parent 1:0 handle 10: netem delay "${latency}ms" loss "${loss}%" +done < "${tmpfile}.stage2" + +rm -f "${tmpfile}.stage2" +""", + encoding="utf-8", + ) + + def _stage_base_image_contexts(self) -> None: + used_images = sorted(getattr(self, "_used_images", set())) + if os.path.isdir("base_images"): + shutil.rmtree("base_images") + if not used_images: + return + + for image in used_images: + digest = md5(image.encode("utf-8")).hexdigest() + dst_root = os.path.join("base_images", digest) + os.makedirs(dst_root, exist_ok=True) + dockerfile = os.path.join(dst_root, "Dockerfile") + with open(dockerfile, "w", encoding="utf-8") as handle: + handle.write(f"FROM {image}\n") diff --git a/seedemu/core/AutonomousSystem.py b/seedemu/core/AutonomousSystem.py index d4da28409..25e57851c 100644 --- a/seedemu/core/AutonomousSystem.py +++ b/seedemu/core/AutonomousSystem.py @@ -63,17 +63,6 @@ def createBgpCluster(self, address: str) -> AutonomousSystem: return self - def createCluster(self, address: str) -> AutonomousSystem: - """! - @brief Backward-compatible alias for createBgpCluster. - - @param address cluster ID rendered into the routing backend's Route - Reflector cluster field. - - @returns self, for chaining API calls. - """ - return self.createBgpCluster(address) - def _validate_cluster_integrity(self, data: Dict[str, Tuple[Set[str], Set[str]]]): """! @brief Validate Route Reflector cluster membership. diff --git a/seedemu/layers/Ebgp.py b/seedemu/layers/Ebgp.py index 0a565ffa5..c9270d588 100644 --- a/seedemu/layers/Ebgp.py +++ b/seedemu/layers/Ebgp.py @@ -64,11 +64,10 @@ def __recordPeer( nextHopSelf: bool = True, routeServerClient: bool = False, ) -> None: - render_name = name if str(name).startswith("Ebgp_") else "Ebgp_{}".format(name) install_router_bgp_session( node, { - "name": render_name, + "name": name, "kind": "ebgp", "local_address": str(localAddress), "local_asn": localAsn, From eae98610b377d66e570198004413e0238c2b55b3 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Wed, 17 Jun 2026 11:52:29 +0800 Subject: [PATCH 5/6] Remove native Kubernetes PR checks --- .github/workflows/k8s-check.yaml | 136 ------------------------------- 1 file changed, 136 deletions(-) delete mode 100644 .github/workflows/k8s-check.yaml diff --git a/.github/workflows/k8s-check.yaml b/.github/workflows/k8s-check.yaml deleted file mode 100644 index d2954887d..000000000 --- a/.github/workflows/k8s-check.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: Check Native Kubernetes - -on: - pull_request: - branches: - - master - - development - - k8s-new - paths: - - ".github/workflows/k8s-check.yaml" - - "MANIFEST.in" - - "development.env" - - "requirements.txt" - - "setup.py" - - "docker_images/multiarch/**" - - "seedemu/compiler/Docker.py" - - "seedemu/compiler/kubernetes.py" - - "seedemu/compiler/__init__.py" - - "seedemu/k8sTools/**" - - "examples/internet/b61_k8s_compile/**" - - "tests/k8s/**" - workflow_dispatch: - inputs: - build_images: - description: "Build and push native K8s workload images to a local registry" - type: boolean - required: false - default: true - -permissions: - contents: read - -env: - PYTHON_VERSION: "3.10" - K8S_NAMESPACE: seedemu-k8s-b61 - K8S_IMAGE_REGISTRY_PREFIX: seedemu - K8S_OUTPUT_DIR: /tmp/seedemu-k8s-output - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - k8s-static: - name: K8s Static - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install static-check dependencies - run: python -m pip install PyYAML - - name: Validate K8s source files - run: python3 tests/k8s/validateK8sStatic.py - - k8s-compile: - name: K8s Compile - runs-on: ubuntu-latest - needs: k8s-static - timeout-minutes: 20 - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install compile dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Compile native K8s example - run: | - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Archive native K8s compile output - uses: actions/upload-artifact@v6 - with: - name: native-k8s-output - path: ${{ env.K8S_OUTPUT_DIR }} - if-no-files-found: error - - k8s-build-images: - name: K8s Build Images - runs-on: ubuntu-latest - needs: k8s-compile - timeout-minutes: 45 - if: ${{ github.event_name == 'pull_request' || github.event.inputs.build_images == 'true' }} - steps: - - name: Check out the source repository - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "requirements.txt" - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - name: Check Docker buildx - run: | - docker version - docker buildx version - - name: Compile native K8s example for image build - run: | - rm -rf "${K8S_OUTPUT_DIR}" - source development.env - python3 examples/internet/b61_k8s_compile/mini_internet_k8s.py \ - --output-dir "${K8S_OUTPUT_DIR}" - - name: Validate native K8s build output - run: | - python3 tests/k8s/validateNativeK8sOutput.py \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --expected-namespace "${K8S_NAMESPACE}" \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" - - name: Build and push images to local registry - run: | - bash tests/k8s/buildNativeK8sImages.sh \ - --output-dir "${K8S_OUTPUT_DIR}" \ - --registry-prefix 127.0.0.1:5000 \ - --image-registry-prefix "${K8S_IMAGE_REGISTRY_PREFIX}" From 4d016a21845ba830173ff6e561d4dafef995c024 Mon Sep 17 00:00:00 2001 From: lxl706423651 <706423651@qq.com> Date: Wed, 17 Jun 2026 19:36:02 +0800 Subject: [PATCH 6/6] update --- examples/internet/b61_k8s_compile/README.md | 91 ++++++++-- examples/internet/b61_k8s_compile/SKILL.md | 2 +- .../b61_k8s_compile/mini_internet_k8s.py | 14 +- seedemu/compiler/kubernetes.py | 82 +++++---- seedemu/k8sTools/README.md | 4 + .../resources/running/buildRegistryImages.py | 2 +- .../resources/running/manageK8sManifest.py | 2 +- .../resources/running/manageRunningStage.py | 167 ++++++++++++++++++ 8 files changed, 313 insertions(+), 51 deletions(-) diff --git a/examples/internet/b61_k8s_compile/README.md b/examples/internet/b61_k8s_compile/README.md index 7e2d9b4a3..9c4095e99 100644 --- a/examples/internet/b61_k8s_compile/README.md +++ b/examples/internet/b61_k8s_compile/README.md @@ -35,9 +35,16 @@ The default output directory is `./output`. To write elsewhere: python3 ./mini_internet_k8s.py --output-dir /tmp/seedemu-b61-output ``` -The compiler writes `k8s.kube-ovn.yaml`, `images.yaml`, and one Docker build -context per compiled SeedEMU node. The default logical image prefix is -`seedemu`, and the default namespace is `seedemu-k8s-b61`. +To compile macvlan manifests instead of the default Kube-OVN/OVS manifests: + +```bash +python3 ./mini_internet_k8s.py --cni-type macvlan +``` + +The default compiler mode writes `k8s.kube-ovn.yaml`, `images.yaml`, and one +Docker build context per compiled SeedEMU node. `--cni-type macvlan` writes +`k8s.yaml` with macvlan NetworkAttachmentDefinitions. `k8sTools.py up` also +accepts `images.txt` metadata when present. The default namespace is `seedemu`. ## Common Deploy Commands @@ -47,17 +54,19 @@ Every setup mode follows the same command shape: python3 ./k8sTools.py build \ --input \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml python3 ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml -python3 ./k8sTools.py down -f ./output -k kubeconfig.yaml +python3 ./k8sTools.py clean -f ./output -k kubeconfig.yaml python3 ./k8sTools.py destroy -d configK3s.yaml ``` -`build` creates or prepares infrastructure and writes `configK3s.yaml` plus -`kubeconfig.yaml`. `up` builds/pushes workload images and applies the SeedEMU -namespace. `down` removes only the workload namespace. `destroy` removes the -K3s/OVN setup and any KVM infrastructure recorded in `configK3s.yaml`. +`build` creates or prepares infrastructure and writes `configK3s.yaml`, +`kubeconfig.yaml`, and `inventory.yaml`. `up` builds/pushes workload images and +applies the SeedEMU namespace. `clean` removes only the workload namespace. +`destroy` removes the K3s/OVN setup and any KVM infrastructure recorded in +`configK3s.yaml`. Add `--keep-temp` to any `k8sTools.py` command when you need to inspect its temporary setup/running directory after a failure. @@ -72,7 +81,8 @@ Use this when you want the shortest local KVM configuration. python3 ./k8sTools.py build \ --input configKvmOvnSimply.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` This file is a minimal template for `kind: kvmOvn`. It only specifies the @@ -89,7 +99,8 @@ Use this for a more explicit local KVM deployment. python3 ./k8sTools.py build \ --input configKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Compared with `configKvmOvnSimply.yaml`, this file pins VM names, IP ranges, @@ -106,7 +117,8 @@ Use this when Kubernetes should be built on existing physical machines. python3 ./k8sTools.py build \ --input configK3sOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` Each `nodes[]` entry describes a real machine: `name`, `role`, management `ip`, @@ -126,7 +138,8 @@ K3s cluster. python3 ./k8sTools.py build \ --input configMultiHostKvmOvn.yaml \ --config-k3s configK3s.yaml \ - --kubeconfig kubeconfig.yaml + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml ``` The `hypervisors[]` section describes each physical KVM host. Each hypervisor @@ -175,10 +188,58 @@ ssh -i ~/.ssh/seedemu_k8s seed@192.0.2.11 'sudo -n true' ## Runtime Notes - `k8sTools.py up` infers the logical compiler image prefix from - `output/images.yaml`; use `--image-registry-prefix` only for custom outputs. -- `k8sTools.py down` keeps the cluster infrastructure and only removes the + `output/images.yaml` or `output/images.txt`; use `--image-registry-prefix` + only for custom outputs. +- During `up`, generated Dockerfile `FROM` images such as + `handsonsecurity/seedemu-base:2.0` and `handsonsecurity/seedemu-router:2.0` + are prepared on the local host first and then loaded into the registry/master + Docker daemon before remote buildx runs. +- `k8sTools.py clean` keeps the cluster infrastructure and only removes the deployed SeedEMU workload. - `k8sTools.py destroy` uses metadata embedded in generated `configK3s.yaml`; do not delete that file before cleanup. - Real `build`, `up`, `down`, and `destroy` commands modify local or remote infrastructure. The compile step alone is non-destructive. + +## Quick Kubernetes Debug Commands + +Run these commands from this directory. Use the local stable kubeconfig +explicitly so a stale `KUBECONFIG` from another experiment is not used by +mistake: + +```bash +export B61_NS=seedemu +export B61_KUBECONFIG=./examples/internet/b61_k8s_compile/kubeconfig.yaml +``` + +Check namespace, nodes, and pod status: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" get ns +kubectl --kubeconfig "${B61_KUBECONFIG}" get namespace "${B61_NS}" +kubectl --kubeconfig "${B61_KUBECONFIG}" get nodes -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get deploy -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get events --sort-by='.lastTimestamp' +``` + +Inspect or enter one pod/deployment. Replace the object name with a real name +from `kubectl get pods` or `kubectl get deploy`: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" describe pod +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" logs --tail=80 +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it -- bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it deploy/ -- sh +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec deploy/ -- birdc show protocols +``` +as1862brd-r13-1.13.7.70-7564dbb49c-9wctj + +Large post-BIRD runs can intermittently return `TLS handshake timeout`, +`Client.Timeout exceeded`, or `http2: client connection lost` while VM CPUs are +saturated. First retry the same command with an explicit request timeout: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" --request-timeout=120s -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +``` diff --git a/examples/internet/b61_k8s_compile/SKILL.md b/examples/internet/b61_k8s_compile/SKILL.md index 5bdec745e..b6b1839a2 100644 --- a/examples/internet/b61_k8s_compile/SKILL.md +++ b/examples/internet/b61_k8s_compile/SKILL.md @@ -36,7 +36,7 @@ unless the task specifically concerns runtime artifacts. Think of the workflow as three separate contracts: 1. Compile contract: - - `NativeKubernetesCompiler` turns a rendered SeedEMU emulator into + - `KubernetesCompiler` turns a rendered SeedEMU emulator into Kubernetes manifests and Docker build contexts. - The B61 example writes compiler output to `./output` by default. - Important files are `k8s.kube-ovn.yaml` or `k8s.yaml`, `images.yaml`, diff --git a/examples/internet/b61_k8s_compile/mini_internet_k8s.py b/examples/internet/b61_k8s_compile/mini_internet_k8s.py index 7b65d2f50..243ca8868 100644 --- a/examples/internet/b61_k8s_compile/mini_internet_k8s.py +++ b/examples/internet/b61_k8s_compile/mini_internet_k8s.py @@ -6,6 +6,7 @@ NetworkAttachmentDefinitions, Deployments. - images.yaml: image build contexts consumed by the generated running/ stage. - per-node Docker build contexts and optional base_images/ contexts. +Default CNI is kube-ovn. Use --cni-type macvlan for macvlan manifests. """ from __future__ import annotations @@ -34,7 +35,7 @@ def findRepoRoot(start: Path) -> Path: from seedemu.utilities import Makers from seedemu.compiler import Platform -from seedemu.compiler import NativeKubernetesCompiler +from seedemu.compiler import KubernetesCompiler HOSTS_PER_AS = 2 @@ -56,6 +57,12 @@ def parse_args() -> argparse.Namespace: default="amd64", help="Target platform for Docker build contexts.", ) + parser.add_argument( + "--cni-type", + choices=("kube-ovn", "ovn", "macvlan"), + default="kube-ovn", + help="Secondary CNI backend for SeedEMU links. Defaults to kube-ovn.", + ) return parser.parse_args() @@ -127,7 +134,7 @@ def main() -> int: output_dir = (SCRIPT_DIR / output_dir).resolve() platform = Platform.AMD64 if args.platform == "amd64" else Platform.ARM64 - compiler = NativeKubernetesCompiler(platform=platform) + compiler = KubernetesCompiler(platform=platform, cni_type=args.cni_type) emu = build_mini_internet(hosts_per_as=HOSTS_PER_AS) emu.compile(compiler, str(output_dir), override=True) @@ -138,7 +145,8 @@ def main() -> int: print("Config file: not used") print(f"Output directory: {output_dir}") print("Image registry prefix: seedemu") - print("Namespace: seedemu-k8s-b61") + print("Namespace: seedemu") + print(f"CNI type: {args.cni_type}") print("Runtime workflow: seedemu.k8sTools") print("Inventory required for compile: no") return 0 diff --git a/seedemu/compiler/kubernetes.py b/seedemu/compiler/kubernetes.py index d5ad8d962..46b13749a 100644 --- a/seedemu/compiler/kubernetes.py +++ b/seedemu/compiler/kubernetes.py @@ -22,10 +22,23 @@ SEEDEMU_NODE_TYPES = ["rnode", "csnode", "hnode", "rs", "snode"] -class NativeKubernetesCompiler(Docker): +class SchedulingStrategy: + """Compatibility constants for older Kubernetes compiler callers.""" + + NONE = "none" + AUTO = "auto" + BY_AS = "by_as" + BY_AS_HARD = "by_as_hard" + BY_ROLE = "by_role" + CUSTOM = "custom" + + +class KubernetesCompiler(Docker): """Minimal Kubernetes compiler for a k8s-native baseline. Design goals: + - public compiler name is KubernetesCompiler + - kube-ovn is the default secondary CNI, macvlan is the only alternative - no inventory dependency during compile - no nodeSelector / affinity / topology spreading - no node-aware preload planning @@ -45,28 +58,37 @@ def __init__( self, image_registry_prefix: str = "seedemu", registry_prefix: str | None = None, - namespace: str = "seedemu-k8s-b61", + namespace: str = "seedemu", + use_multus: bool = True, cni_type: str = "kube-ovn", + local_link_cni_type: str | None = None, cni_master_interface: str = "ens2", image_pull_policy: str = "Always", - # use_multus: bool = True, - # create_namespace: bool = True, + scheduling_strategy: str = SchedulingStrategy.NONE, + node_labels: Dict[str, Dict[str, str]] | None = None, + default_resources: Dict[str, Dict[str, str]] | None = None, + generate_services: bool = False, + service_type: str = "ClusterIP", **kwargs: Any, ) -> None: kwargs["selfManagedNetwork"] = True super().__init__(**kwargs) + if not use_multus: + raise ValueError("KubernetesCompiler requires Multus network attachments.") if registry_prefix is not None: image_registry_prefix = registry_prefix self.__image_registry_prefix = image_registry_prefix.strip().strip("/") self.__namespace = namespace.strip() or "seedemu" - self.__cni_type = cni_type.strip().lower() or "bridge" + self.__cni_type = self._normalizeCniType(cni_type) + if local_link_cni_type is not None and str(local_link_cni_type).strip(): + self._normalizeCniType(str(local_link_cni_type)) self.__cni_master_interface = cni_master_interface.strip() or "eth0" self.__image_pull_policy = image_pull_policy.strip() or "Always" self.__manifests = [] self.__image_entries = [] def getName(self) -> str: - return "NativeKubernetes" + return "Kubernetes" def getManifestName(self) -> str: """Return the manifest filename generated by this compiler.""" @@ -74,13 +96,26 @@ def getManifestName(self) -> str: return "k8s.kube-ovn.yaml" return "k8s.yaml" + @staticmethod + def _normalizeCniType(cni_type: str | None) -> str: + """Normalize and validate the secondary CNI mode.""" + value = (cni_type or "kube-ovn").strip().lower().replace("_", "-") + if value in {"kube-ovn", "ovn"}: + return "kube-ovn" + if value == "macvlan": + return "macvlan" + raise ValueError( + f"KubernetesCompiler supports only cni_type \"kube-ovn\" or " + f"cni_type \"macvlan\"; got {cni_type!r}." + ) + @staticmethod def _safeBridgeName(name: str) -> str: return f"br-{md5(name.encode()).hexdigest()[:12]}" def _usesKubeOvn(self) -> bool: """Return true when the compiler should emit Kube-OVN resources.""" - return self.__cni_type in {"kube-ovn", "kube_ovn", "ovn"} + return self.__cni_type == "kube-ovn" def _doCompile(self, emulator) -> None: registry = emulator.getRegistry() @@ -116,39 +151,24 @@ def _doCompile(self, emulator) -> None: def _compileNetK8s(self, net: Network) -> Dict[str, Any]: name = self._getRealNetName(net).replace("_", "-").lower() - prefix = str(net.getPrefix()) - if self.__cni_type == "macvlan": + if self._usesKubeOvn(): config = { "cniVersion": "0.3.1", - "type": "macvlan", - "master": self.__cni_master_interface, - "mode": "bridge", - "ipam": {"type": "static"}, + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": f"{name}.{self.__namespace}.ovn", } - elif self.__cni_type == "ipvlan": + elif self.__cni_type == "macvlan": config = { "cniVersion": "0.3.1", - "type": "ipvlan", + "type": "macvlan", "master": self.__cni_master_interface, - "mode": "l2", + "mode": "bridge", "ipam": {"type": "static"}, } - elif self.__cni_type == "host-local": - config = { - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "isGateway": False, - "ipam": {"type": "host-local", "subnet": prefix}, - } else: - config = { - "cniVersion": "0.3.1", - "type": "bridge", - "bridge": self._safeBridgeName(f"{self.__namespace}:{name}"), - "ipam": {"type": "static"}, - } + raise ValueError(f"unsupported Kubernetes CNI type: {self.__cni_type}") return { "apiVersion": "k8s.cni.cncf.io/v1", @@ -567,3 +587,5 @@ def _stage_base_image_contexts(self) -> None: dockerfile = os.path.join(dst_root, "Dockerfile") with open(dockerfile, "w", encoding="utf-8") as handle: handle.write(f"FROM {image}\n") + +NativeKubernetesCompiler = KubernetesCompiler diff --git a/seedemu/k8sTools/README.md b/seedemu/k8sTools/README.md index 0d6204e0b..7012b7a01 100644 --- a/seedemu/k8sTools/README.md +++ b/seedemu/k8sTools/README.md @@ -67,6 +67,10 @@ Python entrypoints there: `kustomization.yaml`, rewrites OVN manifests when needed, reads image metadata from `images.yaml` or `images.txt`, builds/pushes images to the registry resolved from `configK3s.yaml`, applies the manifest, and waits for readiness. +Before a remote registry-host build, it scans generated Dockerfiles for external +`FROM` images, ensures those images exist on the local Docker daemon, and loads +them into the registry host Docker daemon so buildx does not need to resolve +compiler base images through Docker Hub from the master node. No persistent setup/running working directory is created by default. Persistent outputs are limited to the user-requested `configK3s.yaml`, `kubeconfig.yaml`, diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.py b/seedemu/k8sTools/resources/running/buildRegistryImages.py index 921081de5..e4de2feeb 100755 --- a/seedemu/k8sTools/resources/running/buildRegistryImages.py +++ b/seedemu/k8sTools/resources/running/buildRegistryImages.py @@ -152,7 +152,7 @@ continue else echo "Missing compiler base image context: ${context}/Dockerfile" >&2 - echo "Re-run compile with the current NativeKubernetesCompiler so base_images/ is generated." >&2 + echo "Re-run compile with the current KubernetesCompiler so base_images/ is generated." >&2 missing=1 fi done < <(sort -u "${tmpfile}") diff --git a/seedemu/k8sTools/resources/running/manageK8sManifest.py b/seedemu/k8sTools/resources/running/manageK8sManifest.py index bc4c9cbd6..157181521 100755 --- a/seedemu/k8sTools/resources/running/manageK8sManifest.py +++ b/seedemu/k8sTools/resources/running/manageK8sManifest.py @@ -409,7 +409,7 @@ def load_images(path: str) -> list[dict[str, str]]: def imagesFromText(text: str) -> list[dict[str, str]]: - """Parse one-image-reference-per-line metadata from NativeKubernetesCompiler. + """Parse one-image-reference-per-line metadata from KubernetesCompiler. Args: text: Contents of images.txt. diff --git a/seedemu/k8sTools/resources/running/manageRunningStage.py b/seedemu/k8sTools/resources/running/manageRunningStage.py index 6328d25e9..3896579f7 100755 --- a/seedemu/k8sTools/resources/running/manageRunningStage.py +++ b/seedemu/k8sTools/resources/running/manageRunningStage.py @@ -142,6 +142,171 @@ def streamTarToRemote(source: Path, ssh_command: list[str], remote_extract_comma raise subprocess.CalledProcessError(return_code, ["tar", "-C", str(source), "-czf", "-", "."]) +def firstFromImage(dockerfile: Path) -> str | None: + """Return the first Dockerfile FROM image, handling optional flags. + + Args: + dockerfile: Dockerfile to inspect. + """ + for raw_line in dockerfile.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if not parts or parts[0].upper() != "FROM": + continue + index = 1 + while index < len(parts) and parts[index].startswith("--"): + index += 1 + if index < len(parts): + return parts[index] + return None + + +def isCompilerHashBaseTag(image: str) -> bool: + """Return true for generated compiler hash base tags. + + Args: + image: Docker image reference from a FROM line. + """ + tag = image.removesuffix(":latest") + return len(tag) == 32 and all(char in "0123456789abcdef" for char in tag) + + +def externalBuildBaseImages(output_dir: Path) -> list[str]: + """Collect non-generated Docker FROM images needed by remote buildx. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + """ + images: list[str] = [] + seen: set[str] = set() + for dockerfile in sorted(output_dir.rglob("Dockerfile")): + image = firstFromImage(dockerfile) + if not image: + continue + if image == "scratch" or image.startswith("$") or "${" in image: + continue + if isCompilerHashBaseTag(image): + continue + if image not in seen: + seen.add(image) + images.append(image) + return images + + +def dockerImageExists(image: str) -> bool: + """Return true when the local Docker daemon has one image reference. + + Args: + image: Docker image reference. + """ + return subprocess.run( + ["docker", "image", "inspect", image], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def dockerIoMirrorRef(image: str) -> str | None: + """Return a Docker Hub mirror reference for unqualified Docker Hub images. + + Args: + image: Docker image reference. + """ + if "/" in image: + first = image.split("/", 1)[0] + if "." in first or ":" in first or first == "localhost": + return None + return f"docker.m.daocloud.io/{image}" + return f"docker.m.daocloud.io/library/{image}" + + +def ensureLocalDockerImage(image: str) -> None: + """Ensure the local Docker daemon has one image, pulling if required. + + Args: + image: Docker image reference. + """ + if dockerImageExists(image): + print(f"[k8s_build] local Docker already has {image}") + return + + print(f"[k8s_build] pulling build base image on local host: {image}") + result = subprocess.run(["docker", "pull", image], check=False) + if result.returncode == 0: + return + + mirror_image = dockerIoMirrorRef(image) + if mirror_image is None: + raise subprocess.CalledProcessError(result.returncode, ["docker", "pull", image]) + + print(f"[k8s_build] pulling build base image from mirror: {mirror_image}") + runCommand(["docker", "pull", mirror_image]) + runCommand(["docker", "tag", mirror_image, image]) + + +def remoteDockerImageExists(image: str, ssh_command: list[str]) -> bool: + """Return true when the remote Docker daemon has one image reference. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + return subprocess.run( + [*ssh_command, f"sudo -n docker image inspect {shlex.quote(image)} >/dev/null 2>&1"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def streamDockerImageToRemote(image: str, ssh_command: list[str]) -> None: + """Load one local Docker image into the remote Docker daemon over SSH. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + if remoteDockerImageExists(image, ssh_command): + print(f"[k8s_build] registry host Docker already has {image}") + return + + ensureLocalDockerImage(image) + print("+ " + " ".join(["docker", "save", image]) + " | " + " ".join(ssh_command + ["sudo -n docker load"])) + producer = subprocess.Popen(["docker", "save", image], stdout=subprocess.PIPE) + try: + assert producer.stdout is not None + subprocess.run([*ssh_command, "sudo -n docker load"], stdin=producer.stdout, check=True) + finally: + if producer.stdout is not None: + producer.stdout.close() + return_code = producer.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, ["docker", "save", image]) + + +def ensureBuildBaseImages(output_dir: Path, ssh_command: list[str] | None) -> None: + """Ensure Dockerfile FROM images are available where buildx will run. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + ssh_command: Base SSH argv for a remote registry host, or None when the + registry/build host is local. + """ + images = externalBuildBaseImages(output_dir) + if not images: + return + print("[k8s_build] ensuring Docker build base images: " + ", ".join(images)) + if ssh_command is None: + for image in images: + ensureLocalDockerImage(image) + return + for image in images: + streamDockerImageToRemote(image, ssh_command) + + def buildImages(config: Path) -> None: """Stage compiler output on the registry node and build/push images. @@ -173,6 +338,7 @@ def buildImages(config: Path) -> None: (remote_root / "running").mkdir(parents=True) copyTreeContents(output_dir, remote_root / "output") copyTreeContents(SCRIPT_DIR, remote_root / "running") + ensureBuildBaseImages(output_dir, None) print("[k8s_build] running local build") runCommand(["sudo", "-n", *build_command], cwd=remote_root / "running") return @@ -193,6 +359,7 @@ def buildImages(config: Path) -> None: ssh_target, ] print(f"[k8s_build] uploading compile output to {ssh_target}:{remote_dir}/output") + ensureBuildBaseImages(output_dir, ssh_command) runCommand( [ *ssh_command,