diff --git a/.github/workflows/on-pull-request.yaml b/.github/workflows/on-pull-request.yaml index ca4885b50..8cac7fed8 100644 --- a/.github/workflows/on-pull-request.yaml +++ b/.github/workflows/on-pull-request.yaml @@ -5,196 +5,456 @@ on: branches: - master - development + - development-ipv6 workflow_dispatch: inputs: - run-quick-tests: - description: "Run quick tests | ignores other options for testing (they still apply for compile/build)" + selected-example: + description: "Run one selected example code, or all" + type: choice + required: true + default: all + options: + - all + - A00 + - A01 + - A02a + - A03a + - A03b + - A05 + - A06 + - A63 + - A62 + - B00 + - B02 + - B31 + - D01 + - D10 + - D20 + - D21 + - D22 + - D23 + run-basic-examples: + description: "Run selected basic examples" type: boolean required: false - default: "false" - run-basic-tests: - description: "Run basic tests | Compile and builds all the basic examples - Runs basic tests" + default: true + run-internet-examples: + description: "Run selected internet examples" type: boolean required: false - default: "true" - run-internet-tests: - description: "Run internet tests | Compile and builds all the internet examples - Runs internet tests" + default: true + run-blockchain-examples: + description: "Run selected blockchain examples" type: boolean required: false - default: "true" - run-blockchain-tests: - description: "Run blockchain tests | Compile and builds all the blockchain examples - Runs blockchain tests" + default: true + run-full-lifecycle: + description: "Run Docker build/up/probe/test/down lifecycle" type: boolean required: false - default: "false" - run-scion-tests: - description: "Run SCION tests | Compile and builds all the SCION examples - Runs SCION tests" - type: boolean - required: false - default: "false" + default: true + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +env: + SELECTED_EXAMPLE: ${{ github.event.inputs['selected-example'] || 'all' }} + RUN_BASIC_EXAMPLES: ${{ github.event.inputs['run-basic-examples'] || 'true' }} + RUN_INTERNET_EXAMPLES: ${{ github.event.inputs['run-internet-examples'] || 'true' }} + RUN_BLOCKCHAIN_EXAMPLES: ${{ github.event.inputs['run-blockchain-examples'] || 'true' }} + RUN_FULL_LIFECYCLE: ${{ github.event.inputs['run-full-lifecycle'] || 'true' }} jobs: - compile-examples: - name: Compile Examples + static-checks: + name: Static Checks runs-on: ubuntu-latest permissions: contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} steps: - name: Check out the source repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: 'pip' + cache: pip cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ env.SCION_TESTS == 'true' }} + + - name: Install dependencies run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Compile Examples + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + + - name: Compile testing framework and selected example tests run: | - source development.env - export PATH=$PATH:$PWD/bin - CMD="tests/compile-and-build-test/compile-test.py" - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" - fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD + python -m py_compile \ + seedemu/testing/base.py \ + seedemu/testing/internet.py \ + seedemu/testing/blockchain.py \ + seedemu/testing/satellite.py \ + seedemu/testing/registry.py \ + seedemu/testing/runtime.py \ + seedemu/testing/cli.py \ + examples/basic/A00_simple_as/simple_as.py \ + examples/basic/A00_simple_as/test_runtime.py \ + examples/basic/A01_transit_as/transit_as.py \ + examples/basic/A01_transit_as/test_runtime.py \ + examples/basic/A02a_transit_as_mpls/transit_as_mpls.py \ + examples/basic/A02a_transit_as_mpls/test_runtime.py \ + examples/basic/A03a_out_to_real_world/out_to_real_world.py \ + examples/basic/A03a_out_to_real_world/test_runtime.py \ + examples/basic/A03b_from_real_world/from_real_world.py \ + examples/basic/A03b_from_real_world/test_runtime.py \ + examples/basic/A05_components/components.py \ + examples/basic/A05_components/test_runtime.py \ + examples/basic/A06_merge_emulation/merge_emulation.py \ + examples/basic/A06_merge_emulation/test_runtime.py \ + examples/basic/A63_control_plane_regression/control_plane_regression.py \ + examples/basic/A63_control_plane_regression/test_runtime.py \ + examples/basic/A62_route_reflector/route_reflector.py \ + examples/basic/A62_route_reflector/test_runtime.py \ + examples/internet/B00_mini_internet/mini_internet.py \ + examples/internet/B00_mini_internet/test_runtime.py \ + examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py \ + examples/internet/B02_mini_internet_with_dns/test_runtime.py \ + examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py \ + examples/internet/B31_mini_internet_mpls/test_runtime.py \ + examples/blockchain/D01_ethereum_pos/ethereum_pos.py \ + examples/blockchain/D01_ethereum_pos/test_runtime.py \ + examples/blockchain/D10_eth_explorer/ethereum_pos.py \ + examples/blockchain/D10_eth_explorer/test_runtime.py \ + examples/blockchain/D20_faucet/faucet.py \ + examples/blockchain/D20_faucet/test_runtime.py \ + examples/blockchain/D21_deploy_contract/deploy_contract.py \ + examples/blockchain/D21_deploy_contract/test_runtime.py \ + examples/blockchain/D22_oracle/simple_oracle.py \ + examples/blockchain/D22_oracle/test_runtime.py \ + examples/blockchain/D23_validator/validator.py \ + examples/blockchain/D23_validator/test_runtime.py \ + examples/blockchain/D23_validator/test_vc_at_running.py - run-tests: - name: Run Tests - needs: compile-examples # Only run heavy tests if quick compile test succeeds + - name: Validate selected example manifests + run: | + python seedemu/testing/cli.py clean examples/basic/A00_simple_as/example.yaml + python seedemu/testing/cli.py clean examples/basic/A01_transit_as/example.yaml + python seedemu/testing/cli.py clean examples/basic/A02a_transit_as_mpls/example.yaml + python seedemu/testing/cli.py clean examples/basic/A03a_out_to_real_world/example.yaml + python seedemu/testing/cli.py clean examples/basic/A03b_from_real_world/example.yaml + python seedemu/testing/cli.py clean examples/basic/A05_components/example.yaml + python seedemu/testing/cli.py clean examples/basic/A06_merge_emulation/example.yaml + python seedemu/testing/cli.py clean examples/basic/A63_control_plane_regression/example.yaml + python seedemu/testing/cli.py clean examples/basic/A62_route_reflector/example.yaml + python seedemu/testing/cli.py clean examples/internet/B00_mini_internet/example.yaml + python seedemu/testing/cli.py clean examples/internet/B02_mini_internet_with_dns/example.yaml + python seedemu/testing/cli.py clean examples/internet/B31_mini_internet_mpls/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D01_ethereum_pos/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D10_eth_explorer/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D20_faucet/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D21_deploy_contract/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D22_oracle/example.yaml + python seedemu/testing/cli.py clean examples/blockchain/D23_validator/example.yaml + + compile-selected-examples: + name: Compile ${{ matrix.example.code }} + needs: static-checks runs-on: ubuntu-latest permissions: contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} + strategy: + fail-fast: false + matrix: + example: + - code: A00 + category: basic + requires_mpls: false + manifest: examples/basic/A00_simple_as/example.yaml + - code: A01 + category: basic + requires_mpls: false + manifest: examples/basic/A01_transit_as/example.yaml + - code: A02a + category: basic + requires_mpls: true + manifest: examples/basic/A02a_transit_as_mpls/example.yaml + - code: A03a + category: basic + requires_mpls: false + manifest: examples/basic/A03a_out_to_real_world/example.yaml + - code: A03b + category: basic + requires_mpls: false + manifest: examples/basic/A03b_from_real_world/example.yaml + - code: A05 + category: basic + requires_mpls: false + manifest: examples/basic/A05_components/example.yaml + - code: A06 + category: basic + requires_mpls: false + manifest: examples/basic/A06_merge_emulation/example.yaml + - code: A63 + category: basic + requires_mpls: false + manifest: examples/basic/A63_control_plane_regression/example.yaml + - code: A62 + category: basic + requires_mpls: false + manifest: examples/basic/A62_route_reflector/example.yaml + - code: B00 + category: internet + requires_mpls: false + manifest: examples/internet/B00_mini_internet/example.yaml + - code: B02 + category: internet + requires_mpls: false + manifest: examples/internet/B02_mini_internet_with_dns/example.yaml + - code: B31 + category: internet + requires_mpls: true + manifest: examples/internet/B31_mini_internet_mpls/example.yaml + - code: D01 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D01_ethereum_pos/example.yaml + - code: D10 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D10_eth_explorer/example.yaml + - code: D20 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D20_faucet/example.yaml + - code: D21 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D21_deploy_contract/example.yaml + - code: D22 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D22_oracle/example.yaml + - code: D23 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D23_validator/example.yaml steps: - name: Check out the source repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: 'pip' + cache: pip cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ env.SCION_TESTS == 'true' }} + + - name: Install dependencies run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Run tests + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + + - name: Compile selected example + if: >- + ${{ + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} run: | source development.env - export PATH=$PATH:$PWD/bin - CMD="tests/run-tests.py" - if [ "$QUICK_TESTS" == "true" ]; then - CMD+=" --ci" - fi - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" - fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD - - name: Archive test results - uses: actions/upload-artifact@v4 + python seedemu/testing/cli.py compile \ + ${{ matrix.example.manifest }} \ + --artifact-dir ci-artifacts/${{ matrix.example.code }}-compile + + - name: Upload compile artifacts + if: >- + ${{ + always() && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + uses: actions/upload-artifact@v6 with: - name: test_result - path: | - tests/test_result.txt - tests/**/test_log - - name: Check for errors - shell: bash - run: set +e; grep -E "^score" tests/test_result.txt | grep -Eq '[1-9][0-9]* errors|[1-9][0-9]* failures'; test $? -eq 1 - - build-examples: - name: Build Examples - needs: compile-examples # Only run heavy tests if quick compile test succeeds + name: example-${{ matrix.example.code }}-compile-artifacts + path: ci-artifacts/${{ matrix.example.code }}-compile + + run-selected-examples: + name: Run ${{ matrix.example.code }} + needs: compile-selected-examples runs-on: ubuntu-latest permissions: contents: read - env: - SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }} - BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }} - INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }} - BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }} - QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }} + strategy: + fail-fast: false + matrix: + example: + - code: A00 + category: basic + requires_mpls: false + manifest: examples/basic/A00_simple_as/example.yaml + - code: A01 + category: basic + requires_mpls: false + manifest: examples/basic/A01_transit_as/example.yaml + - code: A02a + category: basic + requires_mpls: true + manifest: examples/basic/A02a_transit_as_mpls/example.yaml + - code: A03a + category: basic + requires_mpls: false + manifest: examples/basic/A03a_out_to_real_world/example.yaml + - code: A03b + category: basic + requires_mpls: false + manifest: examples/basic/A03b_from_real_world/example.yaml + - code: A05 + category: basic + requires_mpls: false + manifest: examples/basic/A05_components/example.yaml + - code: A06 + category: basic + requires_mpls: false + manifest: examples/basic/A06_merge_emulation/example.yaml + - code: A63 + category: basic + requires_mpls: false + manifest: examples/basic/A63_control_plane_regression/example.yaml + - code: A62 + category: basic + requires_mpls: false + manifest: examples/basic/A62_route_reflector/example.yaml + - code: B00 + category: internet + requires_mpls: false + manifest: examples/internet/B00_mini_internet/example.yaml + - code: B02 + category: internet + requires_mpls: false + manifest: examples/internet/B02_mini_internet_with_dns/example.yaml + - code: B31 + category: internet + requires_mpls: true + manifest: examples/internet/B31_mini_internet_mpls/example.yaml + - code: D01 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D01_ethereum_pos/example.yaml + - code: D10 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D10_eth_explorer/example.yaml + - code: D20 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D20_faucet/example.yaml + - code: D21 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D21_deploy_contract/example.yaml + - code: D22 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D22_oracle/example.yaml + - code: D23 + category: blockchain + requires_mpls: false + manifest: examples/blockchain/D23_validator/example.yaml steps: - name: Check out the source repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" - cache: 'pip' + cache: pip cache-dependency-path: "**/*requirements.txt" - - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt - - name: Get scion-pki - if: ${{ github.event.inputs.run-scion-tests == 'true' }} + + - name: Install dependencies run: | - curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz - mkdir bin - tar -C bin -xzf scion_0.12.0_amd64_linux.tar.gz scion-pki - - name: Build Examples + pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt + + - name: Load MPLS kernel modules + id: mpls + if: >- + ${{ + matrix.example.requires_mpls && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} run: | - source development.env - export PATH=$PATH:$PWD/bin - cd tests/compile-and-build-test - CMD="compile-and-build-test.py" - if [ "$BASIC_TESTS" == "true" ]; then - CMD+=" --basic" + set +e + sudo modprobe mpls_router + sudo modprobe mpls_iptunnel + sudo modprobe mpls_gso + if test -d /proc/sys/net/mpls && lsmod | grep -q '^mpls_router'; then + echo "available=true" >> "$GITHUB_OUTPUT" + lsmod | grep '^mpls_' + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::warning::MPLS kernel modules are not available on this runner. Skipping the MPLS runtime lifecycle for ${{ matrix.example.code }}. Use a self-hosted runner with mpls_router, mpls_iptunnel, and mpls_gso for full MPLS runtime testing." fi - if [ "$INTERNET_TESTS" == "true" ]; then - CMD+=" --internet" - fi - if [ "$BLOCKCHAIN_TESTS" == "true" ]; then - CMD+=" --blockchain" - fi - if [ "$SCION_TESTS" == "true" ]; then - CMD+=" --scion" - fi - python $CMD - - name: Archive test results - uses: actions/upload-artifact@v4 + + - name: Report skipped MPLS lifecycle + if: >- + ${{ + matrix.example.requires_mpls && + steps.mpls.outputs.available == 'false' && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + echo "Skipping ${{ matrix.example.code }} full lifecycle because this GitHub runner does not provide MPLS kernel modules." + + - name: Run selected example lifecycle + if: >- + ${{ + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + (!matrix.example.requires_mpls || steps.mpls.outputs.available == 'true') && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + run: | + source development.env + python seedemu/testing/cli.py all \ + ${{ matrix.example.manifest }} \ + --artifact-dir ci-artifacts/${{ matrix.example.code }} + + - name: Upload lifecycle artifacts + if: >- + ${{ + always() && + env.RUN_FULL_LIFECYCLE == 'true' && + (env.SELECTED_EXAMPLE == 'all' || env.SELECTED_EXAMPLE == matrix.example.code) && + (!matrix.example.requires_mpls || steps.mpls.outputs.available == 'true') && + ( + (matrix.example.category == 'basic' && env.RUN_BASIC_EXAMPLES == 'true') || + (matrix.example.category == 'internet' && env.RUN_INTERNET_EXAMPLES == 'true') || + (matrix.example.category == 'blockchain' && env.RUN_BLOCKCHAIN_EXAMPLES == 'true') + ) + }} + uses: actions/upload-artifact@v6 with: - name: test_log - path: | - tests/compile-and-build-test/test_log/build_log.txt - tests/compile-and-build-test/test_log/log.txt - - name: Check for errors - shell: bash - run: set +e; grep -E "^score" tests/compile-and-build-test/test_log/log.txt | grep -Eq '[1-9][0-9]* errors|[1-9][0-9]* failures'; test $? -eq 1 + name: example-${{ matrix.example.code }}-lifecycle-artifacts + path: ci-artifacts/${{ matrix.example.code }} diff --git a/.github/workflows/sync-gitee.yaml b/.github/workflows/sync-gitee.yaml deleted file mode 100644 index a90a33764..000000000 --- a/.github/workflows/sync-gitee.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: sync2gitee -on: - push: - branches: - - master -jobs: - repo-sync: - env: - SSH_PRIVATE_KEY: ${{ secrets.GITEE_SSH_KEY }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - - name: Configure Git - run: | - git config --global --add safe.directory /github/workspace - - - name: sync github -> gitee - uses: wearerequired/git-mirror-action@master - if: env.SSH_PRIVATE_KEY - with: - source-repo: "git@github.com:${{ github.repository }}.git" - destination-repo: "git@gitee.com:seedlab/seed-emulator.git" diff --git a/.gitignore b/.gitignore index f728dddbf..e9cfac8dd 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ test_result.txt **/auto-imports.d.ts **/components.d.ts +ci-artifacts diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..610b96bfa --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +# K8sTools uses importlib.resources to copy these templates into temporary +# setup/running directories after users install the seedemu package. +# Only portable source templates are matched here, so runtime caches such as +# __pycache__, *.pyc, and image-cache/*.tar are left out of source packages. +include seedemu/k8sTools/README.md +recursive-include seedemu/k8sTools/resources *.py *.sh *.yml *.yaml README.md diff --git a/development.env b/development.env index 59e25b17a..8a2f682dd 100644 --- a/development.env +++ b/development.env @@ -1 +1 @@ -export PYTHONPATH="`pwd`:$PYTHONPATH" +export PYTHONPATH="`pwd`:${PYTHONPATH:-}" diff --git a/docker_images/seedemu-base/Dockerfile b/docker_images/seedemu-base/Dockerfile index 037f7eb97..b7882ed75 100644 --- a/docker_images/seedemu-base/Dockerfile +++ b/docker_images/seedemu-base/Dockerfile @@ -1,7 +1,7 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive RUN echo 'exec zsh' > /root/.bashrc -RUN apt-get update && apt-get install -y --no-install-recommends curl dnsutils ipcalc iproute2 iputils-ping jq mtr-tiny nano netcat tcpdump termshark vim-nox zsh +RUN apt-get update && apt-get install -y --no-install-recommends curl dnsutils ipcalc iproute2 iputils-ping jq mtr-tiny nano netcat-openbsd tcpdump termshark vim-nox zsh COPY zshrc /root/.zshrc \ No newline at end of file diff --git a/docker_images/seedemu-base/docker-compose.yml b/docker_images/seedemu-base/docker-compose.yml index 7ba5b857b..d1217146f 100644 --- a/docker_images/seedemu-base/docker-compose.yml +++ b/docker_images/seedemu-base/docker-compose.yml @@ -4,6 +4,6 @@ services: build: context: . dockerfile: Dockerfile - image: handsonsecurity/seedemu-base + image: handsonsecurity/seedemu-base:2.0 diff --git a/docker_images/seedemu-ethereum/docker-compose.yml b/docker_images/seedemu-ethereum/docker-compose.yml index 66fbc9483..dd24adce2 100644 --- a/docker_images/seedemu-ethereum/docker-compose.yml +++ b/docker_images/seedemu-ethereum/docker-compose.yml @@ -12,3 +12,9 @@ services: context: ./pos dockerfile: Dockerfile image: handsonsecurity/seedemu-ethereum:pos + + seedemu-pos2.0: + build: + context: ./pos2.0 + dockerfile: Dockerfile + image: handsonsecurity/seedemu-ethereum:pos2.0 diff --git a/docker_images/seedemu-ethereum/pos2.0/Dockerfile b/docker_images/seedemu-ethereum/pos2.0/Dockerfile new file mode 100644 index 000000000..5fc834a56 --- /dev/null +++ b/docker_images/seedemu-ethereum/pos2.0/Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu:20.04 AS builder1 + +ENV DEBIAN_FRONTEND=noninteractive +ENV GOLANG_VERSION=1.24.0 +ENV PATH="/usr/local/go/bin:$PATH" + +WORKDIR / + +# Install dependencies +RUN apt-get update && apt-get install -y \ + wget git build-essential curl make python3 python3-pip ca-certificates \ + && apt-get clean + +# Install Go manually +RUN curl -OL https://go.dev/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go${GOLANG_VERSION}.linux-amd64.tar.gz && \ + rm go${GOLANG_VERSION}.linux-amd64.tar.gz + +RUN go version # Just to confirm installation + + +# Intalling eth-beacon-genesis and eth2-val-tools +RUN git clone https://github.com/ethpandaops/eth-beacon-genesis.git \ + && cd eth-beacon-genesis && git checkout 703e97aaa244eec4b077f974e01d02449e1ba2ba && make \ + && go install github.com/protolambda/eth2-val-tools@0d6d1ddb36479e73d7d876b29ac2d10ab3988e85 + +# Downloading geth +RUN wget https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.15.11-36b2371c.tar.gz \ + && tar -xzf geth-linux-amd64-1.15.11-36b2371c.tar.gz + +# Downloading lighthouse +RUN wget https://github.com/sigp/lighthouse/releases/download/v8.1.3/lighthouse-v8.1.3-x86_64-unknown-linux-gnu.tar.gz \ + && tar -xzf lighthouse-v8.1.3-x86_64-unknown-linux-gnu.tar.gz + + +FROM golang:1.18 AS builder2 +ARG DEBIAN_FRONTEND=noninteractive +WORKDIR / + +# Build bootnode from go-ethereum v1.10.26 source +RUN go install github.com/ethereum/go-ethereum/cmd/bootnode@v1.10.26 + + + +# Final image + +FROM handsonsecurity/seedemu-base + +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common python3 python3-pip +RUN pip install web3 + +COPY --from=builder1 /eth-beacon-genesis/bin/eth-beacon-genesis /usr/local/bin/eth-beacon-genesis +COPY --from=builder1 /root/go/bin/eth2-val-tools /usr/local/bin/eth2-val-tools +COPY --from=builder1 /geth-linux-amd64-1.15.11-36b2371c/geth /usr/local/bin/geth +COPY --from=builder1 /lighthouse /usr/local/bin/lighthouse + +COPY --from=builder2 /go/bin/bootnode /usr/local/bin/bootnode diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/Dockerfile b/docker_images/seedemu-ethexplorer/beaconchain/backend/Dockerfile new file mode 100644 index 000000000..14f7d9c63 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/Dockerfile @@ -0,0 +1,38 @@ +# this is meant to be built with the root of this repo as build-context +FROM golang:1.23-bookworm AS build-env + +WORKDIR / +RUN git clone https://github.com/gobitfly/beaconchain.git +WORKDIR /beaconchain/backend +COPY . . +RUN go mod download +ARG target=all +RUN git config --global --add safe.directory '*' && make -B $target + + +# final stage +FROM debian:bookworm-slim +ENV POSTGRES_PORT=$POSTGRES_PORT\ + ALLOY_PORT=$ALLOY_PORT\ + LBT_PORT=$LBT_PORT\ + RBT_PORT=$RBT_PORT\ + EL_PORT=$EL_PORT\ + CL_PORT=$CL_PORT\ + REDIS_PORT=$REDIS_PORT\ + REDIS_SESSIONS_PORT=$REDIS_SESSIONS_PORT\ + SERVER_HOST=$SERVER_HOST +RUN apt-get update && \ + apt-get install -y \ + curl \ + jq \ + ca-certificates \ + postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build-env /beaconchain/backend/bin /usr/local/bin/ +COPY --from=build-env /beaconchain/backend/local_deployment/provision-explorer-config-custom.sh /app/local_deployment/provision-explorer-config-custom.sh +COPY --from=build-env /beaconchain/backend/local_deployment/mainnet.chain.yml /app/local_deployment/mainnet.chain.yml +COPY start.sh / +RUN chmod +x /start.sh \ + && chmod +x /app/local_deployment/provision-explorer-config-custom.sh +ENTRYPOINT ["/start.sh"] diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/mainnet.chain.yml b/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/mainnet.chain.yml new file mode 100644 index 000000000..1cb5197f7 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/mainnet.chain.yml @@ -0,0 +1,307 @@ +# Mainnet config + +# Extends the mainnet preset +PRESET_BASE: "mainnet" + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: "mainnet" + +# Transition +# --------------------------------------------------------------- +# TBD, 2**256-2**10 is a placeholder +TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Dec 1, 2020, 12pm UTC +MIN_GENESIS_TIME: 1606824000 +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x00000000 +# 604800 seconds (7 days) +GENESIS_DELAY: 604800 + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x01000000 +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +# Bellatrix +BELLATRIX_FORK_VERSION: 0x02000000 +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +# Capella +CAPELLA_FORK_VERSION: 0x03000000 +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +# Deneb +DENEB_FORK_VERSION: 0x04000000 +DENEB_FORK_EPOCH: 18446744073709551615 +# Byzantium +BYZANTIUM_FORK_BLOCK: 4370000 +# Constantinople +CONSTANTINOPLE_FORK_BLOCK: 7280000 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum PoW Mainnet +DEPOSIT_CHAIN_ID: 1337 +DEPOSIT_NETWORK_ID: 1337 +DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + +# Mainnet preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 + diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/provision-explorer-config-custom.sh b/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/provision-explorer-config-custom.sh new file mode 100644 index 000000000..de3cee848 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/local_deployment/provision-explorer-config-custom.sh @@ -0,0 +1,271 @@ +#! /bin/bash + +POSTGRES_HOST=${POSTGRES_HOST:-postgres} +POSTGRES_PORT=${POSTGRES_PORT:-5432} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pass} +POSTGRES_USER=${POSTGRES_USER:-postgres} +POSTGRES_DB=${POSTGRES_DB:-db} +ALLOY_PORT=${ALLOY_PORT:-5432} +CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-9000} +LBT_PORT=${LBT_PORT:-9000} +RBT_PORT=${RBT_PORT:-9000} +EL_HOST=${EL_HOST:-192.168.254.128} +EL_PORT=${EL_PORT:-8545} +CL_HOST=${CL_HOST:-192.168.254.128} +CL_PORT=${CL_PORT:-8000} +REDIS_PORT=${REDIS_PORT:-6379} +REDIS_SESSIONS_PORT=${REDIS_SESSIONS_PORT:-6379} +SERVER_PORT=${SERVER_PORT:-5000} +CLICKHOUSE_DB=${CLICKHOUSE_DB:-beaconchain} + +export POSTGRES_HOST=$POSTGRES_HOST POSTGRES_PORT=$POSTGRES_PORT POSTGRES_PASSWORD=$POSTGRES_PASSWORD POSTGRES_USER=$POSTGRES_USER POSTGRES_DB=$POSTGRES_DB CLICKHOUSE_DB=$CLICKHOUSE_DB + +######################################## +# Common Retry Function - JSON RPC +######################################## + +retry_rpc_method() { + local rpc_url="$1" + local method="$2" + + local resp + local value + + local i=0 + while true; do + i=$((i + 1)) + echo "[$method] Attempt $i..." >&2 + + resp=$(curl -sS --fail -X POST "$rpc_url" \ + -H "Content-Type: application/json" \ + -d "{ + \"jsonrpc\":\"2.0\", + \"method\":\"$method\", + \"params\":[], + \"id\":1 + }" 2>/dev/null) + + value=$(echo "$resp" | jq -er '.result // empty' 2>/dev/null || true) + + if [ -n "$value" ]; then + echo "[$method] Success: $value" >&2 + echo "$value" + + return 0 + fi + + echo "[$method] Failed response:" >&2 + echo "$resp" >&2 + + echo "[$method] Retrying in 30s..." >&2 + sleep 30 + done + + echo "[$method] Failed after retries" >&2 + return 1 +} + +######################################## +# Beacon Genesis +######################################## + +URL="http://${CL_HOST}:${CL_PORT}/eth/v1/beacon/genesis" + +count=0 +while true; do + count=$((count + 1)) + echo "[Genesis] Attempt $count..." + + RESP=$(curl -sS --fail "$URL" 2>/dev/null) + + GENESIS_TIMESTAMP=$(echo "$RESP" | jq -er '.data.genesis_time // empty' 2>/dev/null || true) + GENESIS_VALIDATORSROOT=$(echo "$RESP" | jq -er '.data.genesis_validators_root // empty' 2>/dev/null || true) + + if [ -n "$GENESIS_TIMESTAMP" ] && [ -n "$GENESIS_VALIDATORSROOT" ]; then + echo "[Genesis] Success!" + echo "[Genesis] GENESIS_TIMESTAMP=$GENESIS_TIMESTAMP" + echo "[Genesis] GENESIS_VALIDATORS_ROOT=$GENESIS_VALIDATORSROOT" + break + fi + + echo "[Genesis] Failed response:" + echo "$RESP" + + echo "[Genesis] Retrying in 30s..." + sleep 30 +done + +if [ -z "$GENESIS_TIMESTAMP" ] || [ -z "$GENESIS_VALIDATORSROOT" ]; then + echo "Failed to get GENESIS_TIMESTAMP or GENESIS_VALIDATORSROOT" + exit 1 +fi + +######################################## +# Execution Layer RPC +######################################## + +RPC_URL="http://${EL_HOST}:${EL_PORT}" + +CHAIN_HEX=$( + retry_rpc_method \ + "$RPC_URL" \ + "eth_chainId" +) + +NETWORK_ID=$( + retry_rpc_method \ + "$RPC_URL" \ + "net_version" +) + +if [ -z "$CHAIN_HEX" ]; then + echo "Failed to get CHAIN_HEX" + exit 1 +fi + +if [ -z "$NETWORK_ID" ]; then + echo "Failed to get NETWORK_ID" + exit 1 +fi + +CHAIN_ID=$((CHAIN_HEX)) + +echo "chainId (dec): $CHAIN_ID" +echo "networkId (dec): $NETWORK_ID" + +FILE="/app/local_deployment/mainnet.chain.yml" +sed -i "s/^DEPOSIT_CHAIN_ID:.*/DEPOSIT_CHAIN_ID: ${CHAIN_ID}/" "$FILE" +sed -i "s/^DEPOSIT_NETWORK_ID:.*/DEPOSIT_NETWORK_ID: ${NETWORK_ID}/" "$FILE" + +touch config.yml + +cat >config.yml < lastValidatorIndex { + return nil, fmt.Errorf("validator index outside of mapped range (%d is not within 0-%d)", index, lastValidatorIndex) + } + res[i] = mapping.ValidatorPubkeys[index] + } + return res, nil +} + +func (s *Services) GetIndexSliceFromPubkeySlice(pubkeys []string) ([]constypes.ValidatorIndex, error) { + res := make([]constypes.ValidatorIndex, len(pubkeys)) + mapping, err := s.GetCurrentValidatorMapping() + if err != nil { + return nil, err + } + for i, pubkey := range pubkeys { + p, ok := mapping.ValidatorIndices[pubkey] + if !ok { + return nil, fmt.Errorf("pubkey %s not found in mapping", pubkey) + } + res[i] = p + } + return res, nil +} diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db/clickhouse.go b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db/clickhouse.go new file mode 100644 index 000000000..bfac401f0 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db/clickhouse.go @@ -0,0 +1,176 @@ +package db + +import ( + "context" + "fmt" + "net" + "runtime" + "time" + + ch "github.com/ClickHouse/clickhouse-go/v2" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" + "github.com/gobitfly/beaconchain/pkg/commons/types" + "github.com/gobitfly/beaconchain/pkg/commons/version" + "golang.org/x/sync/errgroup" +) + +var ClickHouseNativeWriter ch.Conn + +func MustInitClickhouseNative(writer *types.DatabaseConfig) ch.Conn { + if writer.MaxOpenConns == 0 { + writer.MaxOpenConns = 50 + } + if writer.MaxIdleConns == 0 { + writer.MaxIdleConns = 10 + } + if writer.MaxOpenConns < writer.MaxIdleConns { + writer.MaxIdleConns = writer.MaxOpenConns + } + var hosts []string + hosts = append(hosts, net.JoinHostPort(writer.Host, writer.Port)) + if len(writer.Failovers) > 0 { + for _, f := range writer.Failovers { + hosts = append(hosts, net.JoinHostPort(f.Host, f.Port)) + } + } + + log.Infof("initializing clickhouse native writer db connection to %v/%v with %v/%v conn limit", hosts, writer.Name, writer.MaxIdleConns, writer.MaxOpenConns) + dbWriter, err := ch.Open(&ch.Options{ + MaxOpenConns: writer.MaxOpenConns, + MaxIdleConns: writer.MaxIdleConns, + // ConnMaxLifetime: time.Minute, + // the following lowers traffic between client and server + Compression: &ch.Compression{ + Method: ch.CompressionLZ4, + }, + Addr: hosts, + ConnOpenStrategy: ch.ConnOpenInOrder, + Auth: ch.Auth{ + Username: writer.Username, + Password: writer.Password, + Database: writer.Name, + }, + Debug: false, + // TLS: &tls.Config{InsecureSkipVerify: false, MinVersion: tls.VersionTLS12}, + TLS: nil, + // this gets only called when debug is true + Debugf: func(s string, p ...interface{}) { + log.Debugf("CH NATIVE WRITER: "+s, p...) + }, + Settings: ch.Settings{ + // https://clickhouse.com/docs/operations/settings/settings#deduplicate_blocks_in_dependent_materialized_views + // when an insert to a table with dependent materialized views fails during the materialized view processing, said table will still retain the rows inserted. + // this setting ensure that when the insert query gets retried by our code, the attempt doesn't get filtered out by the target table doing de-duplication, + // and instead ensures that all dependent materialized views receive the data anyways + "deduplicate_blocks_in_dependent_materialized_views": "1", + // trade of higher background overhead for lower query specific memory pressure + // reduces memory usage by 20-30% in our prod insert queries + "optimize_on_insert": "0", + }, + ClientInfo: ch.ClientInfo{ + Products: []struct { + Name string + Version string + }{ + {Name: "beaconchain-explorer", Version: version.Version}, + }, + }, + }) + if err != nil { + log.Fatal(err, "Error connecting to clickhouse native writer", 0) + } + // verify connection + ClickHouseTestConnection(dbWriter, writer.Name) + + return dbWriter +} + +func ClickHouseTestConnection(db ch.Conn, dataBaseName string) { + v, err := db.ServerVersion() + if err != nil { + log.Fatal(fmt.Errorf("failed to ping clickhouse database %s: %w", dataBaseName, err), "", 0) + } + log.Debugf("connected to clickhouse database %s with version %s", dataBaseName, v) +} + +type UltraFastClickhouseStruct interface { + Get(string) any + Extend(UltraFastClickhouseStruct) error +} + +func UltraFastDumpToClickhouse[T UltraFastClickhouseStruct](data T, target_table string, insert_uuid string) error { + start := time.Now() + // add metrics + defer func() { + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_overall", target_table)).Observe(time.Since(start).Seconds()) + }() + now := time.Now() + // get column order & names from clickhouse + var columns []string + err := ClickHouseReader.Select(&columns, "SELECT name FROM system.columns where table=$1 and database=currentDatabase() order by position;", target_table) + if err != nil { + return err + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_get_columns", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() + // prepare batch + abortCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{ + "insert_deduplication_token": insert_uuid, // this is used by tables & materialized views to correctly handle retries of inserts (skipping them if they already have the resulting rows, for example) + "insert_deduplicate": true, // enforce deduplication to be done by tables & materialized views + }), ch.WithLogs(func(l *ch.Log) { + log.Debugf("CH NATIVE WRITER: %s", l.Text) + }), + ) + batch, err := ClickHouseNativeWriter.PrepareBatch(ctx, `INSERT INTO `+target_table) + if err != nil { + return err + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_prepare_batch", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() + defer func() { + if batch.IsSent() { + return + } + err := batch.Abort() + if err != nil { + log.Warnf("failed to abort batch: %v", err) + } + }() + var g errgroup.Group + g.SetLimit(runtime.NumCPU()) + // iterate columns retrieved from clickhouse + for i, n := range columns { + // Capture the loop variable + col_index := i + col_name := n + if col_name == "_inserted_at" { + continue + } + // Start a new goroutine for each column + g.Go(func() error { + // get it from the struct + column := data.Get(col_name) + if column == nil { + return fmt.Errorf("column %s not found in struct", col_name) + } + // Perform the type assertion and append operation + err = batch.Column(col_index).Append(column) + log.Debugf("appended column %s in %s", col_name, time.Since(now)) + return err + }) + } + if err := g.Wait(); err != nil { + return err + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_append_columns", target_table)).Observe(time.Since(now).Seconds()) + now = time.Now() + err = batch.Send() + if err != nil { + return err + } + metrics.TaskDuration.WithLabelValues(fmt.Sprintf("clickhouse_dump_%s_send_batch", target_table)).Observe(time.Since(now).Seconds()) + return nil +} diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db2/raw/tables.go b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db2/raw/tables.go new file mode 100644 index 000000000..4b14a4914 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/db2/raw/tables.go @@ -0,0 +1,23 @@ +package raw + +const Table = "blocks_raw" + +const BT_COLUMNFAMILY_BLOCK = "b" +const BT_COLUMN_BLOCK = "b" +const BT_COLUMNFAMILY_RECEIPTS = "r" +const BT_COLUMN_RECEIPTS = "r" +const BT_COLUMNFAMILY_TRACES = "t" +const BT_COLUMN_TRACES = "t" +const BT_COLUMNFAMILY_UNCLES = "u" +const BT_COLUMN_UNCLES = "u" + +const MAX_EL_BLOCK_NUMBER = int64(1_000_000_000_000 - 1) + +var Schema = map[string][]string{ + Table: { + BT_COLUMNFAMILY_BLOCK, + BT_COLUMNFAMILY_RECEIPTS, + BT_COLUMNFAMILY_TRACES, + BT_COLUMNFAMILY_UNCLES, + }, +} diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/utils/config.go b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/utils/config.go new file mode 100644 index 000000000..abd6217cb --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/pkg/commons/utils/config.go @@ -0,0 +1,574 @@ +package utils + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "net" + "os" + "slices" + "strings" + + "github.com/ethereum/go-ethereum/params" + "github.com/sethvargo/go-envconfig" + + "github.com/gobitfly/beaconchain/pkg/commons/config" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/types" + "github.com/gobitfly/beaconchain/pkg/consapi" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +var Config *types.Config + +func readConfigFile(cfg *types.Config, path string) error { + if path == "" { + return yaml.Unmarshal([]byte(config.DefaultConfigYml), cfg) + } + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening config file %v: %v", path, err) + } + + decoder := yaml.NewDecoder(f) + err = decoder.Decode(cfg) + if err != nil { + return fmt.Errorf("error decoding config file %v: %v", path, err) + } + + return nil +} + +func readConfigEnv(cfg *types.Config) error { + return envconfig.ProcessWith(context.Background(), &envconfig.Config{ + Target: cfg, + DefaultOverwrite: true, + }) +} + +func readConfigSecrets(cfg *types.Config) error { + return ProcessSecrets(cfg) +} + +func confSanityCheck(cfg *types.Config) { + if cfg.Chain.ClConfig.SlotsPerEpoch == 0 || cfg.Chain.ClConfig.SecondsPerSlot == 0 { + log.Fatal(nil, "invalid chain configuration specified, you must specify the slots per epoch, seconds per slot and genesis timestamp in the config file", 0) + } +} + +func ReadConfig(cfg *types.Config, path string) error { + configPathFromEnv := os.Getenv("BEACONCHAIN_CONFIG") + + if configPathFromEnv != "" { // allow the location of the config file to be passed via env args + path = configPathFromEnv + } + if strings.HasPrefix(path, "projects/") { + x, err := AccessSecretVersion(path) + if err != nil { + return fmt.Errorf("error getting config from secret store: %v", err) + } + err = yaml.Unmarshal([]byte(*x), cfg) + if err != nil { + return fmt.Errorf("error decoding config file %v: %v", path, err) + } + + log.Infof("seeded config file from secret store") + } else { + err := readConfigFile(cfg, path) + if err != nil { + return err + } + } + + err := readConfigEnv(cfg) + if err != nil { + return err + } + err = readConfigSecrets(cfg) + if err != nil { + return err + } + + if cfg.Frontend.SiteBrand == "" { + cfg.Frontend.SiteBrand = "beaconcha.in" + } + + switch strings.ToLower(os.Getenv("LOG_LEVEL")) { + case "trace": + logrus.SetLevel(logrus.TraceLevel) + case "debug": + logrus.SetLevel(logrus.DebugLevel) + case "info": + logrus.SetLevel(logrus.InfoLevel) + case "warn": + logrus.SetLevel(logrus.WarnLevel) + case "error": + logrus.SetLevel(logrus.ErrorLevel) + case "fatal": + logrus.SetLevel(logrus.FatalLevel) + case "panic": + logrus.SetLevel(logrus.PanicLevel) + } + + err = setCLConfig(cfg) + if err != nil { + return err + } + + err = setELConfig(cfg) + if err != nil { + return err + } + + cfg.Chain.Name = cfg.Chain.ClConfig.ConfigName + + // match DeploymentType to development, staging, production. if its empty fallback to development + validTypes := []string{"development", "development_noisy", "staging", "production"} + if cfg.DeploymentType == "" { + log.Warn("DeploymentType not set, defaulting to development") + cfg.DeploymentType = validTypes[0] + } + if !slices.Contains(validTypes, cfg.DeploymentType) { + log.Fatal(fmt.Errorf("invalid DeploymentType: %v (valid types: %v)", cfg.DeploymentType, validTypes), "", 0) + } + + if cfg.Chain.GenesisTimestamp == 0 { + switch cfg.Chain.Name { + case "mainnet": + cfg.Chain.GenesisTimestamp = 1606824023 + case "prater": + cfg.Chain.GenesisTimestamp = 1616508000 + case "sepolia": + cfg.Chain.GenesisTimestamp = 1655733600 + case "zhejiang": + cfg.Chain.GenesisTimestamp = 1675263600 + case "gnosis": + cfg.Chain.GenesisTimestamp = 1638993340 + case "holesky": + cfg.Chain.GenesisTimestamp = 1695902400 + case "hoodi": + cfg.Chain.GenesisTimestamp = 1742213400 + case "pectra-devnet-5": + cfg.Chain.GenesisTimestamp = 1737034260 + case "pectra-devnet-6": + cfg.Chain.GenesisTimestamp = 1738603860 + default: + return fmt.Errorf("tried to set known genesis-timestamp, but unknown chain-name") + } + } + + if cfg.Chain.GenesisValidatorsRoot == "" { + switch cfg.Chain.Name { + case "mainnet": + cfg.Chain.GenesisValidatorsRoot = "0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95" + case "prater": + cfg.Chain.GenesisValidatorsRoot = "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb" + case "sepolia": + cfg.Chain.GenesisValidatorsRoot = "0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078" + case "zhejiang": + cfg.Chain.GenesisValidatorsRoot = "0x53a92d8f2bb1d85f62d16a156e6ebcd1bcaba652d0900b2c2f387826f3481f6f" + case "gnosis": + cfg.Chain.GenesisValidatorsRoot = "0xf5dcb5564e829aab27264b9becd5dfaa017085611224cb3036f573368dbb9d47" + case "holesky": + cfg.Chain.GenesisValidatorsRoot = "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1" + case "hoodi": + cfg.Chain.GenesisValidatorsRoot = "0x212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f" + default: + return fmt.Errorf("tried to set known genesis-validators-root, but unknown chain-name") + } + } + + if cfg.Chain.DomainBLSToExecutionChange == "" { + cfg.Chain.DomainBLSToExecutionChange = "0x0A000000" + } + if cfg.Chain.DomainVoluntaryExit == "" { + cfg.Chain.DomainVoluntaryExit = "0x04000000" + } + + if cfg.Frontend.ClCurrency == "" { + switch cfg.Chain.Name { + case "gnosis": + cfg.Frontend.MainCurrency = "GNO" + cfg.Frontend.ClCurrency = "mGNO" + cfg.Frontend.ClCurrencyDecimals = 18 + cfg.Frontend.ClCurrencyDivisor = 1e9 + default: + cfg.Frontend.MainCurrency = "ETH" + cfg.Frontend.ClCurrency = "ETH" + cfg.Frontend.ClCurrencyDecimals = 18 + cfg.Frontend.ClCurrencyDivisor = 1e9 + } + } + + if cfg.Frontend.ElCurrency == "" { + switch cfg.Chain.Name { + case "gnosis": + cfg.Frontend.ElCurrency = "xDAI" + cfg.Frontend.ElCurrencyDecimals = 18 + cfg.Frontend.ElCurrencyDivisor = 1e18 + default: + cfg.Frontend.ElCurrency = "ETH" + cfg.Frontend.ElCurrencyDecimals = 18 + cfg.Frontend.ElCurrencyDivisor = 1e18 + } + } + + if cfg.Frontend.SiteTitle == "" { + cfg.Frontend.SiteTitle = "Open Source Ethereum Explorer" + } + + if cfg.Frontend.Keywords == "" { + cfg.Frontend.Keywords = "open source ethereum block explorer, ethereum block explorer, beacon chain explorer, ethereum blockchain explorer" + } + + if cfg.Frontend.Ratelimits.FreeDay == 0 { + cfg.Frontend.Ratelimits.FreeDay = 30000 + } + if cfg.Frontend.Ratelimits.FreeMonth == 0 { + cfg.Frontend.Ratelimits.FreeMonth = 30000 + } + if cfg.Frontend.Ratelimits.SapphierDay == 0 { + cfg.Frontend.Ratelimits.SapphierDay = 100000 + } + if cfg.Frontend.Ratelimits.SapphierMonth == 0 { + cfg.Frontend.Ratelimits.SapphierMonth = 500000 + } + if cfg.Frontend.Ratelimits.EmeraldDay == 0 { + cfg.Frontend.Ratelimits.EmeraldDay = 200000 + } + if cfg.Frontend.Ratelimits.EmeraldMonth == 0 { + cfg.Frontend.Ratelimits.EmeraldMonth = 1000000 + } + if cfg.Frontend.Ratelimits.DiamondDay == 0 { + cfg.Frontend.Ratelimits.DiamondDay = 6000000 + } + if cfg.Frontend.Ratelimits.DiamondMonth == 0 { + cfg.Frontend.Ratelimits.DiamondMonth = 6000000 + } + + if cfg.Chain.Id != 0 { + switch cfg.Chain.Name { + case "mainnet", "ethereum": + cfg.Chain.Id = 1 + case "prater", "goerli": + cfg.Chain.Id = 5 + case "holesky": + cfg.Chain.Id = 17000 + case "hoodi": + cfg.Chain.Id = 560048 + case "sepolia": + cfg.Chain.Id = 11155111 + case "gnosis": + cfg.Chain.Id = 100 + } + } + + // we check for machine chain id just for safety + if cfg.Chain.Id != 0 && cfg.Chain.Id != cfg.Chain.ClConfig.DepositChainID { + log.Fatal(fmt.Errorf("cfg.Chain.Id != cfg.Chain.ClConfig.DepositChainID: %v != %v", cfg.Chain.Id, cfg.Chain.ClConfig.DepositChainID), "", 0) + } + + cfg.Chain.Id = cfg.Chain.ClConfig.DepositChainID + + if cfg.RedisSessionStoreEndpoint == "" && cfg.RedisCacheEndpoint != "" { + log.Warnf("using RedisCacheEndpoint %s as RedisSessionStoreEndpoint as no dedicated RedisSessionStoreEndpoint was provided", cfg.RedisCacheEndpoint) + cfg.RedisSessionStoreEndpoint = cfg.RedisCacheEndpoint + } + + if cfg.Bigtable.TableNameBeaconchain == "" { + cfg.Bigtable.TableNameBeaconchain = "beaconchain" + } + if cfg.Bigtable.TableNameValidators == "" { + cfg.Bigtable.TableNameValidators = "beaconchain_validators" + } + if cfg.Bigtable.TableNameValidatorsHistory == "" { + cfg.Bigtable.TableNameValidatorsHistory = "beaconchain_validators_history" + } + if cfg.Bigtable.TableNameBlocks == "" { + cfg.Bigtable.TableNameBlocks = "blocks" + } + if cfg.Bigtable.TableNameData == "" { + cfg.Bigtable.TableNameData = "data" + } + if cfg.Bigtable.TableNameMachineMetrics == "" { + cfg.Bigtable.TableNameMachineMetrics = "machine_metrics" + } + if cfg.Bigtable.TableNameMetadata == "" { + cfg.Bigtable.TableNameMetadata = "metadata" + } + if cfg.Bigtable.TableNameMetadataUpdates == "" { + cfg.Bigtable.TableNameMetadataUpdates = "metadata_updates" + } + if cfg.Bigtable.TableNameBlocksRaw == "" { + cfg.Bigtable.TableNameBlocksRaw = "blocks_raw" + } + + confSanityCheck(cfg) + + log.InfoWithFields(log.Fields{ + "genesisTimestamp": cfg.Chain.GenesisTimestamp, + "genesisValidatorsRoot": cfg.Chain.GenesisValidatorsRoot, + "configName": cfg.Chain.ClConfig.ConfigName, + "depositChainID": cfg.Chain.ClConfig.DepositChainID, + "depositNetworkID": cfg.Chain.ClConfig.DepositNetworkID, + "depositContractAddress": cfg.Chain.ClConfig.DepositContractAddress, + "clCurrency": cfg.Frontend.ClCurrency, + "elCurrency": cfg.Frontend.ElCurrency, + "mainCurrency": cfg.Frontend.MainCurrency, + }, "did init config") + + Config = cfg + return nil +} + +func setELConfig(cfg *types.Config) error { + var err error + type MinimalELConfig struct { + ByzantiumBlock *big.Int `yaml:"BYZANTIUM_FORK_BLOCK,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ConstantinopleBlock *big.Int `yaml:"CONSTANTINOPLE_FORK_BLOCK,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) + } + if cfg.Chain.ElConfigPath == "" { + minimalCfg := MinimalELConfig{} + switch cfg.Chain.Name { + case "mainnet": + err = yaml.Unmarshal([]byte(config.MainnetChainYml), &minimalCfg) + case "prater": + err = yaml.Unmarshal([]byte(config.PraterChainYml), &minimalCfg) + case "ropsten": + err = yaml.Unmarshal([]byte(config.RopstenChainYml), &minimalCfg) + case "sepolia": + err = yaml.Unmarshal([]byte(config.SepoliaChainYml), &minimalCfg) + case "gnosis": + err = yaml.Unmarshal([]byte(config.GnosisChainYml), &minimalCfg) + case "holesky": + err = yaml.Unmarshal([]byte(config.HoleskyChainYml), &minimalCfg) + case "hoodi": + err = yaml.Unmarshal([]byte(config.HoodiChainYml), &minimalCfg) + case "mekong": + err = yaml.Unmarshal([]byte(config.MekongChainYml), &minimalCfg) + case "pectra-devnet-5": + err = yaml.Unmarshal([]byte(config.PectraDevnet5ChainYml), &minimalCfg) + case "pectra-devnet-6": + err = yaml.Unmarshal([]byte(config.PectraDevnet6ChainYml), &minimalCfg) + default: + return fmt.Errorf("tried to set known chain-config, but unknown chain-name: %v (path: %v)", cfg.Chain.Name, cfg.Chain.ElConfigPath) + } + if err != nil { + return err + } + if minimalCfg.ByzantiumBlock == nil { + minimalCfg.ByzantiumBlock = big.NewInt(0) + } + if minimalCfg.ConstantinopleBlock == nil { + minimalCfg.ConstantinopleBlock = big.NewInt(0) + } + cfg.Chain.ElConfig = ¶ms.ChainConfig{ + ChainID: big.NewInt(int64(cfg.Chain.Id)), + ByzantiumBlock: minimalCfg.ByzantiumBlock, + ConstantinopleBlock: minimalCfg.ConstantinopleBlock, + } + } else { + f, err := os.Open(cfg.Chain.ElConfigPath) + if err != nil { + return fmt.Errorf("error opening EL Chain Config file %v: %w", cfg.Chain.ElConfigPath, err) + } + var chainConfig *params.ChainConfig + decoder := json.NewDecoder(f) + err = decoder.Decode(&chainConfig) + if err != nil { + return fmt.Errorf("error decoding EL Chain Config file %v: %v", cfg.Chain.ElConfigPath, err) + } + cfg.Chain.ElConfig = chainConfig + } + return nil +} + +var MaxForkEpoch = uint64(18446744073709551615) // placeholder for unset / inactive fork. Can't be 0 to not confuse with genesis + +func setCLConfig(cfg *types.Config) error { + var err error + if cfg.Chain.ClConfigPath == "" { + // var prysmParamsConfig *prysmParams.BeaconChainConfig + switch cfg.Chain.Name { + case "mainnet": + err = yaml.Unmarshal([]byte(config.MainnetChainYml), &cfg.Chain.ClConfig) + case "prater": + err = yaml.Unmarshal([]byte(config.PraterChainYml), &cfg.Chain.ClConfig) + case "ropsten": + err = yaml.Unmarshal([]byte(config.RopstenChainYml), &cfg.Chain.ClConfig) + case "sepolia": + err = yaml.Unmarshal([]byte(config.SepoliaChainYml), &cfg.Chain.ClConfig) + case "gnosis": + err = yaml.Unmarshal([]byte(config.GnosisChainYml), &cfg.Chain.ClConfig) + case "holesky": + err = yaml.Unmarshal([]byte(config.HoleskyChainYml), &cfg.Chain.ClConfig) + case "hoodi": + err = yaml.Unmarshal([]byte(config.HoodiChainYml), &cfg.Chain.ClConfig) + case "pectra-devnet-5": + err = yaml.Unmarshal([]byte(config.PectraDevnet5ChainYml), &cfg.Chain.ClConfig) + case "pectra-devnet-6": + err = yaml.Unmarshal([]byte(config.PectraDevnet6ChainYml), &cfg.Chain.ClConfig) + default: + return fmt.Errorf("tried to set known chain-config, but unknown chain-name: %v (path: %v)", cfg.Chain.Name, cfg.Chain.ClConfigPath) + } + if err != nil { + return err + } + // err = prysmParams.SetActive(prysmParamsConfig) + // if err != nil { + // return fmt.Errorf("error setting chainConfig (%v) for prysmParams: %w", cfg.Chain.Name, err) + // } + } else if cfg.Chain.ClConfigPath == "node" { + nodeEndpoint := fmt.Sprintf("http://%s", net.JoinHostPort(cfg.Indexer.Node.Host, cfg.Indexer.Node.Port)) + client := consapi.NewClient(nodeEndpoint) + + jr, err := client.GetSpec() + if err != nil { + return err + } + + if jr.Data.AltairForkEpoch == nil { + log.Warnf("AltairForkEpoch not set, defaulting to maxForkEpoch") + jr.Data.AltairForkEpoch = &MaxForkEpoch + } + if jr.Data.BellatrixForkEpoch == nil { + log.Warnf("BellatrixForkEpoch not set, defaulting to maxForkEpoch") + jr.Data.BellatrixForkEpoch = &MaxForkEpoch + } + if jr.Data.CapellaForkEpoch == nil { + log.Warnf("CapellaForkEpoch not set, defaulting to maxForkEpoch") + jr.Data.CapellaForkEpoch = &MaxForkEpoch + } + if jr.Data.DenebForkEpoch == nil { + log.Warnf("DenebForkEpoch not set, defaulting to maxForkEpoch") + jr.Data.DenebForkEpoch = &MaxForkEpoch + } + if jr.Data.ElectraForkEpoch == nil { + log.Warnf("ElectraForkEpoch not set, defaulting to maxForkEpoch") + jr.Data.ElectraForkEpoch = &MaxForkEpoch + } + + chainCfg := types.ClChainConfig{ + PresetBase: jr.Data.PresetBase, + ConfigName: jr.Data.ConfigName, + TerminalTotalDifficulty: jr.Data.TerminalTotalDifficulty, + TerminalBlockHash: jr.Data.TerminalBlockHash, + TerminalBlockHashActivationEpoch: jr.Data.TerminalBlockHashActivationEpoch, + MinGenesisActiveValidatorCount: uint64(jr.Data.MinGenesisActiveValidatorCount), + MinGenesisTime: jr.Data.MinGenesisTime, + GenesisForkVersion: jr.Data.GenesisForkVersion, + GenesisDelay: uint64(jr.Data.GenesisDelay), + AltairForkVersion: jr.Data.AltairForkVersion, + AltairForkEpoch: *jr.Data.AltairForkEpoch, + BellatrixForkVersion: jr.Data.BellatrixForkVersion, + BellatrixForkEpoch: *jr.Data.BellatrixForkEpoch, + CapellaForkVersion: jr.Data.CapellaForkVersion, + CapellaForkEpoch: *jr.Data.CapellaForkEpoch, + DenebForkVersion: jr.Data.DenebForkVersion, + DenebForkEpoch: *jr.Data.DenebForkEpoch, + ElectraForkVersion: jr.Data.ElectraForkVersion, + ElectraForkEpoch: *jr.Data.ElectraForkEpoch, + SecondsPerSlot: uint64(jr.Data.SecondsPerSlot), + SecondsPerEth1Block: uint64(jr.Data.SecondsPerEth1Block), + MinValidatorWithdrawabilityDelay: uint64(jr.Data.MinValidatorWithdrawabilityDelay), + ShardCommitteePeriod: uint64(jr.Data.ShardCommitteePeriod), + Eth1FollowDistance: uint64(jr.Data.Eth1FollowDistance), + InactivityScoreBias: uint64(jr.Data.InactivityScoreBias), + InactivityScoreRecoveryRate: uint64(jr.Data.InactivityScoreRecoveryRate), + EjectionBalance: uint64(jr.Data.EjectionBalance), + MinPerEpochChurnLimit: uint64(jr.Data.MinPerEpochChurnLimit), + ChurnLimitQuotient: uint64(jr.Data.ChurnLimitQuotient), + ProposerScoreBoost: uint64(jr.Data.ProposerScoreBoost), + DepositChainID: uint64(jr.Data.DepositChainID), + DepositNetworkID: uint64(jr.Data.DepositNetworkID), + DepositContractAddress: jr.Data.DepositContractAddress, + MaxCommitteesPerSlot: uint64(jr.Data.MaxCommitteesPerSlot), + TargetCommitteeSize: uint64(jr.Data.TargetCommitteeSize), + MaxValidatorsPerCommittee: uint64(jr.Data.MaxValidatorsPerCommittee), + ShuffleRoundCount: uint64(jr.Data.ShuffleRoundCount), + HysteresisQuotient: uint64(jr.Data.HysteresisQuotient), + HysteresisDownwardMultiplier: uint64(jr.Data.HysteresisDownwardMultiplier), + HysteresisUpwardMultiplier: uint64(jr.Data.HysteresisUpwardMultiplier), + SafeSlotsToUpdateJustified: uint64(jr.Data.SafeSlotsToUpdateJustified), + MinDepositAmount: uint64(jr.Data.MinDepositAmount), + MaxEffectiveBalance: uint64(jr.Data.MaxEffectiveBalance), + EffectiveBalanceIncrement: uint64(jr.Data.EffectiveBalanceIncrement), + MinAttestationInclusionDelay: uint64(jr.Data.MinAttestationInclusionDelay), + SlotsPerEpoch: uint64(jr.Data.SlotsPerEpoch), + MinSeedLookahead: uint64(jr.Data.MinSeedLookahead), + MaxSeedLookahead: uint64(jr.Data.MaxSeedLookahead), + EpochsPerEth1VotingPeriod: uint64(jr.Data.EpochsPerEth1VotingPeriod), + SlotsPerHistoricalRoot: uint64(jr.Data.SlotsPerHistoricalRoot), + MinEpochsToInactivityPenalty: uint64(jr.Data.MinEpochsToInactivityPenalty), + EpochsPerHistoricalVector: uint64(jr.Data.EpochsPerHistoricalVector), + EpochsPerSlashingsVector: uint64(jr.Data.EpochsPerSlashingsVector), + HistoricalRootsLimit: uint64(jr.Data.HistoricalRootsLimit), + ValidatorRegistryLimit: uint64(jr.Data.ValidatorRegistryLimit), + BaseRewardFactor: uint64(jr.Data.BaseRewardFactor), + WhistleblowerRewardQuotient: uint64(jr.Data.WhistleblowerRewardQuotient), + ProposerRewardQuotient: uint64(jr.Data.ProposerRewardQuotient), + InactivityPenaltyQuotient: uint64(jr.Data.InactivityPenaltyQuotient), + MinSlashingPenaltyQuotient: uint64(jr.Data.MinSlashingPenaltyQuotient), + ProportionalSlashingMultiplier: uint64(jr.Data.ProportionalSlashingMultiplier), + MaxProposerSlashings: uint64(jr.Data.MaxProposerSlashings), + MaxAttesterSlashings: uint64(jr.Data.MaxAttesterSlashings), + MaxAttestations: uint64(jr.Data.MaxAttestations), + MaxDeposits: uint64(jr.Data.MaxDeposits), + MaxVoluntaryExits: uint64(jr.Data.MaxVoluntaryExits), + InvactivityPenaltyQuotientAltair: uint64(jr.Data.InactivityPenaltyQuotientAltair), + MinSlashingPenaltyQuotientAltair: uint64(jr.Data.MinSlashingPenaltyQuotientAltair), + ProportionalSlashingMultiplierAltair: uint64(jr.Data.ProportionalSlashingMultiplierAltair), + SyncCommitteeSize: uint64(jr.Data.SyncCommitteeSize), + EpochsPerSyncCommitteePeriod: uint64(jr.Data.EpochsPerSyncCommitteePeriod), + MinSyncCommitteeParticipants: uint64(jr.Data.MinSyncCommitteeParticipants), + InvactivityPenaltyQuotientBellatrix: uint64(jr.Data.InactivityPenaltyQuotientBellatrix), + MinSlashingPenaltyQuotientBellatrix: uint64(jr.Data.MinSlashingPenaltyQuotientBellatrix), + ProportionalSlashingMultiplierBellatrix: uint64(jr.Data.ProportionalSlashingMultiplierBellatrix), + MaxBytesPerTransaction: uint64(jr.Data.MaxBytesPerTransaction), + MaxTransactionsPerPayload: uint64(jr.Data.MaxTransactionsPerPayload), + BytesPerLogsBloom: uint64(jr.Data.BytesPerLogsBloom), + MaxExtraDataBytes: uint64(jr.Data.MaxExtraDataBytes), + MaxWithdrawalsPerPayload: uint64(jr.Data.MaxWithdrawalsPerPayload), + MaxValidatorsPerWithdrawalSweep: uint64(jr.Data.MaxValidatorsPerWithdrawalsSweep), + MaxBlsToExecutionChange: uint64(jr.Data.MaxBlsToExecutionChanges), + MaxEffectiveBalanceElectra: uint64(jr.Data.MaxEffectiveBalanceElectra), + MinPerEpochChurnLimitElectra: uint64(jr.Data.MinPerEpochChurnLimitElectra), + MaxPerEpochActivationExitChurnLimit: uint64(jr.Data.MaxPerEpochActivationExitChurnLimit), + BlobSidecarSubnetCountElectra: uint64(jr.Data.BlobSidecarSubnetCountElectra), + MaxBlobsPerBlockElectra: uint64(jr.Data.MaxBlobsPerBlockElectra), + MaxRequestBlobSidecarsElectra: uint64(jr.Data.MaxRequestBlobSidecarsElectra), + MinActivationBalance: uint64(jr.Data.MinActivationBalance), + MaxPendingDepositsPerEpoch: uint64(jr.Data.MaxPendingDepositsPerEpoch), + } + + cfg.Chain.ClConfig = chainCfg + + gtr, err := client.GetGenesis() + if err != nil { + return err + } + + cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) + cfg.Chain.GenesisValidatorsRoot = gtr.Data.GenesisValidatorsRoot + + log.Infof("loaded chain config from node with genesis time %s", gtr.Data.GenesisTime) + } else { + f, err := os.Open(cfg.Chain.ClConfigPath) + if err != nil { + return fmt.Errorf("error opening Chain Config file %v: %w", cfg.Chain.ClConfigPath, err) + } + var chainConfig *types.ClChainConfig + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&chainConfig) + if err != nil { + return fmt.Errorf("error decoding Chain Config file %v: %v", cfg.Chain.ClConfigPath, err) + } + cfg.Chain.ClConfig = *chainConfig + } + + return nil +} diff --git a/docker_images/seedemu-ethexplorer/beaconchain/backend/start.sh b/docker_images/seedemu-ethexplorer/beaconchain/backend/start.sh new file mode 100644 index 000000000..b391ce383 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/beaconchain/backend/start.sh @@ -0,0 +1,35 @@ +#! /bin/bash + +stopit() { + echo "trapped SIGINT" + exit 0 +} + +trap stopit SIGINT + +echo "run provision-explorer-config-custom.sh ..." +cd /app/local_deployment +chmod +x provision-explorer-config-custom.sh +./provision-explorer-config-custom.sh + +if [ $? -eq 0 ]; then + echo "SUCCESS: script executed normally" +else + echo "ERROR: script failed" + exit 1 +fi + +cd /app +echo "start ..." +echo "run eth1indexer" +bc eth1indexer -config /app/local_deployment/config.yml -concurrency 1 -blocks.tracemode 'geth' & +echo "run rewards-exporter" +bc rewards-exporter -config /app/local_deployment/config.yml & +echo "run statistics" +bc statistics -config /app/local_deployment/config.yml --charts.enabled --graffiti.enabled -validators.enabled -deposits.enabled & +echo "end." + +while true; do + date + sleep 1 +done diff --git a/docker_images/seedemu-ethexplorer/docker-compose.yml b/docker_images/seedemu-ethexplorer/docker-compose.yml new file mode 100644 index 000000000..e8a0aa34c --- /dev/null +++ b/docker_images/seedemu-ethexplorer/docker-compose.yml @@ -0,0 +1,26 @@ +services: + seedemu-ethexplorer-backend: + build: + context: beaconchain/backend + image: handsonsecurity/seedemu-ethexplorer-backend:1.0 + container_name: seedemu_beaconchain_backend + cap_add: + - ALL + pid: host + privileged: true + + seedemu-ethexplorer-web: + build: + context: eth2-beaconchain-explorer/ + image: handsonsecurity/seedemu-ethexplorer-web:1.0 + container_name: seedemu_eth2_beaconchain_explorer + cap_add: + - ALL + pid: host + privileged: true + ports: + - 5000:5000 + +networks: + beacon-network: + driver: bridge \ No newline at end of file diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/Dockerfile b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/Dockerfile new file mode 100644 index 000000000..bea829c71 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1.7 + +# -------------------------- +# Stage 0: Node runtime +# -------------------------- +FROM node:24-bookworm AS node-env +# -------------------------- +# Stage 1: Build Go binaries +# -------------------------- +FROM golang:1.23-bookworm AS build-env + +COPY --from=node-env /usr/local /usr/local +WORKDIR / +RUN git clone https://github.com/gobitfly/eth2-beaconchain-explorer.git +WORKDIR /eth2-beaconchain-explorer +COPY . . + +RUN --mount=type=cache,target=/root/.npm \ + npm ci + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +ARG target=all + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + git config --global --add safe.directory '*' && \ + make -B "${target}" +# -------------------------- +# Stage 2: Runtime image +# -------------------------- +FROM debian:bookworm-slim AS runtime + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3 \ + curl \ + jq \ + && rm -rf /var/lib/apt/lists/* + +ENV POSTGRES_PORT=$POSTGRES_PORT \ + ALLOY_PORT=$ALLOY_PORT \ + LBT_PORT=$LBT_PORT \ + RBT_PORT=$RBT_PORT \ + EL_PORT=$EL_PORT \ + CL_PORT=$CL_PORT \ + REDIS_PORT=$REDIS_PORT \ + REDIS_SESSIONS_PORT=$REDIS_SESSIONS_PORT \ + SERVER_HOST=$SERVER_HOST + +WORKDIR /app + +RUN mkdir -p /app/local-deployment + +COPY --from=build-env /eth2-beaconchain-explorer/bin/explorer /app/ +COPY --from=build-env /eth2-beaconchain-explorer/bin/validator-tagger /app/ +COPY --from=build-env /eth2-beaconchain-explorer/bin/frontend-data-updater /app/ +COPY --from=build-env /eth2-beaconchain-explorer/config/ /app/config/ + +COPY local-deployment/provision-explorer-config-custom.sh \ + local-deployment/mainnet.chain.yml \ + /app/local-deployment/ + +COPY start.sh /start.sh + +RUN chmod +x /start.sh \ + && chmod +x /app/local-deployment/provision-explorer-config-custom.sh + +ENTRYPOINT ["/start.sh"] \ No newline at end of file diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/clickhouse_integration_test/main.go b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/clickhouse_integration_test/main.go new file mode 100644 index 000000000..5ce78e8b8 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/clickhouse_integration_test/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/gobitfly/eth2-beaconchain-explorer/db" + "github.com/gobitfly/eth2-beaconchain-explorer/types" + + itypes "github.com/gobitfly/eth-rewards/types" + "github.com/gobitfly/eth2-beaconchain-explorer/utils" + "github.com/gobitfly/eth2-beaconchain-explorer/version" + "github.com/sirupsen/logrus" +) + +func main() { + configPath := flag.String("config", "config/default.config.yml", "Path to the config file") + + flag.Parse() + + cfg := &types.Config{} + err := utils.ReadConfig(cfg, *configPath) + if err != nil { + logrus.Fatalf("error reading config file: %v", err) + } + utils.Config = cfg + logrus.WithFields(logrus.Fields{ + "config": *configPath, + "version": version.Version, + "chainName": utils.Config.Chain.ClConfig.ConfigName}).Printf("starting") + + db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.WriterDatabase.Username, + Password: cfg.WriterDatabase.Password, + Name: cfg.WriterDatabase.Name, + Host: cfg.WriterDatabase.Host, + Port: cfg.WriterDatabase.Port, + MaxOpenConns: cfg.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.WriterDatabase.MaxIdleConns, + SSL: cfg.WriterDatabase.SSL, + }, &types.DatabaseConfig{ + Username: cfg.ReaderDatabase.Username, + Password: cfg.ReaderDatabase.Password, + Name: cfg.ReaderDatabase.Name, + Host: cfg.ReaderDatabase.Host, + Port: cfg.ReaderDatabase.Port, + MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns, + SSL: cfg.ReaderDatabase.SSL, + }, "pgx", "postgres") + + db.MustInitClickhouseDB(nil, &types.DatabaseConfig{ + Username: cfg.ClickHouse.ReaderDatabase.Username, + Password: cfg.ClickHouse.ReaderDatabase.Password, + Name: cfg.ClickHouse.ReaderDatabase.Name, + Host: cfg.ClickHouse.ReaderDatabase.Host, + Port: cfg.ClickHouse.ReaderDatabase.Port, + MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + SSL: cfg.ClickHouse.ReaderDatabase.SSL, + // SSL: true, + }, "clickhouse", "clickhouse") + + bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) + if err != nil { + logrus.Fatalf("error connecting to bigtable: %v", err) + } + db.BigtableClient = bt + + utils.Config.ClickhouseDelay = 0 + + // verification funcs are to be run against mainnet + // normal attestation + logrus.Infof("verifying history for normal attestations") + verifyHistory([]uint64{653161}, uint64(323260), uint64(323270), true, true) + + // income details for sync rewards (this currently fails !!!) + logrus.Infof("verifying history for sync rewards") + verifyHistory([]uint64{653162}, uint64(323260), uint64(323261), true, true) + + // block proposed + logrus.Infof("verifying history for proposer rewards at end of an epoch") + verifyHistory([]uint64{388033}, uint64(323268), uint64(323270), true, true) + + logrus.Infof("verifying history for proposer rewards at start of an epoch") + verifyHistory([]uint64{1284148}, uint64(323268), uint64(323270), true, true) + + logrus.Infof("verifying history for proposer rewards at during an epoch") + verifyHistory([]uint64{1208852}, uint64(323268), uint64(323270), true, true) + + // missed attestations + logrus.Infof("verifying history for missed attestations") + verifyHistory([]uint64{76040}, uint64(323260), uint64(323270), true, true) + + // missed slots + logrus.Infof("verifying history for missed slots") + verifyHistory([]uint64{858473}, uint64(323250), uint64(323260), true, true) + + // slashing (5902 was slashed by 792015) + logrus.Infof("verifying history for a slashed validator") + verifyHistory([]uint64{5902}, uint64(314635), uint64(314645), true, true) + logrus.Infof("verifying history for a slashing validator") + verifyHistory([]uint64{792015}, uint64(314635), uint64(314645), true, true) + + // validator during activation + logrus.Infof("verifying history for a validator during activation") + verifyHistory([]uint64{894572}, uint64(266960), uint64(266970), true, true) + + // validator during exit + logrus.Infof("verifying history for a validator during exit") + verifyHistory([]uint64{1646687}, uint64(323090), uint64(323110), true, true) + +} + +func verifyHistory(validatorIndices []uint64, epochStart, epochEnd uint64, income, balance bool) { + slotsPerEpoch := utils.Config.Chain.ClConfig.SlotsPerEpoch + logrus.Infof("verifying history for validator indices %v from epoch %d to %d", validatorIndices, epochStart, epochEnd) + if income { + compare(db.BigtableClient.GetValidatorIncomeDetailsHistory, validatorIndices, epochStart, epochEnd) + } + if balance { + compare(db.BigtableClient.GetValidatorBalanceHistory, validatorIndices, epochStart, epochEnd) + } + compare(db.BigtableClient.GetValidatorAttestationHistory, validatorIndices, epochStart, epochEnd) + compare(db.BigtableClient.GetValidatorMissedAttestationHistory, validatorIndices, epochStart, epochEnd) + compare(db.BigtableClient.GetValidatorSyncDutiesHistory, validatorIndices, epochStart*slotsPerEpoch, epochEnd*slotsPerEpoch) +} + +type GenericFunc[T any] func([]uint64, uint64, uint64) (T, error) + +func compare[T any](compareFunc GenericFunc[T], validatorIndices []uint64, epochStart, epochEnd uint64) { + utils.Config.ClickHouseEnabled = false + bigtableData, err := compareFunc(validatorIndices, epochStart, epochEnd) + if err != nil { + logrus.Fatalf("error getting validator income details history from bigtable: %v", err) + } + utils.Config.ClickHouseEnabled = true + clickhouseData, err := compareFunc(validatorIndices, epochStart, epochEnd) + if err != nil { + logrus.Fatalf("error getting validator income details history from clickhouse: %v", err) + } + diff := cmp.Diff(bigtableData, clickhouseData, cmpopts.IgnoreUnexported(itypes.ValidatorEpochIncome{})) + if diff != "" { + logrus.Infof("bigtable") + spew.Dump(bigtableData) + logrus.Infof("clickhouse") + spew.Dump(clickhouseData) + logrus.Info(diff) + logrus.Fatalf("bigtable and clickhouse data do not match") + } +} diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/explorer/main.go b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/explorer/main.go new file mode 100644 index 000000000..6bd03dd6d --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/explorer/main.go @@ -0,0 +1,695 @@ +package main + +import ( + "context" + "encoding/gob" + "encoding/hex" + "errors" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "strings" + "sync" + "time" + + "github.com/gobitfly/eth2-beaconchain-explorer/cache" + "github.com/gobitfly/eth2-beaconchain-explorer/db" + ethclients "github.com/gobitfly/eth2-beaconchain-explorer/ethClients" + "github.com/gobitfly/eth2-beaconchain-explorer/exporter" + "github.com/gobitfly/eth2-beaconchain-explorer/handlers" + "github.com/gobitfly/eth2-beaconchain-explorer/metrics" + "github.com/gobitfly/eth2-beaconchain-explorer/price" + "github.com/gobitfly/eth2-beaconchain-explorer/rpc" + "github.com/gobitfly/eth2-beaconchain-explorer/services" + "github.com/gobitfly/eth2-beaconchain-explorer/static" + "github.com/gobitfly/eth2-beaconchain-explorer/types" + "github.com/gobitfly/eth2-beaconchain-explorer/utils" + "github.com/gobitfly/eth2-beaconchain-explorer/version" + + "github.com/sirupsen/logrus" + + _ "net/http/pprof" + + "github.com/gorilla/csrf" + "github.com/gorilla/mux" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/phyber/negroni-gzip/gzip" + "github.com/stripe/stripe-go/v72" + "github.com/urfave/negroni" + "github.com/zesik/proxyaddr" +) + +func initStripe(http *mux.Router) error { + if utils.Config == nil { + return fmt.Errorf("error no config found") + } + stripe.Key = utils.Config.Frontend.Stripe.SecretKey + http.HandleFunc("/stripe/create-checkout-session", handlers.StripeCreateCheckoutSession).Methods("POST", "OPTIONS") + http.HandleFunc("/stripe/customer-portal", handlers.StripeCustomerPortal).Methods("POST", "OPTIONS") + return nil +} + +func init() { + gob.Register(types.DataTableSaveState{}) + gob.Register(types.UserV1Notification(0)) +} + +var frontendHttpServer *http.Server + +func main() { + configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used") + versionFlag := flag.Bool("version", false, "Show version and exit") + flag.Parse() + + if *versionFlag { + fmt.Println(version.Version) + fmt.Println(version.GoVersion) + return + } + log.SetFlags(log.LstdFlags | log.Lshortfile) + + cfg := &types.Config{} + err := utils.ReadConfig(cfg, *configPath) + if err != nil { + logrus.Fatalf("error reading config file: %v", err) + } + utils.Config = cfg + logrus.WithFields(logrus.Fields{ + "config": *configPath, + "version": version.Version, + "chainName": utils.Config.Chain.ClConfig.ConfigName}).Printf("starting") + + if utils.Config.Chain.ClConfig.SlotsPerEpoch == 0 || utils.Config.Chain.ClConfig.SecondsPerSlot == 0 { + utils.LogFatal(err, "invalid chain configuration specified, you must specify the slots per epoch, seconds per slot and genesis timestamp in the config file", 0) + } + + if utils.Config.Pprof.Enabled { + go func() { + logrus.Infof("starting pprof http server on port %s", utils.Config.Pprof.Port) + logrus.Info(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%s", utils.Config.Pprof.Port), nil)) + }() + } + + wg := &sync.WaitGroup{} + + wg.Add(1) + go func() { + defer wg.Done() + db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.WriterDatabase.Username, + Password: cfg.WriterDatabase.Password, + Name: cfg.WriterDatabase.Name, + Host: cfg.WriterDatabase.Host, + Port: cfg.WriterDatabase.Port, + MaxOpenConns: cfg.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.WriterDatabase.MaxIdleConns, + SSL: cfg.WriterDatabase.SSL, + }, &types.DatabaseConfig{ + Username: cfg.ReaderDatabase.Username, + Password: cfg.ReaderDatabase.Password, + Name: cfg.ReaderDatabase.Name, + Host: cfg.ReaderDatabase.Host, + Port: cfg.ReaderDatabase.Port, + MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns, + SSL: cfg.ReaderDatabase.SSL, + }, "pgx", "postgres") + }() + + wg.Add(1) + go func() { + defer wg.Done() + db.MustInitFrontendDB(&types.DatabaseConfig{ + Username: cfg.Frontend.WriterDatabase.Username, + Password: cfg.Frontend.WriterDatabase.Password, + Name: cfg.Frontend.WriterDatabase.Name, + Host: cfg.Frontend.WriterDatabase.Host, + Port: cfg.Frontend.WriterDatabase.Port, + MaxOpenConns: cfg.Frontend.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.Frontend.WriterDatabase.MaxIdleConns, + SSL: cfg.Frontend.WriterDatabase.SSL, + }, &types.DatabaseConfig{ + Username: cfg.Frontend.ReaderDatabase.Username, + Password: cfg.Frontend.ReaderDatabase.Password, + Name: cfg.Frontend.ReaderDatabase.Name, + Host: cfg.Frontend.ReaderDatabase.Host, + Port: cfg.Frontend.ReaderDatabase.Port, + MaxOpenConns: cfg.Frontend.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.Frontend.ReaderDatabase.MaxIdleConns, + SSL: cfg.Frontend.ReaderDatabase.SSL, + }, "pgx", "postgres") + }() + + if utils.Config.ClickHouseEnabled { + wg.Add(1) + go func() { + defer wg.Done() + db.MustInitClickhouseDB(nil, &types.DatabaseConfig{ + Username: cfg.ClickHouse.ReaderDatabase.Username, + Password: cfg.ClickHouse.ReaderDatabase.Password, + Name: cfg.ClickHouse.ReaderDatabase.Name, + Host: cfg.ClickHouse.ReaderDatabase.Host, + Port: cfg.ClickHouse.ReaderDatabase.Port, + MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + SSL: cfg.ClickHouse.ReaderDatabase.SSL, + // SSL: true, + }, "clickhouse", "clickhouse") + }() + } + + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + rpc.CurrentErigonClient, err = rpc.NewErigonClient(utils.Config.Eth1ErigonEndpoint) + if err != nil { + logrus.Fatalf("error initializing erigon client: %v", err) + } + + erigonChainId, err := rpc.CurrentErigonClient.GetNativeClient().ChainID(ctx) + if err != nil { + logrus.Fatalf("error retrieving erigon chain id: %v", err) + } + + rpc.CurrentGethClient, err = rpc.NewGethClient(utils.Config.Eth1GethEndpoint) + if err != nil { + logrus.Fatalf("error initializing geth client: %v", err) + } + + gethChainId, err := rpc.CurrentGethClient.GetNativeClient().ChainID(ctx) + if err != nil { + logrus.Fatalf("error retrieving geth chain id: %v", err) + } + + if !(erigonChainId.String() == gethChainId.String() && erigonChainId.String() == fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) { + logrus.Fatalf("chain id mismatch: erigon chain id %v, geth chain id %v, requested chain id %v", erigonChainId.String(), gethChainId.String(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) + if err != nil { + logrus.Fatalf("error connecting to bigtable: %v", err) + } + db.BigtableClient = bt + }() + + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { + wg.Add(1) + go func() { + defer wg.Done() + cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) + logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) + + }() + } + + wg.Wait() + + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis)") + } + + defer db.ReaderDb.Close() + defer db.WriterDb.Close() + defer db.FrontendReaderDB.Close() + defer db.FrontendWriterDB.Close() + defer db.BigtableClient.Close() + + if utils.Config.Metrics.Enabled { + go metrics.MonitorDB(db.WriterDb) + DBInfo := []string{ + cfg.WriterDatabase.Username, + cfg.WriterDatabase.Password, + cfg.WriterDatabase.Host, + cfg.WriterDatabase.Port, + cfg.WriterDatabase.Name} + DBStr := strings.Join(DBInfo, "-") + frontendDBInfo := []string{ + cfg.Frontend.WriterDatabase.Username, + cfg.Frontend.WriterDatabase.Password, + cfg.Frontend.WriterDatabase.Host, + cfg.Frontend.WriterDatabase.Port, + cfg.Frontend.WriterDatabase.Name} + frontendDBStr := strings.Join(frontendDBInfo, "-") + if DBStr != frontendDBStr { + go metrics.MonitorDB(db.FrontendWriterDB) + } + } + + logrus.Infof("database connection established") + + if utils.Config.Indexer.Enabled { + var rpcClient rpc.Client + + chainID := new(big.Int).SetUint64(utils.Config.Chain.ClConfig.DepositChainID) + if utils.Config.Indexer.Node.Type == "lighthouse" { + rpcClient, err = rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainID) + if err != nil { + utils.LogFatal(err, "new explorer lighthouse client error", 0) + } + } else { + logrus.Fatalf("invalid note type %v specified. supported node types are prysm and lighthouse", utils.Config.Indexer.Node.Type) + } + + go services.StartHistoricPriceService() + go exporter.Start(rpcClient) + } + + if cfg.Frontend.Enabled { + + if cfg.Frontend.OnlyAPI { + services.ReportStatus("api", "Running", nil) + } else { + services.ReportStatus("frontend", "Running", nil) + } + + router := mux.NewRouter() + + apiV1Router := router.PathPrefix("/api/v1").Subrouter() + apiV1Router.HandleFunc("/docs", handlers.ApiDocs).Methods("GET") + apiV1Router.HandleFunc("/latestState", handlers.ApiLatestState).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/epoch/{epoch}", handlers.ApiEpoch).Methods("GET", "OPTIONS") + + apiV1Router.HandleFunc("/epoch/{epoch}/blocks", handlers.ApiEpochSlots).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/epoch/{epoch}/slots", handlers.ApiEpochSlots).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slotOrHash}", handlers.ApiSlots).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/attestations", handlers.ApiSlotAttestations).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/deposits", handlers.ApiSlotDeposits).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/attesterslashings", handlers.ApiSlotAttesterSlashings).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/proposerslashings", handlers.ApiSlotProposerSlashings).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/voluntaryexits", handlers.ApiSlotVoluntaryExits).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/withdrawals", handlers.ApiSlotWithdrawals).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/consolidation_requests", handlers.ApiSlotConsolidationRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/switch_to_compounding_requests", handlers.ApiSlotSwitchToCompoundingRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/slot/{slot}/deposit_requests", handlers.ApiSlotDepositRequests).Methods("GET", "OPTIONS") + + // deprecated, use slot equivalents + apiV1Router.HandleFunc("/block/{slotOrHash}", handlers.ApiSlots).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/attestations", handlers.ApiSlotAttestations).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/deposits", handlers.ApiSlotDeposits).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/attesterslashings", handlers.ApiSlotAttesterSlashings).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/proposerslashings", handlers.ApiSlotProposerSlashings).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/voluntaryexits", handlers.ApiSlotVoluntaryExits).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/consolidation_requests", handlers.ApiSlotConsolidationRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/switch_to_compounding_requests", handlers.ApiSlotSwitchToCompoundingRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/block/{slot}/deposit_requests", handlers.ApiSlotDepositRequests).Methods("GET", "OPTIONS") + + apiV1Router.HandleFunc("/sync_committee/{period}", handlers.ApiSyncCommittee).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/eth1deposit/{txhash}", handlers.ApiEth1Deposit).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/leaderboard", handlers.ApiValidatorLeaderboard).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}", handlers.ApiValidatorGet).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator", handlers.ApiValidatorPost).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/withdrawals", handlers.ApiValidatorWithdrawals).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/blsChange", handlers.ApiValidatorBlsChange).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/balancehistory", handlers.ApiValidatorBalanceHistory).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/incomedetailhistory", handlers.ApiValidatorIncomeDetailsHistory).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/performance", handlers.ApiValidatorPerformance).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/execution/performance", handlers.ApiValidatorExecutionPerformance).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/attestations", handlers.ApiValidatorAttestations).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/proposals", handlers.ApiValidatorProposals).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/consolidation_requests", handlers.ApiValidatorConsolidationRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/switch_to_compounding_requests", handlers.ApiValidatorSwitchToCompoundingRequests).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/deposits", handlers.ApiValidatorDeposits).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/attestationefficiency", handlers.ApiValidatorAttestationEfficiency).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/attestationeffectiveness", handlers.ApiValidatorAttestationEffectiveness).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/stats/{index}", handlers.ApiValidatorDailyStats).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/eth1/{address}", handlers.ApiValidatorByEth1Address).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validator/withdrawalCredentials/{withdrawalCredentialsOrEth1address}", handlers.ApiWithdrawalCredentialsValidators).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validators/queue", handlers.ApiValidatorQueue).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validators/proposalLuck", handlers.ApiProposalLuck).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/graffitiwall", handlers.ApiGraffitiwall).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/chart/{chart}", handlers.ApiChart).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/user/token", handlers.APIGetToken).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/dashboard/data/allbalances", handlers.DashboardDataBalanceCombined).Methods("GET", "OPTIONS") // consensus & execution + apiV1Router.HandleFunc("/dashboard/data/proposals", handlers.DashboardDataProposals).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/stripe/webhook", handlers.StripeWebhook).Methods("POST") + apiV1Router.HandleFunc("/stats/{apiKey}/{machine}", handlers.ClientStatsPostOld).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/stats/{apiKey}", handlers.ClientStatsPostOld).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/client/metrics", handlers.ClientStatsPostNew).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/app/dashboard", handlers.ApiDashboard).Methods("POST", "OPTIONS") + apiV1Router.HandleFunc("/rocketpool/stats", handlers.ApiRocketpoolStats).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/rocketpool/validator/{indexOrPubkey}", handlers.ApiRocketpoolValidators).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/ethstore/{day}", handlers.ApiEthStoreDay).Methods("GET", "OPTIONS") + + apiV1Router.HandleFunc("/execution/gasnow", handlers.ApiEth1GasNowData).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/execution/block/{blockNumber}", handlers.ApiETH1ExecBlocks).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/execution/{addressIndexOrPubkey}/produced", handlers.ApiETH1AccountProducedBlocks).Methods("GET", "OPTIONS") + + apiV1Router.HandleFunc("/execution/address/{address}", handlers.ApiEth1Address).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/execution/address/{address}/erc20tokens", handlers.ApiEth1AddressERC20Tokens).Methods("GET", "OPTIONS") + + apiV1Router.HandleFunc("/validator/{indexOrPubkey}/widget", handlers.GetMobileWidgetStatsGet).Methods("GET") + apiV1Router.HandleFunc("/dashboard/widget", handlers.GetMobileWidgetStatsPost).Methods("POST") + apiV1Router.HandleFunc("/ens/lookup/{domain}", handlers.ResolveEnsDomain).Methods("GET", "OPTIONS") + apiV1Router.Use(utils.CORSMiddleware) + + apiV1AuthRouter := apiV1Router.PathPrefix("/user").Subrouter() + apiV1AuthRouter.HandleFunc("/mobile/notify/register", handlers.MobileNotificationUpdatePOST).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/mobile/settings", handlers.MobileDeviceSettings).Methods("GET", "OPTIONS") + apiV1AuthRouter.HandleFunc("/mobile/settings", handlers.MobileDeviceSettingsPOST).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/validator/saved", handlers.MobileTagedValidators).Methods("GET", "OPTIONS") + apiV1AuthRouter.HandleFunc("/subscription/register", handlers.RegisterMobileSubscriptions).Methods("POST", "OPTIONS") + + apiV1AuthRouter.HandleFunc("/validator/{pubkey}/add", handlers.UserValidatorWatchlistAdd).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/validator/{pubkey}/remove", handlers.UserValidatorWatchlistRemove).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/dashboard/save", handlers.UserDashboardWatchlistAdd).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/dashboard/remove", handlers.UserDashboardWatchlistRemove).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/notifications/bundled/subscribe", handlers.MultipleUsersNotificationsSubscribe).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/notifications/bundled/unsubscribe", handlers.MultipleUsersNotificationsUnsubscribe).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/notifications/subscribe", handlers.UserNotificationsSubscribe).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/notifications/unsubscribe", handlers.UserNotificationsUnsubscribe).Methods("POST", "OPTIONS") + apiV1AuthRouter.HandleFunc("/notifications", handlers.UserNotificationsSubscribed).Methods("POST", "GET", "OPTIONS") + apiV1AuthRouter.HandleFunc("/stats", handlers.ClientStats).Methods("GET", "OPTIONS") + apiV1AuthRouter.HandleFunc("/stats/{offset}/{limit}", handlers.ClientStats).Methods("GET", "OPTIONS") + apiV1AuthRouter.HandleFunc("/ethpool", handlers.RegisterEthpoolSubscription).Methods("POST", "OPTIONS") + + apiV1AuthRouter.Use(utils.CORSMiddleware) + apiV1AuthRouter.Use(utils.AuthorizedAPIMiddleware) + + router.HandleFunc("/api/healthz", handlers.ApiHealthz).Methods("GET", "HEAD") + router.HandleFunc("/api/healthz-loadbalancer", handlers.ApiHealthzLoadbalancer).Methods("GET", "HEAD") + + logrus.Infof("initializing prices") + price.Init(utils.Config.Chain.ClConfig.DepositChainID, utils.Config.Eth1ErigonEndpoint, utils.Config.Frontend.ClCurrency, utils.Config.Frontend.ElCurrency) + + logrus.Infof("prices initialized") + if !utils.Config.Frontend.Debug { + logrus.Infof("initializing ethclients") + ethclients.Init() + logrus.Infof("ethclients initialized") + } + + if cfg.Frontend.SessionSecret == "" { + logrus.Fatal("session secret is empty, please provide a secure random string.") + return + } + + utils.InitSessionStore(cfg.Frontend.SessionSecret) + + if !utils.Config.Frontend.OnlyAPI { + if utils.Config.Frontend.SiteDomain == "" { + utils.Config.Frontend.SiteDomain = "beaconcha.in" + } + + csrfBytes, err := hex.DecodeString(cfg.Frontend.CsrfAuthKey) + if err != nil { + logrus.WithError(err).Error("error decoding csrf auth key falling back to empty csrf key") + } + + csrfHandler := csrf.Protect( + csrfBytes, + csrf.FieldName("CsrfField"), + csrf.Secure(!cfg.Frontend.CsrfInsecure), + csrf.Path("/"), + // csrf.Domain(cfg.Frontend.SessionCookieDomain), + csrf.SameSite(csrf.SameSiteNoneMode), + ) + + router.HandleFunc("/", handlers.Index).Methods("GET") + router.HandleFunc("/latestState", handlers.LatestState).Methods("GET") + router.HandleFunc("/launchMetrics", handlers.SlotVizMetrics).Methods("GET") + router.HandleFunc("/index/data", handlers.IndexPageData).Methods("GET") + router.HandleFunc("/slot/{slotOrHash}", handlers.Slot).Methods("GET") + router.HandleFunc("/slot/{slotOrHash}/deposits", handlers.SlotDepositData).Methods("GET") + router.HandleFunc("/slot/{slotOrHash}/votes", handlers.SlotVoteData).Methods("GET") + router.HandleFunc("/slot/{slot}/attestations", handlers.SlotAttestationsData).Methods("GET") + router.HandleFunc("/slot/{slot}/withdrawals", handlers.SlotWithdrawalData).Methods("GET") + router.HandleFunc("/slot/{slot}/blsChange", handlers.SlotBlsChangeData).Methods("GET") + router.HandleFunc("/slots/finder", handlers.SlotFinder).Methods("GET") + router.HandleFunc("/slots", handlers.Slots).Methods("GET") + router.HandleFunc("/slots/data", handlers.SlotsData).Methods("GET") + router.HandleFunc("/blocks", handlers.Eth1Blocks).Methods("GET") + router.HandleFunc("/blocks/data", handlers.Eth1BlocksData).Methods("GET") + router.HandleFunc("/blocks/highest", handlers.Eth1BlocksHighest).Methods("GET") + router.HandleFunc("/address/{address}", handlers.Eth1Address).Methods("GET") + router.HandleFunc("/address/{address}/blocks", handlers.Eth1AddressBlocksMined).Methods("GET") + router.HandleFunc("/address/{address}/uncles", handlers.Eth1AddressUnclesMined).Methods("GET") + router.HandleFunc("/address/{address}/withdrawals", handlers.Eth1AddressWithdrawals).Methods("GET") + router.HandleFunc("/address/{address}/transactions", handlers.Eth1AddressTransactions).Methods("GET") + router.HandleFunc("/address/{address}/internalTxns", handlers.Eth1AddressInternalTransactions).Methods("GET") + router.HandleFunc("/address/{address}/blobTxns", handlers.Eth1AddressBlobTransactions).Methods("GET") + router.HandleFunc("/address/{address}/erc20", handlers.Eth1AddressErc20Transactions).Methods("GET") + router.HandleFunc("/address/{address}/erc721", handlers.Eth1AddressErc721Transactions).Methods("GET") + router.HandleFunc("/address/{address}/erc1155", handlers.Eth1AddressErc1155Transactions).Methods("GET") + router.HandleFunc("/token/{token}", handlers.Eth1Token).Methods("GET") + router.HandleFunc("/token/{token}/transfers", handlers.Eth1TokenTransfers).Methods("GET") + router.HandleFunc("/transactions", handlers.Eth1Transactions).Methods("GET") + router.HandleFunc("/transactions/data", handlers.Eth1TransactionsData).Methods("GET") + router.HandleFunc("/block/{block}", handlers.Eth1Block).Methods("GET") + router.HandleFunc("/block/{block}/transactions", handlers.BlockTransactionsData).Methods("GET") + router.HandleFunc("/tx/{hash}", handlers.Eth1TransactionTx).Methods("GET") + router.HandleFunc("/tx/{hash}/data", handlers.Eth1TransactionTxData).Methods("GET") + router.HandleFunc("/mempool", handlers.MempoolView).Methods("GET") + router.HandleFunc("/burn", handlers.Burn).Methods("GET") + router.HandleFunc("/burn/data", handlers.BurnPageData).Methods("GET") + router.HandleFunc("/gasnow", handlers.GasNow).Methods("GET") + router.HandleFunc("/gasnow/data", handlers.GasNowData).Methods("GET") + router.HandleFunc("/correlations", handlers.Correlations).Methods("GET") + router.HandleFunc("/correlations/data", handlers.CorrelationsData).Methods("POST") + + router.HandleFunc("/vis", handlers.Vis).Methods("GET") + router.HandleFunc("/charts", handlers.Charts).Methods("GET") + router.HandleFunc("/charts/{chart}", handlers.Chart).Methods("GET") + router.HandleFunc("/charts/{chart}/data", handlers.GenericChartData).Methods("GET") + router.HandleFunc("/vis/blocks", handlers.VisBlocks).Methods("GET") + router.HandleFunc("/vis/votes", handlers.VisVotes).Methods("GET") + router.HandleFunc("/epoch/{epoch}", handlers.Epoch).Methods("GET") + router.HandleFunc("/epochs", handlers.Epochs).Methods("GET") + router.HandleFunc("/epochs/data", handlers.EpochsData).Methods("GET") + + router.HandleFunc("/validator/{index}", handlers.Validator).Methods("GET") + router.HandleFunc("/validator/{index}/proposedblocks", handlers.ValidatorProposedBlocks).Methods("GET") + router.HandleFunc("/validator/{index}/attestations", handlers.ValidatorAttestations).Methods("GET") + router.HandleFunc("/validator/{index}/withdrawals", handlers.ValidatorWithdrawals).Methods("GET") + router.HandleFunc("/validator/{index}/sync", handlers.ValidatorSync).Methods("GET") + router.HandleFunc("/validator/{index}/history", handlers.ValidatorHistory).Methods("GET") + router.HandleFunc("/validator/{pubkey}/deposits", handlers.ValidatorDeposits).Methods("GET") + router.HandleFunc("/validator/{index}/slashings", handlers.ValidatorSlashings).Methods("GET") + router.HandleFunc("/validator/{index}/effectiveness", handlers.ValidatorAttestationInclusionEffectiveness).Methods("GET") + router.HandleFunc("/validator/{pubkey}/name", handlers.SaveValidatorName).Methods("POST") + router.HandleFunc("/watchlist/add", handlers.UsersModalAddValidator).Methods("POST") + router.HandleFunc("/validator/{pubkey}/remove", handlers.UserValidatorWatchlistRemove).Methods("POST") + router.HandleFunc("/validators", handlers.Validators).Methods("GET") + router.HandleFunc("/validators/data", handlers.ValidatorsData).Methods("GET") + router.HandleFunc("/validators/slashings", handlers.ValidatorsSlashings).Methods("GET") + router.HandleFunc("/validators/slashings/data", handlers.ValidatorsSlashingsData).Methods("GET") + router.HandleFunc("/validators/leaderboard", handlers.ValidatorsLeaderboard).Methods("GET") + router.HandleFunc("/validators/leaderboard/data", handlers.ValidatorsLeaderboardData).Methods("GET") + router.HandleFunc("/validators/withdrawals", handlers.Withdrawals).Methods("GET") + router.HandleFunc("/validators/withdrawals/data", handlers.WithdrawalsData).Methods("GET") + router.HandleFunc("/validators/withdrawals/bls", handlers.BLSChangeData).Methods("GET") + router.HandleFunc("/validators/deposits", handlers.Deposits).Methods("GET") + router.HandleFunc("/validators/initiated-deposits", handlers.Eth1Deposits).Methods("GET") // deprecated, will redirect to /validators/deposits + router.HandleFunc("/validators/initiated-deposits/data", handlers.Eth1DepositsData).Methods("GET") + router.HandleFunc("/validators/deposit-leaderboard", handlers.Eth1DepositsLeaderboard).Methods("GET") + router.HandleFunc("/validators/deposit-leaderboard/data", handlers.Eth1DepositsLeaderboardData).Methods("GET") + router.HandleFunc("/validators/included-deposits", handlers.Eth2Deposits).Methods("GET") // deprecated, will redirect to /validators/deposits + router.HandleFunc("/validators/included-deposits/data", handlers.Eth2DepositsData).Methods("GET") + + router.HandleFunc("/dashboard/save", handlers.UserDashboardWatchlistAdd).Methods("POST") + + router.HandleFunc("/dashboard/data/allbalances", handlers.DashboardDataBalanceCombined).Methods("GET") + + router.HandleFunc("/graffitiwall", handlers.Graffitiwall).Methods("GET") + router.HandleFunc("/calculator", handlers.StakingCalculator).Methods("GET") + router.HandleFunc("/search", handlers.Search).Methods("POST") + router.HandleFunc("/search/{type}/{search}", handlers.SearchAhead).Methods("GET") + router.HandleFunc("/imprint", handlers.Imprint).Methods("GET") + router.HandleFunc("/mobile", handlers.MobilePage).Methods("GET") + router.HandleFunc("/tools/unitConverter", handlers.UnitConverter).Methods("GET") + router.HandleFunc("/tools/broadcast", handlers.Broadcast).Methods("GET") + router.HandleFunc("/tools/broadcast", handlers.BroadcastPost).Methods("POST") + router.HandleFunc("/tools/broadcast/status/{jobID}", handlers.BroadcastStatus).Methods("GET") + + router.HandleFunc("/tables/{tableId}/state", handlers.GetDataTableStateChanges).Methods("GET") + router.HandleFunc("/tables/{tableId}/state", handlers.SetDataTableStateChanges).Methods("PUT") + router.HandleFunc("/ens/{search}", handlers.EnsSearch).Methods("GET") + + router.HandleFunc("/ethstore", handlers.EthStore).Methods("GET") + + router.HandleFunc("/stakingServices", handlers.StakingServices).Methods("GET") + + router.HandleFunc("/ethClients", handlers.EthClientsServices).Methods("GET") + router.HandleFunc("/entities", handlers.Entities).Methods("GET") + router.HandleFunc("/entities/data", handlers.EntitiesData).Methods("GET") + router.HandleFunc("/entity/{entity}/{subEntity}", handlers.EntityDetail).Methods("GET") + router.HandleFunc("/entity/{entity}/{subEntity}/subentities/data", handlers.EntitySubEntitiesData).Methods("GET") + router.HandleFunc("/entity/{entity}/{subEntity}/validators/data", handlers.EntityValidatorsData).Methods("GET") + router.HandleFunc("/relays", handlers.Relays).Methods("GET") + router.HandleFunc("/pools/rocketpool", handlers.PoolsRocketpool).Methods("GET") + router.HandleFunc("/pools/rocketpool/data/minipools", handlers.PoolsRocketpoolDataMinipools).Methods("GET") + router.HandleFunc("/pools/rocketpool/data/nodes", handlers.PoolsRocketpoolDataNodes).Methods("GET") + router.HandleFunc("/pools/rocketpool/data/dao_proposals", handlers.PoolsRocketpoolDataDAOProposals).Methods("GET") + router.HandleFunc("/pools/rocketpool/data/dao_members", handlers.PoolsRocketpoolDataDAOMembers).Methods("GET") + + router.HandleFunc("/advertisewithus", handlers.AdvertiseWithUs).Methods("GET") + router.HandleFunc("/advertisewithus", handlers.AdvertiseWithUsPost).Methods("POST") + + // confirming the email update should not require auth + router.HandleFunc("/settings/email/{hash}", handlers.UserConfirmUpdateEmail).Methods("GET") + router.HandleFunc("/gitcoinfeed", handlers.GitcoinFeed).Methods("GET") + + router.HandleFunc("/notifications/unsubscribe", handlers.UserNotificationsUnsubscribeByHash).Methods("GET") + + router.HandleFunc("/monitoring/{module}", handlers.Monitoring).Methods("GET", "OPTIONS") + + signUpRouter := router.PathPrefix("/").Subrouter() + signUpRouter.HandleFunc("/login", handlers.Login).Methods("GET") + signUpRouter.HandleFunc("/login", handlers.LoginPost).Methods("POST") + signUpRouter.HandleFunc("/logout", handlers.Logout).Methods("GET") + signUpRouter.HandleFunc("/register", handlers.Register).Methods("GET") + signUpRouter.HandleFunc("/register", handlers.RegisterPost).Methods("POST") + signUpRouter.HandleFunc("/resend", handlers.ResendConfirmation).Methods("GET") + signUpRouter.HandleFunc("/resend", handlers.ResendConfirmationPost).Methods("POST") + signUpRouter.HandleFunc("/requestReset", handlers.RequestResetPassword).Methods("GET") + signUpRouter.HandleFunc("/requestReset", handlers.RequestResetPasswordPost).Methods("POST") + signUpRouter.HandleFunc("/reset", handlers.ResetPasswordPost).Methods("POST") + signUpRouter.HandleFunc("/reset/{hash}", handlers.ResetPassword).Methods("GET") + signUpRouter.HandleFunc("/confirm/{hash}", handlers.ConfirmEmail).Methods("GET") + signUpRouter.HandleFunc("/confirmation", handlers.Confirmation).Methods("GET") + signUpRouter.HandleFunc("/pricing", handlers.Pricing).Methods("GET") + signUpRouter.HandleFunc("/pricing", handlers.PricingPost).Methods("POST") + signUpRouter.HandleFunc("/premium", handlers.MobilePricing).Methods("GET") + signUpRouter.Use(csrfHandler) + + oauthRouter := router.PathPrefix("/user").Subrouter() + oauthRouter.HandleFunc("/authorize", handlers.UserAuthorizeConfirm).Methods("GET") + oauthRouter.Use(csrfHandler) + + authRouter := router.PathPrefix("/user").Subrouter() + authRouter.HandleFunc("/mobile/settings", handlers.MobileDeviceSettingsPOST).Methods("POST") + authRouter.HandleFunc("/mobile/delete", handlers.MobileDeviceDeletePOST).Methods("POST", "OPTIONS") + authRouter.HandleFunc("/authorize", handlers.UserAuthorizeConfirmPost).Methods("POST") + authRouter.HandleFunc("/settings", handlers.UserSettings).Methods("GET") + authRouter.HandleFunc("/settings/password", handlers.UserUpdatePasswordPost).Methods("POST") + authRouter.HandleFunc("/settings/flags", handlers.UserUpdateFlagsPost).Methods("POST") + authRouter.HandleFunc("/settings/delete", handlers.UserDeletePost).Methods("POST") + authRouter.HandleFunc("/settings/email", handlers.UserUpdateEmailPost).Methods("POST") + authRouter.HandleFunc("/notifications", handlers.UserNotificationsCenter).Methods("GET") + authRouter.HandleFunc("/notifications/channels", handlers.UsersNotificationChannels).Methods("POST") + authRouter.HandleFunc("/notifications/data", handlers.UserNotificationsData).Methods("GET") + authRouter.HandleFunc("/notifications/subscribe", handlers.UserNotificationsSubscribe).Methods("POST") + authRouter.HandleFunc("/notifications/network/update", handlers.UserModalAddNetworkEvent).Methods("POST") + authRouter.HandleFunc("/watchlist/add", handlers.UsersModalAddValidator).Methods("POST") + authRouter.HandleFunc("/watchlist/remove", handlers.UserModalRemoveSelectedValidator).Methods("POST") + authRouter.HandleFunc("/watchlist/update", handlers.UserModalManageNotificationModal).Methods("POST") + authRouter.HandleFunc("/notifications/unsubscribe", handlers.UserNotificationsUnsubscribe).Methods("POST") + authRouter.HandleFunc("/notifications/bundled/subscribe", handlers.MultipleUsersNotificationsSubscribeWeb).Methods("POST", "OPTIONS") + authRouter.HandleFunc("/global_notification", handlers.UserGlobalNotification).Methods("GET") + authRouter.HandleFunc("/global_notification", handlers.UserGlobalNotificationPost).Methods("POST") + authRouter.HandleFunc("/ad_configuration", handlers.AdConfiguration).Methods("GET") + authRouter.HandleFunc("/ad_configuration", handlers.AdConfigurationPost).Methods("POST") + authRouter.HandleFunc("/ad_configuration/delete", handlers.AdConfigurationDeletePost).Methods("POST") + authRouter.HandleFunc("/explorer_configuration", handlers.ExplorerConfiguration).Methods("GET") + authRouter.HandleFunc("/explorer_configuration", handlers.ExplorerConfigurationPost).Methods("POST") + + authRouter.HandleFunc("/notifications-center", handlers.UserNotificationsCenter).Methods("GET") + authRouter.HandleFunc("/notifications-center/removeall", handlers.RemoveAllValidatorsAndUnsubscribe).Methods("POST") + + authRouter.HandleFunc("/subscriptions/data", handlers.UserSubscriptionsData).Methods("GET") + authRouter.HandleFunc("/generateKey", handlers.GenerateAPIKey).Methods("POST") + authRouter.HandleFunc("/ethClients", handlers.EthClientsServices).Methods("GET") + authRouter.HandleFunc("/webhooks", handlers.NotificationWebhookPage).Methods("GET") + authRouter.HandleFunc("/webhooks/add", handlers.UsersAddWebhook).Methods("POST") + authRouter.HandleFunc("/webhooks/{webhookID}/update", handlers.UsersEditWebhook).Methods("POST") + authRouter.HandleFunc("/webhooks/{webhookID}/delete", handlers.UsersDeleteWebhook).Methods("POST") + + err = initStripe(authRouter) + if err != nil { + logrus.Errorf("error could not init stripe, %v", err) + } + + authRouter.Use(handlers.UserAuthMiddleware) + authRouter.Use(csrfHandler) + authRouter.Use(utils.CORSMiddleware) + + if utils.Config.Frontend.Debug { + logrus.Infof("serving frontend file from static/ directory") + // serve files from the local filesystem when debugging, instead of the go embedded file system + templatesHandler := http.FileServer(http.Dir("templates")) + router.PathPrefix("/templates").Handler(http.StripPrefix("/templates/", templatesHandler)) + + staticHandler := http.FileServer(http.Dir("static/")) + router.PathPrefix("/").Handler(http.StripPrefix("/", staticHandler)) + } else { + fileSys := http.FS(static.Files) + router.PathPrefix("/").Handler(handlers.CustomFileServer(http.FileServer(fileSys), fileSys, handlers.NotFound)) + } + + } + + if utils.Config.Metrics.Enabled { + router.Use(metrics.HttpMiddleware) + } + + // ratelimit.Init() + // router.Use(ratelimit.HttpMiddleware) + + n := negroni.New(negroni.NewRecovery()) + n.Use(gzip.Gzip(gzip.DefaultCompression)) + + pa := &proxyaddr.ProxyAddr{} + _ = pa.Init(proxyaddr.CIDRLoopback) + n.Use(pa) + + n.UseHandler(utils.SessionStore.SCS.LoadAndSave(router)) + + if utils.Config.Frontend.HttpWriteTimeout == 0 { + utils.Config.Frontend.HttpWriteTimeout = time.Second * 15 + } + if utils.Config.Frontend.HttpReadTimeout == 0 { + utils.Config.Frontend.HttpReadTimeout = time.Second * 15 + } + if utils.Config.Frontend.HttpIdleTimeout == 0 { + utils.Config.Frontend.HttpIdleTimeout = time.Minute + } + frontendHttpServer = &http.Server{ + Addr: cfg.Frontend.Server.Host + ":" + cfg.Frontend.Server.Port, + WriteTimeout: utils.Config.Frontend.HttpWriteTimeout, + ReadTimeout: utils.Config.Frontend.HttpReadTimeout, + IdleTimeout: utils.Config.Frontend.HttpIdleTimeout, + Handler: n, + } + + logrus.Printf("http server listening on %v", frontendHttpServer.Addr) + go func() { + if err := frontendHttpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logrus.WithError(err).Fatal("Error serving frontend") + } + }() + } + + if utils.Config.Metrics.Enabled { + go func(addr string) { + logrus.Infof("serving metrics on %v", addr) + if err := metrics.Serve(addr); err != nil { + logrus.WithError(err).Fatal("Error serving metrics") + } + }(utils.Config.Metrics.Address) + } + + if utils.Config.Frontend.ShowDonors.Enabled { + services.InitGitCoinFeed() + } + + utils.WaitForCtrlC() + + if frontendHttpServer != nil { + logrus.Infof("shutting down frontendHttpServer") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + if err := frontendHttpServer.Shutdown(ctx); err != nil { + logrus.WithError(err).Error("error shutting down frontend server") + } + } + + logrus.Println("exiting...") +} diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/validator-tagger/main.go b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/validator-tagger/main.go new file mode 100644 index 000000000..8f00ed343 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/cmd/validator-tagger/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "flag" + "fmt" + + "github.com/gobitfly/eth2-beaconchain-explorer/db" + "github.com/gobitfly/eth2-beaconchain-explorer/metrics" + "github.com/gobitfly/eth2-beaconchain-explorer/rpc" + "github.com/gobitfly/eth2-beaconchain-explorer/services" + "github.com/gobitfly/eth2-beaconchain-explorer/types" + "github.com/gobitfly/eth2-beaconchain-explorer/utils" + "github.com/gobitfly/eth2-beaconchain-explorer/version" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/sirupsen/logrus" +) + +func main() { + var err error + + configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used") + versionFlag := flag.Bool("version", false, "Show version and exit") + scheduleFlag := flag.Bool("schedule", false, "Start scheduler loop (daily 10:00 UTC and hourly precompute)") + runFlag := flag.String("run", "", "Comma-separated steps to run on demand (import,lido,lido_csm,lido_simple_dvt,rocketpool,withdrawal_tagging,deposit_tagging,populate_validator_names,precompute,all)") + flag.Parse() + + if *versionFlag { + fmt.Println(version.Version) + fmt.Println(version.GoVersion) + return + } + + cfg := &types.Config{} + if err := utils.ReadConfig(cfg, *configPath); err != nil { + logrus.Fatalf("error reading config file: %v", err) + } + utils.Config = cfg + logrus.WithFields(logrus.Fields{ + "config": *configPath, + "version": version.Version, + "chainName": utils.Config.Chain.ClConfig.ConfigName, + }).Info("starting validator-tagger") + + // Validate critical chain params (common pattern across binaries) + if utils.Config.Chain.ClConfig.SlotsPerEpoch == 0 || utils.Config.Chain.ClConfig.SecondsPerSlot == 0 || utils.Config.Chain.GenesisTimestamp == 0 { + logrus.Fatal("invalid chain configuration: missing SlotsPerEpoch, SecondsPerSlot, or GenesisTimestamp") + } + + // Initialize primary databases (writer/reader) + db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.WriterDatabase.Username, + Password: cfg.WriterDatabase.Password, + Name: cfg.WriterDatabase.Name, + Host: cfg.WriterDatabase.Host, + Port: cfg.WriterDatabase.Port, + MaxOpenConns: cfg.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.WriterDatabase.MaxIdleConns, + SSL: cfg.WriterDatabase.SSL, + }, &types.DatabaseConfig{ + Username: cfg.ReaderDatabase.Username, + Password: cfg.ReaderDatabase.Password, + Name: cfg.ReaderDatabase.Name, + Host: cfg.ReaderDatabase.Host, + Port: cfg.ReaderDatabase.Port, + MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns, + SSL: cfg.ReaderDatabase.SSL, + }, "pgx", "postgres") + defer db.ReaderDb.Close() + defer db.WriterDb.Close() + + // If ClickHouse HTTP is enabled, fetch the large result as zstd-compressed Parquet and parse with parquet-go + db.MustInitClickhouseDB(nil, &types.DatabaseConfig{ + Username: cfg.ClickHouse.ReaderDatabase.Username, + Password: cfg.ClickHouse.ReaderDatabase.Password, + Name: cfg.ClickHouse.ReaderDatabase.Name, + Host: cfg.ClickHouse.ReaderDatabase.Host, + Port: cfg.ClickHouse.ReaderDatabase.Port, + MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + SSL: cfg.ClickHouse.ReaderDatabase.SSL, + // SSL: true, + }, "clickhouse", "clickhouse") + + // Initialize Erigon RPC client (required for Lido exporter step) + rpc.CurrentErigonClient, err = rpc.NewErigonClient(utils.Config.Eth1ErigonEndpoint) + if err != nil { + logrus.Fatalf("error initializing erigon client: %v", err) + } + + if utils.Config.Metrics.Enabled { + go func(addr string) { + logrus.Infof("serving metrics on %v", addr) + if err := metrics.Serve(addr); err != nil { + logrus.WithError(err).Fatal("Error serving metrics") + } + }(utils.Config.Metrics.Address) + } + + // On-demand single/multi-step run + if *runFlag != "" { + if err := services.RunValidatorTaggerOnDemand(context.Background(), *runFlag); err != nil { + logrus.Fatalf("on-demand run failed: %v", err) + } + logrus.Info("on-demand run completed; exiting") + return + } + + // Scheduler mode (enabled only if flag set or config toggled) + if *scheduleFlag || utils.Config.ValidatorTagger.SchedulerEnabled { + services.RunValidatorTaggerScheduler() + return + } +} diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.mod b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.mod new file mode 100644 index 000000000..62ada7cc7 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.mod @@ -0,0 +1,272 @@ +module github.com/gobitfly/eth2-beaconchain-explorer + +go 1.23 + +require ( + cloud.google.com/go/bigtable v1.35.0 + cloud.google.com/go/secretmanager v1.14.2 + firebase.google.com/go/v4 v4.14.1 + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 + github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 + github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 + github.com/awa/go-iap v1.26.1 + github.com/aws/aws-sdk-go-v2 v1.23.0 + github.com/aws/aws-sdk-go-v2/credentials v1.16.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 + github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e + github.com/carlmjohnson/requests v0.23.4 + github.com/davecgh/go-spew v1.1.1 + github.com/doug-martin/goqu/v9 v9.19.0 + github.com/ethereum/go-ethereum v1.14.6-0.20250124151602-75526bb8e01b + github.com/evanw/esbuild v0.8.23 + github.com/go-redis/redis/v8 v8.11.5 + github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 + github.com/gobitfly/scs/v2 v2.0.0-20240516120302-8754831e6b9b + github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang/protobuf v1.5.4 + github.com/gomodule/redigo v1.8.0 + github.com/google/go-cmp v0.6.0 + github.com/gorilla/context v1.1.1 + github.com/gorilla/csrf v1.7.0 + github.com/gorilla/mux v1.8.0 + github.com/gorilla/websocket v1.5.3 + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d + github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e + github.com/jackc/pgx/v5 v5.4.3 + github.com/jmoiron/sqlx v1.2.0 + github.com/juliangruber/go-intersect v1.1.0 + github.com/kataras/i18n v0.0.5 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/klauspost/pgzip v1.2.6 + github.com/lib/pq v1.10.7 + github.com/mailgun/mailgun-go/v4 v4.1.3 + github.com/mitchellh/mapstructure v1.5.0 + github.com/montanaflynn/stats v0.7.1 + github.com/mssola/user_agent v0.5.2 + github.com/mvdan/xurls v1.1.0 + github.com/parquet-go/parquet-go v0.25.1 + github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 + github.com/pkg/errors v0.9.1 + github.com/pressly/goose/v3 v3.10.0 + github.com/prometheus/client_golang v1.20.0 + github.com/protolambda/zrnt v0.32.2 + github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e + github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388 + github.com/prysmaticlabs/prysm/v5 v5.2.0 + github.com/rocket-pool/rocketpool-go v1.8.3-0.20240618173422-783b8668f5b4 + github.com/rocket-pool/smartnode v1.13.6 + github.com/shopspring/decimal v1.4.0 + github.com/sirupsen/logrus v1.9.3 + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e + github.com/skygeario/go-confusable-homoglyphs v0.0.0-20191212061114-e2b2a60df110 + github.com/stripe/stripe-go/v72 v72.50.0 + github.com/urfave/negroni v1.0.0 + github.com/wealdtech/go-ens/v3 v3.6.0 + github.com/wealdtech/go-eth2-types/v2 v2.8.1 + github.com/wealdtech/go-eth2-util v1.8.1 + github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d + go.uber.org/atomic v1.11.0 + golang.org/x/crypto v0.32.0 + golang.org/x/sync v0.10.0 + golang.org/x/text v0.21.0 + golang.org/x/time v0.9.0 + google.golang.org/api v0.217.0 + google.golang.org/grpc v1.69.4 + google.golang.org/protobuf v1.36.3 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa +) + +require ( + cel.dev/expr v0.16.2 // indirect + cloud.google.com/go v0.118.0 // indirect + cloud.google.com/go/auth v0.14.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.3.1 // indirect + cloud.google.com/go/longrunning v0.6.4 // indirect + cloud.google.com/go/monitoring v1.22.1 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/MicahParks/keyfunc v1.9.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/alessio/shellescape v1.4.1 // indirect + github.com/alexedwards/scs/v2 v2.5.0 // indirect + github.com/allegro/bigcache v1.2.1 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect + github.com/aws/smithy-go v1.17.0 // indirect + github.com/bits-and-blooms/bitset v1.17.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/consensys/bavard v0.1.22 // indirect + github.com/consensys/gnark-crypto v0.14.0 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect + github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect + github.com/deckarep/golang-set/v2 v2.6.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/envoyproxy/go-control-plane v0.13.1 // indirect + github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/go-verkle v0.2.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/glendc/go-external-ip v0.1.0 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.3.2 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/boxo v0.8.0 // indirect + github.com/ipfs/go-bitfield v1.1.0 // indirect + github.com/ipfs/go-block-format v0.1.2 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-ipfs-util v0.0.2 // indirect + github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipld-format v0.4.0 // indirect + github.com/ipfs/go-ipld-legacy v0.1.1 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/ipld/go-codec-dagpb v1.6.0 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/polydawn/refmt v0.89.0 // indirect + github.com/protolambda/zssz v0.1.5 // indirect + github.com/prysmaticlabs/fastssz v0.0.0-20241008181541-518c4ce73516 // indirect + github.com/prysmaticlabs/gohashtree v0.0.4-beta.0.20240624100937-73632381301b // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/wealdtech/go-bytesutil v1.2.1 // indirect + github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a // indirect + github.com/wealdtech/go-multicodec v1.4.0 // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + google.golang.org/appengine/v2 v2.0.2 // indirect + google.golang.org/genproto v0.0.0-20241216192217-9240e9c98484 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + lukechampine.com/blake3 v1.3.0 // indirect + rsc.io/binaryregexp v0.2.0 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) + +require ( + cloud.google.com/go/firestore v1.17.0 // indirect + cloud.google.com/go/storage v1.43.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/attestantio/go-eth2-client v0.19.9 + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coocood/freecache v1.2.3 + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/ferranbt/fastssz v0.1.3 // indirect + github.com/go-chi/chi v4.0.2+incompatible // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/goccy/go-yaml v1.10.0 // indirect + github.com/golang/glog v1.2.2 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/uuid v1.6.0 + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/gorilla/securecookie v1.1.1 // indirect + github.com/hashicorp/go-version v1.6.0 + github.com/herumi/bls-eth-go-binary v1.29.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.17.9 + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/highwayhash v1.0.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/supranational/blst v0.3.14 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d + github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect + golang.org/x/net v0.34.0 + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/json-iterator/go => github.com/prestonvanloon/go v1.1.7-0.20190722034630-4f2e55fcf87b + +// See https://github.com/prysmaticlabs/grpc-gateway/issues/2 +replace github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/prysmaticlabs/grpc-gateway/v2 v2.3.1-0.20210702154020-550e1cd83ec1 + +// replace github.com/prysmaticlabs/prysm/v3 => github.com/gobitfly/prysm/v3 v3.0.0-20230216184552-2f3f1e8190d5 + +replace github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a => github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd + +replace github.com/rocket-pool/rocketpool-go v1.8.2 => github.com/gobitfly/rocketpool-go v0.0.0-20240105082836-5bb7c83a2d08 + +// replace github.com/ethereum/go-ethereum => github.com/gobitfly/go-ethereum v1.8.13-0.20230227100926-e78d720a0bf6 diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.sum b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.sum new file mode 100644 index 000000000..9e3860461 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/go.sum @@ -0,0 +1,1165 @@ +cel.dev/expr v0.16.2 h1:RwRhoH17VhAu9U5CMvMhH1PDVgf0tuz9FT+24AfMLfU= +cel.dev/expr v0.16.2/go.mod h1:gXngZQMkWJoSbE8mOzehJlXQyubn/Vg0vR9/F3W7iw8= +cloud.google.com/go v0.118.0 h1:tvZe1mgqRxpiVa3XlIGMiPcEUbP1gNXELgD4y/IXmeQ= +cloud.google.com/go v0.118.0/go.mod h1:zIt2pkedt/mo+DQjcT4/L3NDxzHPR29j5HcclNH+9PM= +cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= +cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cloud.google.com/go/bigtable v1.35.0 h1:UEacPwaejN2mNbz67i1Iy3G812rxtgcs6ePj1TAg7dw= +cloud.google.com/go/bigtable v1.35.0/go.mod h1:EabtwwmTcOJFXp+oMZAT/jZkyDIjNwrv53TrS4DGrrM= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/firestore v1.17.0 h1:iEd1LBbkDZTFsLw3sTH50eyg4qe8eoG6CjocmEXO9aQ= +cloud.google.com/go/firestore v1.17.0/go.mod h1:69uPx1papBsY8ZETooc71fOhoKkD70Q1DwMrtKuOT/Y= +cloud.google.com/go/iam v1.3.1 h1:KFf8SaT71yYq+sQtRISn90Gyhyf4X8RGgeAVC8XGf3E= +cloud.google.com/go/iam v1.3.1/go.mod h1:3wMtuyT4NcbnYNPLMBzYRFiEfjKfJlLVLrisE7bwm34= +cloud.google.com/go/longrunning v0.6.4 h1:3tyw9rO3E2XVXzSApn1gyEEnH2K9SynNQjMlBi3uHLg= +cloud.google.com/go/longrunning v0.6.4/go.mod h1:ttZpLCe6e7EXvn9OxpBRx7kZEB0efv8yBO6YnVMfhJs= +cloud.google.com/go/monitoring v1.22.1 h1:KQbnAC4IAH+5x3iWuPZT5iN9VXqKMzzOgqcYB6fqPDE= +cloud.google.com/go/monitoring v1.22.1/go.mod h1:AuZZXAoN0WWWfsSvET1Cpc4/1D8LXq8KRDU87fMS6XY= +cloud.google.com/go/secretmanager v1.14.2 h1:2XscWCfy//l/qF96YE18/oUaNJynAx749Jg3u0CjQr8= +cloud.google.com/go/secretmanager v1.14.2/go.mod h1:Q18wAPMM6RXLC/zVpWTlqq2IBSbbm7pKBlM3lCKsmjw= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= +firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= +github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= +github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c h1:m2XNHdllTtwuYPpRJpVdlXKOEGZRR80VvAjo1sfy9LA= +github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c/go.mod h1:ceKFatoD+hfHWWeHOAYue1J+XgOJjE7dw8l3JtIRTGY= +github.com/alexedwards/scs/v2 v2.5.0 h1:zgxOfNFmiJyXG7UPIuw1g2b9LWBeRLh3PjfB9BDmfL4= +github.com/alexedwards/scs/v2 v2.5.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/attestantio/go-eth2-client v0.19.9 h1:g5LLX3X7cLC0KS0oai/MtxBOZz3U3QPIX5qryYMxgVE= +github.com/attestantio/go-eth2-client v0.19.9/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= +github.com/awa/go-iap v1.26.1 h1:0CvM7xkJ0zDdbHIvvXKc92J9gGRbnChxAQloqXGVK88= +github.com/awa/go-iap v1.26.1/go.mod h1:ChJ/FTLV3xMZdmdJghVxgLC1z+yl4hAZ923bHnt9Z+k= +github.com/aws/aws-sdk-go-v2 v1.23.0 h1:PiHAzmiQQr6JULBUdvR8fKlA+UPKLT/8KbiqpFBWiAo= +github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= +github.com/aws/aws-sdk-go-v2/credentials v1.16.2 h1:0sdZ5cwfOAipTzZ7eOL0gw4LAhk/RZnTa16cDqIt8tg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.2/go.mod h1:sDdvGhXrSVT5yzBDR7qXz+rhbpiMpUYfF3vJ01QSdrc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 h1:DUwbD79T8gyQ23qVXFUthjzVMTviSHi3y4z58KvghhM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 h1:AplLJCtIaUZDCbr6+gLYdsYNxne4iuaboJhVt9d+WXI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3/go.mod h1:ify42Rb7nKeDDPkFjKn7q1bPscVPu/+gmHH8d2c+anU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3 h1:lMwCXiWJlrtZot0NJTjbC8G9zl+V3i68gBTBBvDeEXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3/go.mod h1:5yzAuE9i2RkVAttBl8yxZgQr5OCq4D5yDnG7j9x2L0U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3 h1:xbwRyCy7kXrOj89iIKLB6NfE2WCpP9HoKyk8dMDvnIQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3/go.mod h1:R+/S1O4TYpcktbVwddeOYg+uwUfLhADP2S/x4QwsCTM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 h1:kJOolE8xBAD13xTCgOakByZkyP4D/owNmvEiioeUNAg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3/go.mod h1:Owv1I59vaghv1Ax8zz8ELY8DN7/Y0rGS+WWAmjgi950= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 h1:KV0z2RDc7euMtg8aUT1czv5p29zcLlXALNFsd3jkkEc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 h1:cwTuq73Tv6jtNJIMgTDKsih5O2YsVrKGpg20H98tbmo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ= +github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc= +github.com/bazelbuild/rules_go v0.49.0 h1:5vCbuvy8Q11g41lseGJDc5vxhDjJtfxr6nM/IC4VmqM= +github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= +github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/carlmjohnson/requests v0.23.4 h1:AxcvapfB9RPXLSyvAHk9YJoodQ43ZjzNHj6Ft3tQGdg= +github.com/carlmjohnson/requests v0.23.4/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= +github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A= +github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= +github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= +github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto= +github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= +github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= +github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= +github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= +github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= +github.com/doug-martin/goqu/v9 v9.19.0 h1:PD7t1X3tRcUiSdc5TEyOFKujZA5gs3VSA7wxSvBx7qo= +github.com/doug-martin/goqu/v9 v9.19.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= +github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.14.6-0.20250124151602-75526bb8e01b h1:G8jJ0o196aZUEwRE9+FUcllEFwXnKDK3esCzmldjGrI= +github.com/ethereum/go-ethereum v1.14.6-0.20250124151602-75526bb8e01b/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs= +github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= +github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= +github.com/evanw/esbuild v0.8.23 h1:eRRG1fNtQ9KPG3lM62EUYagLVMSuxSTBEgukqY0et3w= +github.com/evanw/esbuild v0.8.23/go.mod h1:y2AFBAGVelPqPodpdtxWWqe6n2jYf5FrsJbligmRmuw= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= +github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/glendc/go-external-ip v0.1.0 h1:iX3xQ2Q26atAmLTbd++nUce2P5ht5P4uD4V7caSY/xg= +github.com/glendc/go-external-ip v0.1.0/go.mod h1:CNx312s2FLAJoWNdJWZ2Fpf5O4oLsMFwuYviHjS4uJE= +github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= +github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.13.0 h1:cFRQdfaSMCOSfGCCLB20MHvuoHb/s5G8L5pu2ppK5AQ= +github.com/go-playground/validator/v10 v10.13.0/go.mod h1:dwu7+CG8/CtBiJFZDz4e+5Upb6OLw04gtBYw0mcG/z4= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 h1:qAEBlj4j25sJjsj5CQraL94rY9jyPIjCZ+hlu84n5l0= +github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7/go.mod h1:JrFv2uq5i+p6yEoRSHralEV139HkPV0wBjZZHWb4he8= +github.com/gobitfly/scs/v2 v2.0.0-20240516120302-8754831e6b9b h1:Svh+MoUvmbie+1EUexn5p5UbDIdBWpP9rLe4q1/9188= +github.com/gobitfly/scs/v2 v2.0.0-20240516120302-8754831e6b9b/go.mod h1:BJs6oUWumklexTEBnf6sgirUTagmYe6e9y36MaFimPg= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ= +github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.0 h1:OXfLQ/k8XpYF8f8sZKd2Df4SDyzbLeC35OsBsB11rYg= +github.com/gomodule/redigo v1.8.0/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/csrf v1.7.0 h1:mMPjV5/3Zd460xCavIkppUdvnl5fPXMpv2uz2Zyg7/Y= +github.com/gorilla/csrf v1.7.0/go.mod h1:+a/4tCmqhG6/w4oafeAZ9pEa3/NZOWYVbD9fV0FwIQA= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/herumi/bls-eth-go-binary v1.29.1 h1:XcNSHYTyNjEUVfWDCE2gtG5r95biTwd7MJUJF09LtSE= +github.com/herumi/bls-eth-go-binary v1.29.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc= +github.com/huandu/go-clone v1.6.0/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= +github.com/huandu/go-clone/generic v1.6.0 h1:Wgmt/fUZ28r16F2Y3APotFD59sHk1p78K0XLdbUYN5U= +github.com/huandu/go-clone/generic v1.6.0/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= +github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= +github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= +github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= +github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= +github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= +github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= +github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.0 h1:vrbA9Ud87g6JdFWkHTJXppVce58qPIdP7N8y0Ml/A7Q= +github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= +github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e h1:i3gQ/Zo7sk4LUVbsAjTNeC4gIjoPNIZVzs4EXstssV4= +github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e/go.mod h1:zUHglCZ4mpDUPgIwqEKoba6+tcUQzRdb1+DPTuYe9pI= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= +github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juliangruber/go-intersect v1.1.0 h1:sc+y5dCjMMx0pAdYk/N6KBm00tD/f3tq+Iox7dYDUrY= +github.com/juliangruber/go-intersect v1.1.0/go.mod h1:WMau+1kAmnlQnKiikekNJbtGtfmILU/mMU6H7AgKbWQ= +github.com/kataras/i18n v0.0.5 h1:X9EQHxDhjpN0zh+Ry0PZvi0ODi9lf5mo4wiXWtOYhlY= +github.com/kataras/i18n v0.0.5/go.mod h1:U0aKF7ANqGmFVs4WCexDTYGf8wg7Rb3mLJCmr/OuDoo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.3 h1:6BE2vPT0lqoz3fmOesHZiaiFh7889ssCo2GMvLCfiuA= +github.com/leodido/go-urn v1.2.3/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-libp2p v0.36.5 h1:DoABsaHO0VXwH6pwCs2F6XKAXWYjFMO4HFBoVxTnF9g= +github.com/libp2p/go-libp2p v0.36.5/go.mod h1:CpszAtXxHYOcyvB7K8rSHgnNlh21eKjYbEfLoMerbEI= +github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= +github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/mailgun/mailgun-go/v4 v4.1.3 h1:KLa5EZaOMMeyvY/lfAhWxv9ealB3mtUsMz0O9XmTtP0= +github.com/mailgun/mailgun-go/v4 v4.1.3/go.mod h1:R9kHUQBptF4iSEjhriCQizplCDwrnDShy8w/iPiOfaM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo= +github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= +github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= +github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= +github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= +github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= +github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo= +github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 h1:d6HcSW4ZoNlUWrPyZtBwIu8yv4WAWIU3R/jorwVkFtQ= +github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029/go.mod h1:94RTq2fypdZCze25ZEZSjtbAQRT3cL/8EuRUqAZC/+w= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= +github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= +github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= +github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= +github.com/pion/interceptor v0.1.30 h1:au5rlVHsgmxNi+v/mjOPazbW1SHzfx7/hYOEYQnUcxA= +github.com/pion/interceptor v0.1.30/go.mod h1:RQuKT5HTdkP2Fi0cuOS5G5WNymTjzXaGF75J4k7z2nc= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= +github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= +github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk= +github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= +github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk= +github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= +github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= +github.com/pressly/goose/v3 v3.10.0 h1:Gn5E9CkPqTtWvfaDVqtJqMjYtsrZ9K5mU/8wzTsvg04= +github.com/pressly/goose/v3 v3.10.0/go.mod h1:c5D3a7j66cT0fhRPj7KsXolfduVrhLlxKZjmCVSey5w= +github.com/prestonvanloon/go v1.1.7-0.20190722034630-4f2e55fcf87b h1:Bt5PzQCqfP4xiLXDSrMoqAfj6CBr3N9DAyyq8OiIWsc= +github.com/prestonvanloon/go v1.1.7-0.20190722034630-4f2e55fcf87b/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/protolambda/zrnt v0.32.2 h1:KZ48T+3UhsPXNdtE/5QEvGc9DGjUaRI17nJaoznoIaM= +github.com/protolambda/zrnt v0.32.2/go.mod h1:A0fezkp9Tt3GBLATSPIbuY4ywYESyAuc/FFmPKg8Lqs= +github.com/protolambda/zssz v0.1.5 h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek= +github.com/protolambda/zssz v0.1.5/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= +github.com/prysmaticlabs/fastssz v0.0.0-20241008181541-518c4ce73516 h1:xuVAdtz5ShYblG2sPyb4gw01DF8InbOI/kBCQjk7NiM= +github.com/prysmaticlabs/fastssz v0.0.0-20241008181541-518c4ce73516/go.mod h1:h2OlIZD/M6wFvV3YMZbW16lFgh3Rsye00G44J2cwLyU= +github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e h1:ATgOe+abbzfx9kCPeXIW4fiWyDdxlwHw07j8UGhdTd4= +github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= +github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388 h1:4bD+ujqGfY4zoDUF3q9MhdmpPXzdp03DYUIlXeQ72kk= +github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388/go.mod h1:VecIJZrewdAuhVckySLFt2wAAHRME934bSDurP8ftkc= +github.com/prysmaticlabs/gohashtree v0.0.4-beta.0.20240624100937-73632381301b h1:VK7thFOnhxAZ/5aolr5Os4beiubuD08WiuiHyRqgwks= +github.com/prysmaticlabs/gohashtree v0.0.4-beta.0.20240624100937-73632381301b/go.mod h1:HRuvtXLZ4WkaB1MItToVH2e8ZwKwZPY5/Rcby+CvvLY= +github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20230228205207-28762a7b9294 h1:q9wE0ZZRdTUAAeyFP/w0SwBEnCqlVy2+on6X2/e+eAU= +github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20230228205207-28762a7b9294/go.mod h1:ZVEbRdnMkGhp/pu35zq4SXxtvUwWK0J1MATtekZpH2Y= +github.com/prysmaticlabs/prysm/v5 v5.2.0 h1:JqKKK5aqehZN9GiSOSSw4M57NpbvG0nIxqFK5KpPnRw= +github.com/prysmaticlabs/prysm/v5 v5.2.0/go.mod h1:cQc+NIMKaHjPvY566HsYcuni763nzuUWnDsDbk45bbs= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd h1:p9KuetSKB9nte9I/MkkiM3pwKFVQgqxxPTQ0y56Ff6s= +github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd/go.mod h1:UE9fof8P7iESVtLn1K9CTSkNRYVFHZHlf96RKbU33kA= +github.com/rocket-pool/rocketpool-go v1.8.3-0.20240618173422-783b8668f5b4 h1:X2cA48Uv5/z0z3yBUfzecAQ8gqGjmTe1HmTcYs6+L/A= +github.com/rocket-pool/rocketpool-go v1.8.3-0.20240618173422-783b8668f5b4/go.mod h1:f2TVsMOYmCwaJOhshG2zRoX89PZmvCkCD7UYJ9waRkI= +github.com/rocket-pool/smartnode v1.13.6 h1:dZCDEb5+ZFN7iU5/Qzxv16efS1zk/fLf6HiewNzjBfY= +github.com/rocket-pool/smartnode v1.13.6/go.mod h1:mW/gljU+C02+JhrXN3dQfftaLnEqw8mIkTtkKOFIyJk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= +github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= +github.com/skygeario/go-confusable-homoglyphs v0.0.0-20191212061114-e2b2a60df110 h1:DWSits/O+2QUbvRsCpy8zol+5QWpx0PWDkjIQVMf7kE= +github.com/skygeario/go-confusable-homoglyphs v0.0.0-20191212061114-e2b2a60df110/go.mod h1:jtRQO9YC/yD/E5FqG0RWijzCrrXPvNKsMf4/lGv7sUI= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stripe/stripe-go/v72 v72.50.0 h1:oy+EsSKMrFS3zzayb8Ic+2LZ04Ux0vJ4990/7psaYsc= +github.com/stripe/stripe-go/v72 v72.50.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= +github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= +github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= +github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= +github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10/go.mod h1:x/Pa0FF5Te9kdrlZKJK82YmAkvL8+f989USgz6Jiw7M= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= +github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= +github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.11.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= +github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/wealdtech/go-bytesutil v1.2.1 h1:TjuRzcG5KaPwaR5JB7L/OgJqMQWvlrblA1n0GfcXFSY= +github.com/wealdtech/go-bytesutil v1.2.1/go.mod h1:RhUDUGT1F4UP4ydqbYp2MWJbAel3M+mKd057Pad7oag= +github.com/wealdtech/go-ens/v3 v3.6.0 h1:EAByZlHRQ3vxqzzwNi0GvEq1AjVozfWO4DMldHcoVg8= +github.com/wealdtech/go-ens/v3 v3.6.0/go.mod h1:hcmMr9qPoEgVSEXU2Bwzrn/9NczTWZ1rE53jIlqUpzw= +github.com/wealdtech/go-eth2-types/v2 v2.8.1 h1:y2N3xSIZ3tVqsnvj4AgPkh48U5sM612vhZwlK3k+3lM= +github.com/wealdtech/go-eth2-types/v2 v2.8.1/go.mod h1:3TJShI4oBzG8pCZsfe3NZAq8QAmXrC2rd45q7Vn/XB8= +github.com/wealdtech/go-eth2-util v1.8.1 h1:nb50hygsNoql94akg7GN6im/weg8ZZgJWHgiyrj8qiU= +github.com/wealdtech/go-eth2-util v1.8.1/go.mod h1:vv+8jVgYRXEGty/VLPNn1RYlbQNYmTht3VR6nfh0z4E= +github.com/wealdtech/go-multicodec v1.4.0 h1:iq5PgxwssxnXGGPTIK1srvt6U5bJwIp7k6kBrudIWxg= +github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3hGpu/snBOS3GQLw4= +github.com/wealdtech/go-string2eth v1.2.1 h1:u9sofvGFkp+uvTg4Nvsvy5xBaiw8AibGLLngfC4F76g= +github.com/wealdtech/go-string2eth v1.2.1/go.mod h1:9uwxm18zKZfrReXrGIbdiRYJtbE91iGcj6TezKKEx80= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= +github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/wlynxg/anet v0.0.4 h1:0de1OFQxnNqAu+x2FAKKCVIrnfGKQbs7FQz++tB0+Uw= +github.com/wlynxg/anet v0.0.4/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d h1:Gsw/uTjNB2vkIEhBO3NAXjKo6QRY6D5B0GzMv980ses= +github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d/go.mod h1:PXKs8dwJ+2BKBQxMlKg+eaO27VucEN/jrrymnRB5tvs= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/api v0.217.0 h1:GYrUtD289o4zl1AhiTZL0jvQGa2RDLyC+kX1N/lfGOU= +google.golang.org/api v0.217.0/go.mod h1:qMc2E8cBAbQlRypBTBWHklNJlaZZJBwDv81B1Iu8oSI= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/appengine/v2 v2.0.2 h1:MSqyWy2shDLwG7chbwBJ5uMyw6SNqJzhJHNDwYB0Akk= +google.golang.org/appengine/v2 v2.0.2/go.mod h1:PkgRUWz4o1XOvbqtWTkBtCitEJ5Tp4HoVEdMMYQR/8E= +google.golang.org/genproto v0.0.0-20241216192217-9240e9c98484 h1:a/U5otbGrI6mYIO598WriFB1172i6Ktr6FGcatZD3Yw= +google.golang.org/genproto v0.0.0-20241216192217-9240e9c98484/go.mod h1:Gmd/M/W9fEyf6VSu/mWLnl+9Be51B9CLdxdsKokYq7Y= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/apimachinery v0.30.4 h1:5QHQI2tInzr8LsT4kU/2+fSeibH1eIHswNx480cqIoY= +k8s.io/apimachinery v0.30.4/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.4 h1:eculUe+HPQoPbixfwmaSZGsKcOf7D288tH6hDAdd+wY= +k8s.io/client-go v0.30.4/go.mod h1:IBS0R/Mt0LHkNHF4E6n+SUDPG7+m2po6RZU7YHeOpzc= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= +lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= +modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.21.0 h1:4aP4MdUf15i3R3M2mx6Q90WHKz3nZLoz96zlB6tNdow= +modernc.org/sqlite v1.21.0/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/handlers/validator.go b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/handlers/validator.go new file mode 100644 index 000000000..470dbb9fc --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/handlers/validator.go @@ -0,0 +1,2569 @@ +package handlers + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "html/template" + "math" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/gobitfly/eth2-beaconchain-explorer/db" + "github.com/gobitfly/eth2-beaconchain-explorer/services" + "github.com/gobitfly/eth2-beaconchain-explorer/templates" + "github.com/gobitfly/eth2-beaconchain-explorer/types" + "github.com/gobitfly/eth2-beaconchain-explorer/utils" + "github.com/shopspring/decimal" + "github.com/sirupsen/logrus" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/crypto" + "github.com/lib/pq" + protomath "github.com/protolambda/zrnt/eth2/util/math" + "golang.org/x/sync/errgroup" + + "github.com/gorilla/csrf" + "github.com/gorilla/mux" + "github.com/juliangruber/go-intersect" + + itypes "github.com/gobitfly/eth-rewards/types" +) + +var validatorEditFlash = "edit_validator_flash" + +// Validator returns validator data using a go template +func Validator(w http.ResponseWriter, r *http.Request) { + validatorTemplateFiles := append(layoutTemplateFiles, + "validator/validator.html", + "validator/heading.html", + "validator/tables/*.html", + "validator/modals.html", + "modals.html", + "validator/overview.html", + "validator/charts.html", + "validator/countdown.html", + "components/flashMessage.html", + "components/rocket.html") + var validatorTemplate = templates.GetTemplate(validatorTemplateFiles...) + + currency := GetCurrency(r) + + timings := struct { + Start time.Time + BasicInfo time.Duration + Earnings time.Duration + Deposits time.Duration + Proposals time.Duration + Charts time.Duration + Effectiveness time.Duration + Statistics time.Duration + SyncStats time.Duration + Rocketpool time.Duration + }{ + Start: time.Now(), + } + + w.Header().Set("Content-Type", "text/html") + vars := mux.Vars(r) + + var index uint64 + var err error + errFields := map[string]interface{}{ + "route": r.URL.String()} + + latestEpoch := services.LatestEpoch() + latestProposedSlot := services.LatestProposedSlot() + lastFinalizedEpoch := services.LatestFinalizedEpoch() + isPreGenesis := false + if latestEpoch == 0 { + latestEpoch = 1 + latestProposedSlot = 1 + lastFinalizedEpoch = 1 + isPreGenesis = true + } + + validatorPageData := types.ValidatorPageData{} + + validatorPageData.CappellaHasHappened = latestEpoch >= (utils.Config.Chain.ClConfig.CappellaForkEpoch) + futureProposalEpoch := uint64(0) + futureSyncDutyEpoch := uint64(0) + + stats := services.GetLatestStats() + epoch := services.LatestEpoch() + latestState := services.LatestState() + + var finalityDelay uint64 = 0 + if latestState != nil { + finalityDelay = uint64(math.Max(1, float64(latestState.FinalityDelay)-2)) + } + if finalityDelay < 1 { + finalityDelay = 1 + } + + validatorPageData.ElectraHasHappened = utils.ElectraHasHappened(epoch) + + var activationChurnRate *uint64 + if utils.ElectraHasHappened(epoch) { + queueData := services.LatestQueueData() + + validatorPageData.ChurnRate = queueData.EnteringBalancePerEpoch + } else { + + churnRate := stats.ValidatorChurnLimit + if churnRate == nil { + churnRate = new(uint64) + } + + if *churnRate == 0 { + *churnRate = 4 + logger.Warning("Churn rate not set in config using 4 as default") + } + validatorPageData.ChurnRate = *churnRate + + activationChurnRate = stats.ValidatorActivationChurnLimit + if activationChurnRate == nil { + activationChurnRate = new(uint64) + } + + if *activationChurnRate == 0 { + *activationChurnRate = 4 + logger.Warning("Activation Churn rate not set in config using 4 as default") + } + } + + validatorPageData.InclusionDelay = int64((utils.Config.Chain.ClConfig.Eth1FollowDistance*utils.Config.Chain.ClConfig.SecondsPerEth1Block+utils.Config.Chain.ClConfig.SecondsPerSlot*utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.EpochsPerEth1VotingPeriod)/3600) + 1 + + data := InitPageData(w, r, "validators", "/validators", "", validatorTemplateFiles) + validatorPageData.NetworkStats = services.LatestIndexPageData() + validatorPageData.User = data.User + validatorPageData.ConsolidationTargetIndex = -1 + validatorPageData.Epoch = latestEpoch + + validatorPageData.FlashMessage, err = utils.GetFlash(w, r, validatorEditFlash) + if err != nil { + utils.LogError(err, "error getting flash message", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + if strings.Contains(vars["index"], "0x") || len(vars["index"]) == 96 { + // Request came with a hash + pubKey, err := hex.DecodeString(strings.Replace(vars["index"], "0x", "", -1)) + if err != nil { + validatorNotFound(data, w, r, vars, "") + return + } + errFields["pubKey"] = pubKey + index, err = db.GetValidatorIndex(pubKey) + if err != nil { + if err != sql.ErrNoRows { + utils.LogError(err, "error getting index for validator based on pubkey", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + // the validator might only have a public key but no index yet + var name string + err := db.ReaderDb.Get(&name, `SELECT name FROM validator_names WHERE publickey = $1`, pubKey) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + utils.LogError(err, "error getting validator-name from db for pubKey", 0, errFields) + validatorNotFound(data, w, r, vars, "") + return + // err == sql.ErrNoRows -> unnamed + } else { + validatorPageData.Name = name + } + + var entity string + err = db.ReaderDb.Get(&entity, `SELECT entity FROM validator_entities WHERE publickey = $1`, pubKey) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + utils.LogError(err, "error getting validator-pool from db for pubkey", 0, errFields) + validatorNotFound(data, w, r, vars, "") + return + // err == sql.ErrNoRows -> (no entity set) + } else { + if validatorPageData.Name == "" { + validatorPageData.Name = fmt.Sprintf("Entity: %s", entity) + } else if validatorPageData.Name != entity { // do not write out the entity name twice if they are the same + validatorPageData.Name += fmt.Sprintf(" / Entity: %s", entity) + } + } + deposits, err := db.GetValidatorDeposits(pubKey) + if err != nil { + utils.LogError(err, "error getting validator-deposits from db for pubkey", 0, errFields) + } + validatorPageData.DepositsCount = uint64(len(deposits.Eth1Deposits)) + validatorPageData.ShowMultipleWithdrawalCredentialsWarning = hasMultipleWithdrawalCredentials(deposits) + if err != nil || len(deposits.Eth1Deposits) == 0 { + validatorNotFound(data, w, r, vars, "") + return + } + + // there is no validator-index but there are eth1-deposits for the publickey + // which means the validator is in DEPOSITED state + // in this state there is nothing to display but the eth1-deposits + validatorPageData.Status = "deposited" + validatorPageData.PublicKey = pubKey + if deposits != nil && len(deposits.Eth1Deposits) > 0 { + deposits.LastEth1DepositTs = deposits.Eth1Deposits[len(deposits.Eth1Deposits)-1].BlockTs + } + validatorPageData.Deposits = deposits + + latestDeposit := time.Now().Unix() + if len(deposits.Eth1Deposits) > 1 { + } else if time.Unix(latestDeposit, 0).Before(utils.SlotToTime(0)) { + validatorPageData.InclusionDelay = 0 + } + + for _, deposit := range deposits.Eth1Deposits { + if deposit.ValidSignature { + validatorPageData.Eth1DepositAddress = deposit.FromAddress + break + } + } + + // check if an invalid deposit exists + for _, deposit := range deposits.Eth1Deposits { + if !deposit.Valid { // the valid flag takes into account all the unique cases with EL deposits + validatorPageData.Status = "deposited_invalid" + break + } + } + + filter := db.WatchlistFilter{ + UserId: data.User.UserID, + Validators: &pq.ByteaArray{validatorPageData.PublicKey}, + Tag: types.ValidatorTagsWatchlist, + JoinValidators: false, + Network: utils.GetNetwork(), + } + watchlist, err := db.GetTaggedValidators(filter) + if err != nil { + errFields["userID"] = data.User.UserID + utils.LogError(err, "error getting tagged validators from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + validatorPageData.Watchlist = watchlist + + if data.User.Authenticated { + events := make([]types.EventNameCheckbox, 0) + for _, ev := range types.AddWatchlistEvents { + events = append(events, types.EventNameCheckbox{ + EventLabel: ev.Desc, + EventName: ev.Event, + Active: false, + Warning: ev.Warning, + Info: ev.Info, + }) + } + validatorPageData.AddValidatorWatchlistModal = &types.AddValidatorWatchlistModal{ + Events: events, + ValidatorIndex: validatorPageData.Index, + CsrfField: csrf.TemplateField(r), + } + } + + g := errgroup.Group{} + g.Go(func() error { + if utils.ElectraHasHappened(validatorPageData.Epoch) { + // deposit processing queue + pendingDeposit, err := db.GetNextPendingDeposit(validatorPageData.PublicKey) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + var lastPendingDeposit types.PendingDeposit + err := db.ReaderDb.Get(&lastPendingDeposit, `SELECT id, est_clear_epoch, amount FROM pending_deposits_queue WHERE id = (select max(id) from pending_deposits_queue)`) + if err != nil { + logrus.Warnf("error getting pending deposits for validator %v: %v", validatorPageData.PublicKey, err) + return nil + } + // no queue position as deposit is too fresh, show estimates of last entry in the queue as rough estimate + validatorPageData.EstimatedActivationEpoch = lastPendingDeposit.EstClearEpoch + 1 + finalityDelay + utils.Config.Chain.ClConfig.MaxSeedLookahead + 1 + estimatedDequeueTs := utils.EpochToTime(validatorPageData.EstimatedActivationEpoch) + validatorPageData.EstimatedActivationTs = estimatedDequeueTs + validatorPageData.PendingDepositAboveMinActivation = true // assume for now + } else { + logrus.Warnf("error getting pending deposits for validator %v: %v", validatorPageData.PublicKey, err) + return nil + } + + return nil // can happen if the deposit is fresh and has not yet picked up by the beaconchain deposit queue + } + + validatorPageData.QueuePosition = uint64(pendingDeposit.ID) + 1 + validatorPageData.EstimatedActivationEpoch = pendingDeposit.EstClearEpoch + 1 + finalityDelay + utils.Config.Chain.ClConfig.MaxSeedLookahead + 1 + estimatedDequeueTs := utils.EpochToTime(validatorPageData.EstimatedActivationEpoch) + validatorPageData.EstimatedActivationTs = estimatedDequeueTs + validatorPageData.EstimatedIndexEpoch = pendingDeposit.EstClearEpoch + validatorPageData.EstimatedIndexTs = utils.EpochToTime(validatorPageData.EstimatedIndexEpoch) + validatorPageData.PendingDepositAboveMinActivation = pendingDeposit.Amount >= utils.Config.Chain.ClConfig.MinActivationBalance + } + return nil + }) + + err = g.Wait() + if err != nil { + utils.LogError(err, "error getting pending deposits for validator", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + data.Data = validatorPageData + if utils.IsApiRequest(r) { + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(data.Data) + } else { + err = validatorTemplate.ExecuteTemplate(w, "layout", data) + } + + if handleTemplateError(w, r, "validator.go", "Validator", "Done (no index)", err) != nil { + return // an error has occurred and was processed + } + return + } + } else { + // Request came with a validator index number + index, err = strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + validatorNotFound(data, w, r, vars, "") + return + } + } + + errFields["index"] = index + + SetPageDataTitle(data, fmt.Sprintf("Validator %v", index)) + data.Meta.Path = fmt.Sprintf("/validator/%v", index) + + // we use MAX(validatorindex)+1 instead of COUNT(*) for querying the rank_count for performance-reasons + err = db.ReaderDb.Get(&validatorPageData, ` + SELECT + validators.pubkey, + validators.validatorindex, + validators.withdrawableepoch, + validators.slashed, + validators.activationeligibilityepoch, + validators.activationepoch, + validators.exitepoch, + validators.withdrawalcredentials, + COALESCE(validator_names.name, '') AS name, + COALESCE(validator_entities.entity, '') AS entity, + COALESCE(validator_performance.rank7d, 0) AS rank7d, + COALESCE(validator_performance_count.total_count, 0) AS rank_count, + validators.status, + COALESCE(validators.balanceactivation, 0) AS balanceactivation, + COALESCE((SELECT ARRAY_AGG(tag) FROM validator_tags WHERE publickey = validators.pubkey),'{}') AS tags + FROM validators + LEFT JOIN validator_names ON validators.pubkey = validator_names.publickey + LEFT JOIN validator_entities ON validators.pubkey = validator_entities.publickey + LEFT JOIN validator_performance ON validators.validatorindex = validator_performance.validatorindex + LEFT JOIN (SELECT MAX(validatorindex)+1 FROM validator_performance WHERE validatorindex < 2147483647 AND validatorindex >= 0) validator_performance_count(total_count) ON true + WHERE validators.validatorindex = $1`, index) + + if errors.Is(err, sql.ErrNoRows) { + validatorNotFound(data, w, r, vars, "") + return + } else if err != nil { + utils.LogError(err, "error getting validator page data info from db", 0, errFields) + validatorNotFound(data, w, r, vars, "") + return + } + + lastAttestationSlots, err := db.BigtableClient.GetLastAttestationSlots([]uint64{index}) + if err != nil { + utils.LogError(err, "error getting last attestation slots from bigtable", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + validatorPageData.LastAttestationSlot = lastAttestationSlots[index] + + lastStatsDay, lastStatsDayErr := services.LatestExportedStatisticDay() + + timings.BasicInfo = time.Since(timings.Start) + + timings.Start = time.Now() + + if strings.HasPrefix(validatorPageData.Entity, "0x") && len(validatorPageData.Entity) == 42 { + validatorPageData.Entity = fmt.Sprintf("Whale_0x%s", validatorPageData.Entity[2:8]) + } + + if validatorPageData.Name != "" && validatorPageData.Entity == "" { + validatorPageData.Name = fmt.Sprintf("Name: %s", validatorPageData.Name) + } else if validatorPageData.Name == "" && validatorPageData.Entity != "" { + validatorPageData.Name = fmt.Sprintf("Entity: %s", validatorPageData.Entity) + } else if validatorPageData.Name == validatorPageData.Entity { + validatorPageData.Name = fmt.Sprintf("Entity: %s", validatorPageData.Entity) + } else { + validatorPageData.Name = fmt.Sprintf("Name: %s / Entity: %s", validatorPageData.Name, validatorPageData.Entity) + } + + if validatorPageData.Rank7d > 0 && validatorPageData.RankCount > 0 { + validatorPageData.RankPercentage = float64(validatorPageData.Rank7d) / float64(validatorPageData.RankCount) + } + + validatorPageData.Index = index + + if data.User.Authenticated { + events := make([]types.EventNameCheckbox, 0) + for _, ev := range types.AddWatchlistEvents { + events = append(events, types.EventNameCheckbox{ + EventLabel: ev.Desc, + EventName: ev.Event, + Active: false, + Warning: ev.Warning, + Info: ev.Info, + }) + } + validatorPageData.AddValidatorWatchlistModal = &types.AddValidatorWatchlistModal{ + Events: events, + ValidatorIndex: validatorPageData.Index, + CsrfField: csrf.TemplateField(r), + } + } + + validatorPageData.ActivationEligibilityTs = utils.EpochToTime(validatorPageData.ActivationEligibilityEpoch) + validatorPageData.ActivationTs = utils.EpochToTime(validatorPageData.ActivationEpoch) + validatorPageData.ExitTs = utils.EpochToTime(validatorPageData.ExitEpoch) + validatorPageData.WithdrawableTs = utils.EpochToTime(validatorPageData.WithdrawableEpoch) + + validatorWithdrawalAddress, err := utils.WithdrawalCredentialsToAddress(validatorPageData.WithdrawCredentials) + if err != nil { + if len(validatorPageData.WithdrawCredentials) <= 0 || !bytes.Equal(validatorPageData.WithdrawCredentials[:1], []byte{0x00}) { + utils.LogWarn(err, "error converting withdrawal credentials to address", 0, errFields) + } + } + + var lowerBoundDay uint64 + if lastStatsDay > 30 { + lowerBoundDay = lastStatsDay - 30 + } + gCtx, cancel := context.WithCancel(context.Background()) + defer cancel() // ensure resources cleaned up in case g.Wait is skipped + g, ctx := errgroup.WithContext(gCtx) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + start := time.Now() + defer func() { + timings.Charts = time.Since(start) + }() + + incomeHistoryChartData, err := db.GetValidatorIncomeHistoryChart([]uint64{index}, currency, lastFinalizedEpoch, lowerBoundDay) + if err != nil { + return fmt.Errorf("error calling db.GetValidatorIncomeHistoryChart: %w", err) + } + + if isPreGenesis { + incomeHistoryChartData = make([]*types.ChartDataPoint, 0) + } + + validatorPageData.IncomeHistoryChartData = incomeHistoryChartData + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + start := time.Now() + defer func() { + timings.Charts = time.Since(start) + }() + + executionIncomeHistoryData, err := getExecutionChartData([]uint64{index}, currency, lowerBoundDay) + if err != nil { + return fmt.Errorf("error calling getExecutionChartData: %w", err) + } + + validatorPageData.ExecutionIncomeHistoryData = executionIncomeHistoryData + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + start := time.Now() + defer func() { + timings.Charts = time.Since(start) + }() + + // Fetch last 30 days of beaconscore (efficiency) history for this validator + type chRow struct { + Ts int64 `db:"ts"` + Dividend float64 `db:"dividend"` + Divisor float64 `db:"divisor"` + } + var rows []chRow + err := db.ClickhouseReaderDb.Select(&rows, ` + SELECT toUnixTimestamp(toStartOfDay(t)) AS ts, + sum(efficiency_dividend) AS dividend, + sum(efficiency_divisor) AS divisor + FROM validator_dashboard_data_daily + WHERE validator_index = $1 AND t >= today() - 30 + GROUP BY ts + ORDER BY ts + `, index) + if err != nil { + return fmt.Errorf("error getting validator beaconscore history (30d): %w", err) + } + + // Build [timestamp_sec, efficiency] points, skipping rows with zero divisor + points := make([][2]float64, 0, len(rows)) + for _, r := range rows { + if r.Divisor <= 0 { + continue + } + eff := r.Dividend / r.Divisor + if eff < 0 { + eff = 0 + } + if eff > 1 { + eff = 1 + } + points = append(points, [2]float64{float64(r.Ts), eff}) + } + + if isPreGenesis { + points = points[:0] + } + + b, err := json.Marshal(points) + if err != nil { + return fmt.Errorf("error marshalling beaconscore history: %w", err) + } + validatorPageData.BeaconscoreChartData = template.JS(b) + return nil + }) + + currentBalanceCh := make(chan uint64, 1) + var avgSyncInterval uint64 + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + // those functions need to be executed sequentially as both require the CurrentBalance value + start := time.Now() + defer func() { + timings.Earnings = time.Since(start) + }() + defer close(currentBalanceCh) + earnings, balances, err := GetValidatorEarnings([]uint64{index}, currency) + if err != nil { + return fmt.Errorf("error getting validator earnings: %w", err) + } + validatorPageData.Income = earnings + + validatorPageData.ValidatorProposalData = earnings.ProposalData + + if latestEpoch < earnings.ProposalData.LastScheduledSlot/data.ChainConfig.SlotsPerEpoch { + futureProposalEpoch = earnings.ProposalData.LastScheduledSlot / data.ChainConfig.SlotsPerEpoch + } + + vbalance, ok := balances[validatorPageData.ValidatorIndex] + if ok { + validatorPageData.CurrentBalance = vbalance.Balance + validatorPageData.EffectiveBalance = vbalance.EffectiveBalance + currentBalanceCh <- vbalance.Balance + + avgSyncInterval = uint64(getAvgSyncCommitteeInterval(validatorPageData.EffectiveBalance / 1e9)) + avgSyncIntervalAsDuration := time.Duration( + utils.Config.Chain.ClConfig.SecondsPerSlot* + utils.SlotsPerSyncCommittee()* + avgSyncInterval) * time.Second + validatorPageData.AvgSyncInterval = &avgSyncIntervalAsDuration + } + + credentialsPrefixBytes := validatorPageData.WithdrawCredentials[:1] + if bytes.Equal(credentialsPrefixBytes, []byte{0x01}) || bytes.Equal(credentialsPrefixBytes, []byte{0x02}) { + // validators can have 0x01 credentials even before the cappella fork + validatorPageData.IsWithdrawableAddress = true + } + + if validatorPageData.CappellaHasHappened { + // if we are currently past the cappella fork epoch, we can calculate the withdrawal information + validatorSlice := []uint64{index} + withdrawalsCount, err := db.GetTotalWithdrawalsCount(validatorSlice) + if err != nil { + return fmt.Errorf("error getting validator withdrawals count from db: %w", err) + } + validatorPageData.WithdrawalCount = withdrawalsCount + lastWithdrawalsEpochs, err := db.GetLastWithdrawalEpoch(validatorSlice) + if err != nil { + return fmt.Errorf("error getting validator last withdrawal epoch from db: %w", err) + } + lastWithdrawalsEpoch := lastWithdrawalsEpochs[index] + + blsChange, err := db.GetValidatorBLSChange(validatorPageData.Index) + if err != nil { + return fmt.Errorf("error getting validator bls change from db: %w", err) + } + validatorPageData.BLSChange = blsChange + + if bytes.Equal(credentialsPrefixBytes, []byte{0x00}) && blsChange != nil { + // blsChanges are only possible afters cappeala + validatorPageData.IsWithdrawableAddress = true + } + + // only calculate the expected next withdrawal if the validator is eligible + maxEB := utils.GetMaxEffectiveBalanceByWithdrawalCredentials(validatorPageData.WithdrawCredentials) + isFullWithdrawal := validatorPageData.CurrentBalance > 0 && validatorPageData.WithdrawableEpoch <= validatorPageData.Epoch + isPartialWithdrawal := validatorPageData.EffectiveBalance == maxEB && validatorPageData.CurrentBalance > maxEB + if stats != nil && stats.LatestValidatorWithdrawalIndex != nil && stats.TotalValidatorCount != nil && validatorPageData.IsWithdrawableAddress && (isFullWithdrawal || isPartialWithdrawal) { + distance, err := GetWithdrawableCountFromCursor(validatorPageData.Epoch, validatorPageData.Index, *stats.LatestValidatorWithdrawalIndex) + if err != nil { + return fmt.Errorf("error getting withdrawable validator count from cursor: %w", err) + } + + timeToWithdrawal := utils.GetTimeToNextWithdrawal(distance) + + // it normally takes two epochs to finalize + if timeToWithdrawal.After(utils.EpochToTime(latestEpoch + (latestEpoch - lastFinalizedEpoch))) { + address, err := utils.WithdrawalCredentialsToAddress(validatorPageData.WithdrawCredentials) + if err != nil { + // warning only as "N/A" will be displayed + logger.Warn("invalid withdrawal credentials") + } + + // create the table data + tableData := make([][]interface{}, 0, 1) + var withdrawalCredentialsTemplate template.HTML + if address != nil { + withdrawalCredentialsTemplate = template.HTML(fmt.Sprintf(`%s`, address, utils.FormatAddress(address, nil, "", false, false, true))) + } else { + withdrawalCredentialsTemplate = `N/A` + } + + var withdrawalAmount uint64 + if isFullWithdrawal { + withdrawalAmount = validatorPageData.CurrentBalance + } else { + withdrawalAmount = validatorPageData.CurrentBalance - maxEB + } + + if latestEpoch == lastWithdrawalsEpoch { + withdrawalAmount = 0 + } + tableData = append(tableData, []interface{}{ + template.HTML(fmt.Sprintf(`~ %s`, utils.FormatEpoch(uint64(utils.TimeToEpoch(timeToWithdrawal))))), + template.HTML(fmt.Sprintf(`~ %s`, utils.FormatBlockSlot(utils.TimeToSlot(uint64(timeToWithdrawal.Unix()))))), + template.HTML(fmt.Sprintf(` ~ %s`, utils.FormatTimestamp(timeToWithdrawal.Unix()))), + withdrawalCredentialsTemplate, + template.HTML(fmt.Sprintf(` %s`, utils.FormatClCurrency(withdrawalAmount, currency, 6, true, false, false, true))), + }) + + validatorPageData.NextWithdrawalRow = tableData + validatorPageData.NextWithdrawalTs = timeToWithdrawal.Unix() + } + } + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + filter := db.WatchlistFilter{ + UserId: data.User.UserID, + Validators: &pq.ByteaArray{validatorPageData.PublicKey}, + Tag: types.ValidatorTagsWatchlist, + JoinValidators: false, + Network: utils.GetNetwork(), + } + + watchlist, err := db.GetTaggedValidators(filter) + if err != nil { + return fmt.Errorf("error getting tagged validators from db: %w", err) + } + + validatorPageData.Watchlist = watchlist + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + start := time.Now() + defer func() { + timings.Deposits = time.Since(start) + }() + deposits, err := db.GetValidatorDeposits(validatorPageData.PublicKey) + if err != nil { + return fmt.Errorf("error getting validator-deposits from db: %w", err) + } + validatorPageData.Deposits = deposits + validatorPageData.DepositsCount = uint64(len(deposits.Eth1Deposits)) + + for _, deposit := range validatorPageData.Deposits.Eth1Deposits { + if deposit.ValidSignature { + validatorPageData.Eth1DepositAddress = deposit.FromAddress + break + } + } + + validatorPageData.ShowMultipleWithdrawalCredentialsWarning = hasMultipleWithdrawalCredentials(validatorPageData.Deposits) + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + if utils.ElectraHasHappened(validatorPageData.Epoch) { + var isPendingActivation = false + + theoreticalActivationEligibilityEpoch := validatorPageData.ActivationEligibilityEpoch // does not consider if deposit actually is above min activation balance + if validatorPageData.ActivationEpoch > 100_000_000 && validatorPageData.ActivationEligibilityEpoch < 100_000_000 { + isPendingActivation = true + validatorPageData.PendingDepositAboveMinActivation = true // already has eligibility + } else if validatorPageData.ActivationEpoch > 100_000_000 { + // Validator either has just processed a deposit and gets their eligibility in the next epoch or + // validator is missing balance for activation + pendingDeposit, err := db.GetNextPendingDeposit(validatorPageData.PublicKey) + if err != nil { + logrus.Warnf("error getting pending deposits for validator %v: %v", validatorPageData.PublicKey, err) + return nil + } + + select { + case currentBalance := <-currentBalanceCh: + validatorPageData.PendingDepositAboveMinActivation = currentBalance+pendingDeposit.Amount >= utils.Config.Chain.ClConfig.MinActivationBalance + case <-ctx.Done(): + return ctx.Err() + } + + theoreticalActivationEligibilityEpoch = pendingDeposit.EstClearEpoch + 1 + if !validatorPageData.PendingDepositAboveMinActivation { + // countdown to when the deposit will be processed if it won't become eligible + validatorPageData.EstimatedActivationTs = utils.EpochToTime(pendingDeposit.EstClearEpoch) + } else { + validatorPageData.ActivationEligibilityEpoch = theoreticalActivationEligibilityEpoch + } + + isPendingActivation = true + } + + if isPendingActivation { + validatorPageData.QueuePosition = 0 + validatorPageData.EstimatedActivationEpoch = theoreticalActivationEligibilityEpoch + finalityDelay + utils.Config.Chain.ClConfig.MaxSeedLookahead + 1 + if validatorPageData.EstimatedActivationTs.IsZero() { + validatorPageData.EstimatedActivationTs = utils.EpochToTime(validatorPageData.EstimatedActivationEpoch) + } + + validatorPageData.EstimatedIndexEpoch = theoreticalActivationEligibilityEpoch - 1 + validatorPageData.EstimatedIndexTs = utils.EpochToTime(validatorPageData.EstimatedIndexEpoch) + } + } else { + + // we only need to get the queue information if we don't have an activation epoch but we have an eligibility epoch + if validatorPageData.ActivationEpoch > 100_000_000 && validatorPageData.ActivationEligibilityEpoch < 100_000_000 { + queueAhead, err := db.GetQueueAheadOfValidator(validatorPageData.Index) + if err != nil { + return fmt.Errorf("failed to retrieve queue ahead of validator %v: %w", validatorPageData.ValidatorIndex, err) + } + validatorPageData.QueuePosition = queueAhead + 1 + epochsToWait := queueAhead / *activationChurnRate + // calculate dequeue epoch + estimatedActivationEpoch := validatorPageData.Epoch + epochsToWait + 1 + // add activation offset + estimatedActivationEpoch += utils.Config.Chain.ClConfig.MaxSeedLookahead + 1 + validatorPageData.EstimatedActivationEpoch = estimatedActivationEpoch + estimatedDequeueTs := utils.EpochToTime(estimatedActivationEpoch) + validatorPageData.EstimatedActivationTs = estimatedDequeueTs + } + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + // Every validator is scheduled to issue an attestation once per epoch + // Hence we can calculate the number of attestations using the current epoch and the activation epoch + // Special care needs to be take for exited and pending validators + if validatorPageData.ExitEpoch != 9223372036854775807 && validatorPageData.ExitEpoch <= validatorPageData.Epoch { + validatorPageData.AttestationsCount = validatorPageData.ExitEpoch - validatorPageData.ActivationEpoch + } else if validatorPageData.ActivationEpoch > validatorPageData.Epoch { + validatorPageData.AttestationsCount = 0 + + return nil + } else if isPreGenesis { + validatorPageData.AttestationsCount = 1 + validatorPageData.MissedAttestationsCount = 0 + validatorPageData.ExecutedAttestationsCount = 0 + validatorPageData.UnmissedAttestationsPercentage = 1 + + return nil + } else { + validatorPageData.AttestationsCount = validatorPageData.Epoch - validatorPageData.ActivationEpoch + 1 + + // Check if the latest epoch still needs to be attested (scheduled) and if so do not count it + attestationData, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{index}, validatorPageData.Epoch, validatorPageData.Epoch) + if err != nil { + return fmt.Errorf("error getting validator attestations data for epoch [%v]: %w", validatorPageData.Epoch, err) + } + + if len(attestationData[index]) > 0 && attestationData[index][0].Status == 0 { + validatorPageData.AttestationsCount-- + } + } + + if validatorPageData.AttestationsCount > 0 { + // get attestationStats from validator_stats + attestationStats := struct { + MissedAttestations uint64 `db:"missed_attestations"` + }{} + if lastStatsDay > 0 { + err := db.ReaderDb.Get(&attestationStats, "SELECT missed_attestations_total AS missed_attestations FROM validator_stats WHERE validatorindex = $1 AND day = $2", index, lastStatsDay) + if err == sql.ErrNoRows { + logger.Warningf("no entry in validator_stats for validator index %v while lastStatsDay = %v", index, lastStatsDay) + } else if err != nil { + return fmt.Errorf("error getting validator attestationStats while lastStatsDay = %v: %w", lastStatsDay, err) + } + } + + // add attestationStats that are not yet in validator_stats (if any) + nextStatsDayFirstEpoch, _ := utils.GetFirstAndLastEpochForDay(lastStatsDay + 1) + if validatorPageData.Epoch > nextStatsDayFirstEpoch { + lookback := validatorPageData.Epoch - nextStatsDayFirstEpoch + missedAttestations, err := db.BigtableClient.GetValidatorMissedAttestationHistory([]uint64{index}, validatorPageData.Epoch-lookback, validatorPageData.Epoch-1) + if err != nil { + return fmt.Errorf("error getting validator attestations not in stats from bigtable: %w", err) + } + attestationStats.MissedAttestations += uint64(len(missedAttestations[index])) + } + + if attestationStats.MissedAttestations > validatorPageData.AttestationsCount { + // save guard against negative values (should never happen but happened once because of wrong data) + attestationStats.MissedAttestations = validatorPageData.AttestationsCount + } + validatorPageData.MissedAttestationsCount = attestationStats.MissedAttestations + validatorPageData.ExecutedAttestationsCount = validatorPageData.AttestationsCount - validatorPageData.MissedAttestationsCount + validatorPageData.UnmissedAttestationsPercentage = float64(validatorPageData.ExecutedAttestationsCount) / float64(validatorPageData.AttestationsCount) + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + var err error + if validatorPageData.Slashed { + var slashingInfo struct { + Slot uint64 + Slasher uint64 + Reason string + } + err = db.ReaderDb.Get(&slashingInfo, + `SELECT block_slot AS slot, proposer AS slasher, 'Attestation Violation' AS reason + FROM blocks_attesterslashings a1 LEFT JOIN blocks b1 ON b1.slot = a1.block_slot + WHERE b1.status = '1' AND $1 = ANY(a1.attestation1_indices) AND $1 = ANY(a1.attestation2_indices) + UNION ALL + SELECT block_slot AS slot, proposer AS slasher, 'Proposer Violation' AS reason + FROM blocks_proposerslashings a2 LEFT JOIN blocks b2 ON b2.slot = a2.block_slot + WHERE b2.status = '1' AND a2.proposerindex = $1 + LIMIT 1`, + index) + if err != nil { + return fmt.Errorf("error getting validator slashing info: %w", err) + } + validatorPageData.SlashedBy = slashingInfo.Slasher + validatorPageData.SlashedAt = slashingInfo.Slot + validatorPageData.SlashedFor = slashingInfo.Reason + } + + err = db.ReaderDb.Get(&validatorPageData.SlashingsCount, `SELECT COALESCE(SUM(attesterslashingscount) + SUM(proposerslashingscount), 0) FROM blocks WHERE blocks.proposer = $1 AND blocks.status = '1'`, index) + if err != nil { + return fmt.Errorf("error getting slashings-count: %w", err) + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + efficiency := struct { + Dividend decimal.Decimal `db:"dividend"` + Divisor decimal.Decimal `db:"divisor"` + }{} + + err := db.ClickhouseReaderDb.Get(&efficiency, "select sum(efficiency_dividend) AS dividend, SUM(efficiency_divisor) AS divisor from _final_validator_dashboard_rolling_30d where validator_index = $1", index) + if err != nil { + return fmt.Errorf("error getting validator effectiveness: %w", err) + } + + validatorPageData.Beaconscore = utils.CalcEfficiency(efficiency.Dividend, efficiency.Divisor) + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + validatorPageData.SlotsPerSyncCommittee = utils.SlotsPerSyncCommittee() + + // sync participation + // get all sync periods this validator has been part of + var actualSyncPeriods []struct { + Period uint64 `db:"period"` + FirstEpoch uint64 `db:"firstepoch"` + LastEpoch uint64 `db:"lastepoch"` + } + allSyncPeriods := actualSyncPeriods + + err := db.ReaderDb.Select(&allSyncPeriods, ` + SELECT period, GREATEST(period*$1, $2) AS firstepoch, ((period+1)*$1)-1 AS lastepoch + FROM sync_committees + WHERE validatorindex = $3 + ORDER BY period desc`, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, utils.Config.Chain.ClConfig.AltairForkEpoch, index) + if err != nil { + return fmt.Errorf("error getting sync participation count data of sync-assignments: %w", err) + } + + if len(allSyncPeriods) > 0 && allSyncPeriods[0].LastEpoch > latestEpoch { + futureSyncDutyEpoch = allSyncPeriods[0].LastEpoch + } + + // remove scheduled committees + for i, syncPeriod := range allSyncPeriods { + if syncPeriod.FirstEpoch <= latestEpoch { + actualSyncPeriods = allSyncPeriods[i:] + break + } + } + + if len(actualSyncPeriods) > 0 { + // get sync stats from validator_stats + syncStats := types.SyncCommitteesStats{} + if lastStatsDay > 0 { + err = db.ReaderDb.Get(&syncStats, ` + SELECT + COALESCE(participated_sync_total, 0) AS participated_sync, + COALESCE(missed_sync_total, 0) AS missed_sync, + COALESCE(orphaned_sync_total, 0) AS orphaned_sync + FROM validator_stats + WHERE validatorindex = $1 AND day = $2`, index, lastStatsDay) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("error getting validator syncStats: %w", err) + } + } + + // if sync duties of last period haven't fully been exported yet, fetch remaining duties from bigtable + lastExportedEpoch := (lastStatsDay+1)*utils.EpochsPerDay() - 1 + + if lastStatsDayErr == db.ErrNoStats { + lastExportedEpoch = 0 + } + lastSyncPeriod := actualSyncPeriods[0] + if lastSyncPeriod.LastEpoch > lastExportedEpoch { + res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch, latestProposedSlot) + if err != nil { + return fmt.Errorf("error getting validator sync participations data from bigtable: %w", err) + } + syncStatsBt := utils.AddSyncStats([]uint64{index}, res, nil) + // if last sync period is the current one, add remaining scheduled slots + if lastSyncPeriod.LastEpoch >= latestEpoch { + syncStatsBt.ScheduledSlots += utils.GetRemainingScheduledSyncDuties(1, syncStatsBt, lastExportedEpoch, lastSyncPeriod.FirstEpoch) + } + + syncStats.MissedSlots += syncStatsBt.MissedSlots + syncStats.ParticipatedSlots += syncStatsBt.ParticipatedSlots + syncStats.ScheduledSlots += syncStatsBt.ScheduledSlots + } + validatorPageData.SlotsDoneInCurrentSyncCommittee = validatorPageData.SlotsPerSyncCommittee - syncStats.ScheduledSlots + + validatorPageData.ParticipatedSyncCountSlots = syncStats.ParticipatedSlots + validatorPageData.MissedSyncCountSlots = syncStats.MissedSlots + validatorPageData.OrphanedSyncCountSlots = syncStats.OrphanedSlots + validatorPageData.ScheduledSyncCountSlots = syncStats.ScheduledSlots + // actual sync duty count and percentage + validatorPageData.SyncCount = uint64(len(actualSyncPeriods)) + validatorPageData.UnmissedSyncPercentage = float64(validatorPageData.ParticipatedSyncCountSlots) / float64(validatorPageData.ParticipatedSyncCountSlots+validatorPageData.MissedSyncCountSlots+validatorPageData.OrphanedSyncCountSlots) + } + // sync luck + if len(allSyncPeriods) > 0 { + maxPeriod := allSyncPeriods[0].Period + nextEstimate := utils.EpochToTime(utils.FirstEpochOfSyncPeriod(maxPeriod + avgSyncInterval)) + validatorPageData.SyncEstimate = &nextEstimate + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + // add rocketpool-data if available + validatorPageData.Rocketpool = &types.RocketpoolValidatorPageData{} + err := db.ReaderDb.Get(validatorPageData.Rocketpool, ` + SELECT + rplm.node_address AS node_address, + rplm.address AS minipool_address, + rplm.node_fee AS minipool_node_fee, + rplm.deposit_type AS minipool_deposit_type, + rplm.status AS minipool_status, + rplm.status_time AS minipool_status_time, + COALESCE(rplm.penalty_count,0) AS penalty_count, + rpln.timezone_location AS node_timezone_location, + rpln.rpl_stake AS node_rpl_stake, + rpln.max_rpl_stake AS node_max_rpl_stake, + rpln.min_rpl_stake AS node_min_rpl_stake, + rpln.rpl_cumulative_rewards AS rpl_cumulative_rewards, + rpln.claimed_smoothing_pool AS claimed_smoothing_pool, + rpln.unclaimed_smoothing_pool AS unclaimed_smoothing_pool, + rpln.unclaimed_rpl_rewards AS unclaimed_rpl_rewards, + COALESCE(node_deposit_balance, 0) AS node_deposit_balance, + COALESCE(node_refund_balance, 0) AS node_refund_balance, + COALESCE(user_deposit_balance, 0) AS user_deposit_balance, + COALESCE(rpln.effective_rpl_stake, 0) AS effective_rpl_stake, + COALESCE(deposit_credit, 0) AS deposit_credit, + COALESCE(is_vacant, false) AS is_vacant, + version, + COALESCE(rpln.smoothing_pool_opted_in, false) AS smoothing_pool_opted_in + FROM validators + LEFT JOIN rocketpool_minipools rplm ON rplm.pubkey = validators.pubkey + LEFT JOIN rocketpool_nodes rpln ON rplm.node_address = rpln.address + WHERE validators.validatorindex = $1 + ORDER BY rplm.status_time DESC + LIMIT 1`, index) + if err == nil && (validatorPageData.Rocketpool.MinipoolAddress != nil || validatorPageData.Rocketpool.NodeAddress != nil) { + validatorPageData.IsRocketpool = true + if utils.Config.Chain.ClConfig.DepositChainID == 1 { + validatorPageData.Rocketpool.RocketscanUrl = "rocketscan.io" + } else if utils.Config.Chain.ClConfig.DepositChainID == 5 { + validatorPageData.Rocketpool.RocketscanUrl = "prater.rocketscan.io" + } + } else if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("error getting rocketpool-data for validator for %v route: %w", r.URL.String(), err) + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + err = db.ReaderDb.Select(&validatorPageData.ConsolidationRequests, ` + SELECT + slot_processed as block_slot, + block_processed_root as block_root, + index_processed as request_index, + sv.validatorindex as source_index, + tv.validatorindex as target_index, + COALESCE(amount_consolidated, 0) AS amount_consolidated + FROM blocks_consolidation_requests_v2 + INNER JOIN blocks ON blocks_consolidation_requests_v2.block_processed_root = blocks.blockroot AND blocks.status = '1' + INNER JOIN validators sv ON (sv.pubkey = source_pubkey) + INNER JOIN validators tv ON (tv.pubkey = target_pubkey) + WHERE + ( source_pubkey = (select pubkey from validators where validatorindex = $1) + OR target_pubkey = (select pubkey from validators where validatorindex = $1) + ) + AND blocks_consolidation_requests_v2.status = 'completed' + ORDER BY slot_processed DESC, index_processed`, index) + if err != nil { + return fmt.Errorf("error retrieving blocks_consolidation_requests_v2 of validator %v: %v", validatorPageData.Index, err) + } + + // find the consolidation target index + for _, cr := range validatorPageData.ConsolidationRequests { + if cr.SourceIndex == int64(validatorPageData.Index) { + validatorPageData.ConsolidationTargetIndex = cr.TargetIndex + break + } + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + validatorPageData.MoveToCompoundingRequest = &types.FrontendMoveToCompoundingRequest{} + // TODO: remove v1 table dependency once eth1id resolving is available + // See https://bitfly1.atlassian.net/browse/BEDS-1522 + err = db.ReaderDb.Get(validatorPageData.MoveToCompoundingRequest, ` + SELECT + slot_processed as block_slot, + block_processed_root as block_root, + index_processed as request_index, + v.validatorindex as validator_index, + COALESCE(v1.address, decode('0000000000000000000000000000000000000000', 'hex')) as address + FROM blocks_switch_to_compounding_requests_v2 + INNER JOIN blocks ON blocks_switch_to_compounding_requests_v2.block_processed_root = blocks.blockroot AND blocks.status = '1' + INNER JOIN validators v ON (v.pubkey = validator_pubkey) + LEFT JOIN blocks_switch_to_compounding_requests v1 ON (blocks_switch_to_compounding_requests_v2.slot_processed = v1.block_slot AND blocks_switch_to_compounding_requests_v2.block_processed_root = v1.block_root AND blocks_switch_to_compounding_requests_v2.index_processed = v1.request_index) + WHERE v.validatorindex = $1 + AND blocks_switch_to_compounding_requests_v2.status = 'completed' + ORDER BY slot_processed DESC, index_processed LIMIT 1`, index) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + validatorPageData.MoveToCompoundingRequest = nil + return nil + } + return fmt.Errorf("error retrieving blocks_switch_to_compounding_requests of validator %v: %v", validatorPageData.Index, err) + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + err = db.ReaderDb.Select(&validatorPageData.ConsensusElExits, ` + SELECT + slot_processed AS slot, + block_processed_root AS block_root, + blocks_exit_requests.status, + COALESCE(reject_reason, '') AS reject_reason, + validator_pubkey, + 'Execution Layer' as triggered_via + FROM blocks_exit_requests + INNER JOIN blocks ON blocks_exit_requests.block_processed_root = blocks.blockroot AND blocks.status = '1' + WHERE blocks_exit_requests.validator_pubkey = $1 + + UNION ALL + + SELECT + block_slot AS slot, + block_root, + 'completed' AS status, + '' AS reject_reason, + validatorindex_pubkey.pubkey AS validator_pubkey, + 'Consensus Layer' as triggered_via + FROM blocks_voluntaryexits + INNER JOIN validators AS validatorindex_pubkey ON blocks_voluntaryexits.validatorindex = validatorindex_pubkey.validatorindex + INNER JOIN blocks ON blocks_voluntaryexits.block_root = blocks.blockroot AND blocks.status = '1' + WHERE validatorindex_pubkey.pubkey = $1 + ORDER BY slot DESC + `, validatorPageData.PublicKey) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + validatorPageData.ConsensusElExits = nil + return nil + } + return fmt.Errorf("error retrieving blocks_exit_requests/blocks_voluntaryexits of validator %v: %v", validatorPageData.Index, err) + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + err = db.ReaderDb.Select(&validatorPageData.ExecutionWithdrawals, ` + SELECT + source_address, + tx_hash, + block_number, + extract(epoch from block_ts)::int as block_ts, + validatorindex as validator_index, + amount + FROM eth1_withdrawal_requests + INNER JOIN validators ON eth1_withdrawal_requests.validator_pubkey = validators.pubkey + WHERE validatorindex = $1 + ORDER BY block_number DESC, tx_index desc, itx_index desc`, index) + if err != nil { + return fmt.Errorf("error retrieving eth1_withdrawal_requests of validator %v: %v", validatorPageData.Index, err) + } + + for _, ew := range validatorPageData.ExecutionWithdrawals { + if !bytes.Equal(ew.SourceAddress.Bytes(), validatorWithdrawalAddress) { + ew.WrongSourceAddress = true + } + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + err = db.ReaderDb.Select(&validatorPageData.ExecutionConsolidations, ` + SELECT + ecr.source_address, + ecr.tx_hash, + ecr.block_number, + extract(epoch from ecr.block_ts)::int as block_ts, + s.validatorindex AS source_validator_index, + t.validatorindex AS target_validator_index, + s.withdrawalcredentials AS source_withdrawalcredentials + FROM eth1_consolidation_requests ecr + INNER JOIN validators s ON ecr.source_pubkey = s.pubkey + INNER JOIN validators t ON ecr.target_pubkey = t.pubkey + WHERE source_pubkey = (select pubkey from validators where validatorindex = $1) + OR target_pubkey = (select pubkey from validators where validatorindex = $1) + ORDER BY ecr.block_number DESC, ecr.tx_index DESC, ecr.itx_index DESC + `, index) + if err != nil { + return fmt.Errorf("error retrieving eth1_consolidation_requests of validator %v: %v", validatorPageData.Index, err) + } + + for _, ew := range validatorPageData.ExecutionConsolidations { + sourceWithdrawalAddress, err := utils.WithdrawalCredentialsToAddress(ew.SourceWithdrawalCredentials) + if err != nil { + logger.Warnf("error converting withdrawal credentials to address: %v", err) + continue + } + if !bytes.Equal(ew.SourceAddress.Bytes(), sourceWithdrawalAddress) { + ew.WrongSourceAddress = true + } + } + return nil + }) + + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + // TODO: remove v1 table dependency once eth1id resolving is available + // See https://bitfly1.atlassian.net/browse/BEDS-1522 + err = db.ReaderDb.Select(&validatorPageData.WithdrawalRequests, ` + SELECT + slot_processed as block_slot, + block_processed_root as block_root, + index_processed as request_index, + COALESCE(v1.source_address, decode('0000000000000000000000000000000000000000', 'hex')) as source_address, + blocks_withdrawal_requests_v2.amount + FROM blocks_withdrawal_requests_v2 + INNER JOIN blocks ON blocks_withdrawal_requests_v2.block_processed_root = blocks.blockroot AND blocks.status = '1' + LEFT JOIN blocks_withdrawal_requests v1 ON (blocks_withdrawal_requests_v2.slot_processed = v1.block_slot AND blocks_withdrawal_requests_v2.block_processed_root = v1.block_root AND blocks_withdrawal_requests_v2.index_processed = v1.request_index) + WHERE blocks_withdrawal_requests_v2.validator_pubkey = $1 + AND blocks_withdrawal_requests_v2.status = 'completed' + ORDER BY slot_processed DESC, index_processed`, validatorPageData.PublicKey) + if err != nil { + return fmt.Errorf("error retrieving blocks_withdrawal_requests_v2 of validator %v: %v", validatorPageData.Index, err) + } + + for _, wr := range validatorPageData.WithdrawalRequests { + if wr.Amount == 0 { + wr.Type = "Exit" + } else { + wr.Type = "Withdrawal" + } + } + return nil + }) + + var withdrawalCount uint64 + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + withdrawalCount, err = db.GetTotalWithdrawalsCount([]uint64{validatorPageData.Index}) + if err != nil { + return fmt.Errorf("error getting withdrawal count: %w", err) + } + return nil + }) + + err = g.Wait() + if err != nil { + utils.LogError(err, "error getting validator data", 0) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + validatorPageData.EnableWithdrawalsTab = validatorPageData.CappellaHasHappened && (withdrawalCount > 0 || len(validatorPageData.ExecutionWithdrawals) > 0 || len(validatorPageData.ConsensusElExits) > 0) + validatorPageData.FutureDutiesEpoch = protomath.MaxU64(futureProposalEpoch, futureSyncDutyEpoch) + + data.Data = validatorPageData + + if utils.IsApiRequest(r) { + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(data.Data) + } else { + err = validatorTemplate.ExecuteTemplate(w, "layout", data) + } + + if handleTemplateError(w, r, "validator.go", "Validator", "Done", err) != nil { + return // an error has occurred and was processed + } +} + +// Returns true if there are more than one different withdrawal credentials within both Eth1Deposits and Eth2Deposits +func hasMultipleWithdrawalCredentials(deposits *types.ValidatorDeposits) bool { + // temporarily disable this check as defined in ticket BEDS-1252 + return false + /* + if deposits == nil { + return false + } + + credential := make([]byte, 0) + + if deposits == nil { + return false + } + + // check Eth1Deposits + + for _, deposit := range deposits.Eth1Deposits { + if len(credential) == 0 { + credential = deposit.WithdrawalCredentials + } else if !bytes.Equal(credential, deposit.WithdrawalCredentials) { + return true + } + } + + // check Eth2Deposits + + for _, deposit := range deposits.Eth2Deposits { + if len(credential) == 0 { + credential = deposit.Withdrawalcredentials + } else if !bytes.Equal(credential, deposit.Withdrawalcredentials) { + return true + } + } + + return false + */ +} + +// ValidatorDeposits returns a validator's deposits in json +func ValidatorDeposits(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + vars := mux.Vars(r) + + pubkey, err := hex.DecodeString(strings.Replace(vars["pubkey"], "0x", "", -1)) + if err != nil { + logger.Warnf("error parsing validator public key %v: %v", vars["pubkey"], err) + http.Error(w, "Error: Invalid parameter public key.", http.StatusBadRequest) + return + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "pubkey": pubkey} + + deposits, err := db.GetValidatorDeposits(pubkey) + if err != nil { + utils.LogError(err, "error getting validator-deposits from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + err = json.NewEncoder(w).Encode(deposits) + if err != nil { + utils.LogError(err, "error encoding validator-deposits", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// ValidatorAttestationInclusionEffectiveness returns a validator's effectiveness in json +func ValidatorAttestationInclusionEffectiveness(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + epoch := services.LatestEpoch() + if epoch > 0 { + epoch = epoch - 1 + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "epoch": epoch} + + eff, err := db.BigtableClient.GetValidatorEffectiveness([]uint64{index}, epoch) + if err != nil { + utils.LogError(err, "error getting validator effectiveness from bigtable", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + type resp struct { + Effectiveness float64 `json:"effectiveness"` + } + + errFields["effectiveness length"] = len(eff) + if len(eff) > 1 { + utils.LogError(err, "error getting validator effectiveness because of invalid length", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } else if len(eff) == 0 { + err = json.NewEncoder(w).Encode(resp{Effectiveness: 0}) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + } else { + err = json.NewEncoder(w).Encode(resp{Effectiveness: eff[0].AttestationEfficiency}) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + } + +} + +// ValidatorProposedBlocks returns a validator's proposed blocks in json +func ValidatorProposedBlocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + start, err := strconv.ParseUint(q.Get("start"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) + return + } + length, err := strconv.ParseUint(q.Get("length"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) + return + } + if length > 100 { + length = 100 + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "draw": draw, + "start": start, + "length": length} + + var totalCount uint64 + + err = db.ReaderDb.Get(&totalCount, "SELECT COUNT(*) FROM blocks WHERE proposer = $1", index) + if err != nil { + utils.LogError(err, "error getting proposed blocks count from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + orderColumn := q.Get("order[0][column]") + orderByMap := map[string]string{ + "0": "epoch", + "2": "status", + "5": "attestationscount", + "6": "depositscount", + "8": "voluntaryexitscount", + "9": "graffiti", + } + orderBy, exists := orderByMap[orderColumn] + if !exists { + orderBy = "epoch" + } + orderDir := q.Get("order[0][dir]") + if orderDir != "desc" && orderDir != "asc" { + orderDir = "desc" + } + + var blocks []*types.IndexPageDataBlocks + err = db.ReaderDb.Select(&blocks, ` + SELECT + epoch, + slot, + proposer, + blockroot, + parentroot, + attestationscount, + depositscount, + COALESCE(withdrawalcount,0) as withdrawalcount, + voluntaryexitscount, + proposerslashingscount, + attesterslashingscount, + status, + graffiti + FROM blocks + WHERE proposer = $1 + ORDER BY `+orderBy+` `+orderDir+` + LIMIT $2 OFFSET $3`, index, length, start) + + if err != nil { + utils.LogError(err, "error getting proposed blocks data from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + tableData := make([][]interface{}, len(blocks)) + for i, b := range blocks { + tableData[i] = []interface{}{ + utils.FormatEpoch(b.Epoch), + utils.FormatBlockSlot(b.Slot), + utils.FormatBlockStatus(b.Status, b.Slot), + utils.FormatTimestamp(utils.SlotToTime(b.Slot).Unix()), + utils.FormatBlockRoot(b.BlockRoot), + b.Attestations, + b.Deposits, + fmt.Sprintf("%v / %v", b.Proposerslashings, b.Attesterslashings), + b.Exits, + utils.FormatGraffiti(b.Graffiti), + } + } + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: totalCount, + RecordsFiltered: totalCount, + Data: tableData, + } + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// ValidatorAttestations returns a validators attestations in json +func ValidatorAttestations(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + start, err := strconv.ParseInt(q.Get("start"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) + return + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "draw": draw, + "start": start} + + length := 10 + + epoch := services.LatestEpoch() + + ae := struct { + ActivationEpoch uint64 + ExitEpoch uint64 + }{} + + err = db.ReaderDb.Get(&ae, "SELECT activationepoch, exitepoch FROM validators WHERE validatorindex = $1", index) + if err != nil { + utils.LogError(err, "error getting attestations count from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + totalCount := epoch - ae.ActivationEpoch + 1 + if ae.ActivationEpoch > epoch { + totalCount = 0 + } + lastAttestationEpoch := epoch + if ae.ExitEpoch != 9223372036854775807 && ae.ExitEpoch <= epoch { + lastAttestationEpoch = ae.ExitEpoch - 1 + totalCount = ae.ExitEpoch - ae.ActivationEpoch + } + + tableData := [][]interface{}{} + + if totalCount > 0 { + endEpoch := int64(lastAttestationEpoch) - start + if endEpoch < 0 { + endEpoch = 0 + } + + startEpoch := endEpoch - int64(length) + 1 + if startEpoch < 0 { + startEpoch = 0 + } + + attestationData, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{index}, uint64(startEpoch), uint64(endEpoch)) + if err != nil { + errFields["startEpoch"] = startEpoch + errFields["endEpoch"] = endEpoch + utils.LogError(err, "error getting validator attestations data from bigtable", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + tableData = make([][]interface{}, len(attestationData[index])) + + for i, history := range attestationData[index] { + + if history.Status == 0 && int64(history.Epoch) < int64(epoch)-1 { + history.Status = 2 + } + tableData[i] = []interface{}{ + utils.FormatEpoch(history.Epoch), + utils.FormatBlockSlot(history.AttesterSlot), + utils.FormatAttestationStatus(history.Status), + utils.FormatTimestamp(utils.SlotToTime(history.AttesterSlot).Unix()), + utils.FormatAttestationInclusionSlot(history.InclusionSlot), + utils.FormatInclusionDelay(history.InclusionSlot, history.Delay), + } + } + } + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: totalCount, + RecordsFiltered: totalCount, + Data: tableData, + } + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// ValidatorWithdrawals returns a validators withdrawals in json +func ValidatorWithdrawals(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + reqCurrency := GetCurrency(r) + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + start, err := strconv.ParseUint(q.Get("start"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) + return + } + + orderColumn := q.Get("order[0][column]") + orderByMap := map[string]string{ + "0": "block_slot", + "1": "block_slot", + "2": "block_slot", + "3": "address", + "4": "amount", + } + orderBy, exists := orderByMap[orderColumn] + if !exists { + orderBy = "block_slot" + } + orderDir := q.Get("order[0][dir]") + if orderDir != "asc" { + orderDir = "desc" + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "draw": draw, + "start": start, + "orderColumn": orderColumn, + "orderBy": orderBy, + "orderDir": orderDir} + + length := uint64(10) + + withdrawalCount, err := db.GetTotalWithdrawalsCount([]uint64{index}) + if err != nil { + utils.LogError(err, "error getting validator withdrawals count from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + withdrawals, err := db.GetValidatorWithdrawals(index, length, start, orderBy, orderDir) + if err != nil { + utils.LogError(err, "error getting validator withdrawals from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + tableData := make([][]interface{}, 0, len(withdrawals)) + + for _, w := range withdrawals { + tableData = append(tableData, []interface{}{ + utils.FormatEpoch(utils.EpochOfSlot(w.Slot)), + utils.FormatBlockSlot(w.Slot), + utils.FormatTimestamp(utils.SlotToTime(w.Slot).Unix()), + utils.FormatWithdrawalAddress(w.Address, nil, "", false, false, true), + utils.FormatClCurrency(w.Amount, reqCurrency, 6, true, false, false, true), + }) + } + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: withdrawalCount, + RecordsFiltered: withdrawalCount, + Data: tableData, + } + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// ValidatorSlashings returns a validators slashings in json +func ValidatorSlashings(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "draw": draw} + + var totalCount uint64 + err = db.ReaderDb.Get(&totalCount, ` + SELECT + ( + SELECT COUNT(*) FROM blocks_attesterslashings a + INNER JOIN blocks b ON b.slot = a.block_slot AND b.proposer = $1 + WHERE attestation1_indices IS NOT null AND attestation2_indices IS NOT null + ) + ( + SELECT COUNT(*) FROM blocks_proposerslashings c + INNER JOIN blocks d ON d.slot = c.block_slot AND d.proposer = $1 + )`, index) + if err != nil { + utils.LogError(err, "error getting totalCount of validator-slashings from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + var attesterSlashings []*types.ValidatorAttestationSlashing + err = db.ReaderDb.Select(&attesterSlashings, ` + SELECT + blocks.slot, + blocks.epoch, + blocks.proposer, + blocks_attesterslashings.attestation1_indices, + blocks_attesterslashings.attestation2_indices + FROM blocks_attesterslashings + INNER JOIN blocks ON blocks.proposer = $1 and blocks_attesterslashings.block_slot = blocks.slot + WHERE attestation1_indices IS NOT NULL AND attestation2_indices IS NOT NULL`, index) + + if err != nil { + utils.LogError(err, "error getting validator attestations data from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + var proposerSlashings []*types.ValidatorProposerSlashing + err = db.ReaderDb.Select(&proposerSlashings, ` + SELECT blocks.slot, blocks.epoch, blocks.proposer, blocks_proposerslashings.proposerindex + FROM blocks_proposerslashings + INNER JOIN blocks ON blocks.proposer = $1 AND blocks_proposerslashings.block_slot = blocks.slot`, index) + if err != nil { + utils.LogError(err, "error getting validator proposer slashings data from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + tableData := make([][]interface{}, 0, len(attesterSlashings)+len(proposerSlashings)) + for _, b := range attesterSlashings { + + inter := intersect.Simple(b.Attestestation1Indices, b.Attestestation2Indices) + slashedValidators := []uint64{} + if len(inter) == 0 { + logger.Warning("No intersection found for attestation violation") + } + for _, v := range inter { + slashedValidators = append(slashedValidators, uint64(v.(int64))) + } + + tableData = append(tableData, []interface{}{ + utils.FormatSlashedValidators(slashedValidators), + utils.SlotToTime(b.Slot).Unix(), + "Attestation Violation", + utils.FormatBlockSlot(b.Slot), + utils.FormatEpoch(b.Epoch), + }) + } + + for _, b := range proposerSlashings { + tableData = append(tableData, []interface{}{ + utils.FormatSlashedValidator(b.ProposerIndex), + utils.SlotToTime(b.Slot).Unix(), + "Proposer Violation", + utils.FormatBlockSlot(b.Slot), + utils.FormatEpoch(b.Epoch), + }) + } + + sort.Slice(tableData, func(i, j int) bool { + return tableData[i][1].(int64) > tableData[j][1].(int64) + }) + + for _, b := range tableData { + b[1] = utils.FormatTimestamp(b[1].(int64)) + } + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: totalCount, + RecordsFiltered: totalCount, + Data: tableData, + } + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// Function checks if the generated ECDSA signature has correct lentgth and if needed sets recovery byte to 0 or 1 +func sanitizeSignature(sig string) ([]byte, error) { + sig = strings.Replace(sig, "0x", "", -1) + decodedSig, _ := hex.DecodeString(sig) + if len(decodedSig) != 65 { + return nil, fmt.Errorf("signature is less than 65 bytes (len = %v)", len(decodedSig)) + } + if decodedSig[crypto.RecoveryIDOffset] == 27 || decodedSig[crypto.RecoveryIDOffset] == 28 { + decodedSig[crypto.RecoveryIDOffset] -= 27 + } + return []byte(decodedSig), nil +} + +// Function tries to find the substring. +// +// If successful it turns string into []byte value and returns it +// +// If it fails, it will try to decode `msg`value from Hexadecimal to string and retry search again +func sanitizeMessage(msg string) ([]byte, error) { + subString := "beaconcha.in" + + if strings.Contains(msg, subString) { + return []byte(msg), nil + } else { + decoded := strings.Replace(msg, "0x", "", -1) + dec, _ := hex.DecodeString(decoded) + decodedString := (string(dec)) + if strings.Contains(decodedString, subString) { + return []byte(decodedString), nil + } + return nil, fmt.Errorf("%v was not found", subString) + } +} + +func SaveValidatorName(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + pubkey := vars["pubkey"] + pubkey = strings.ToLower(pubkey) + pubkey = strings.Replace(pubkey, "0x", "", -1) + + pubkeyDecoded, err := hex.DecodeString(pubkey) + if err != nil { + logger.Warnf("error parsing submitted pubkey %v: %v", pubkey, err) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + name := r.FormValue("name") + if len(name) > 40 { + name = name[:40] + } + + applyNameToAll := r.FormValue("apply-to-all") + + signature := r.FormValue("signature") + signatureWrapper := &types.MyCryptoSignature{} + err = json.Unmarshal([]byte(signature), signatureWrapper) + if err != nil { + logger.Warnf("error decoding submitted signature %v: %v", signature, err) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + msg, err := sanitizeMessage(signatureWrapper.Msg) + if err != nil { + logger.Warnf("Message is invalid %v: %v", signatureWrapper.Msg, err) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided message is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + msgHash := accounts.TextHash(msg) + + sig, err := sanitizeSignature(signatureWrapper.Sig) + if err != nil { + logger.Warnf("error parsing submitted signature %v: %v", signatureWrapper.Sig, err) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + recoveredPubkey, err := crypto.SigToPub(msgHash, sig) + if err != nil { + logger.Warnf("error recovering pubkey: %v", err) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + recoveredAddress := crypto.PubkeyToAddress(*recoveredPubkey) + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "pubkey": pubkey, + "name": name, + "applyNameToAll": applyNameToAll, + "recoveredAddress": recoveredAddress} + + var depositedAddress string + deposits, err := db.GetValidatorDeposits(pubkeyDecoded) + if err != nil { + utils.LogError(err, "error getting validator-deposits from db for signature verification", 0, errFields) + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + } + for _, deposit := range deposits.Eth1Deposits { + if deposit.ValidSignature { + depositedAddress = "0x" + fmt.Sprintf("%x", deposit.FromAddress) + break + } + } + + if strings.EqualFold(depositedAddress, recoveredAddress.Hex()) { + if applyNameToAll == "on" { + res, err := db.WriterDb.Exec(` + INSERT INTO validator_names (publickey, name) + SELECT publickey, $1 as name + FROM (SELECT DISTINCT publickey FROM eth1_deposits WHERE from_address = $2 AND valid_signature) a + ON CONFLICT (publickey) DO UPDATE SET name = excluded.name`, name, recoveredAddress.Bytes()) + if err != nil { + utils.LogError(err, "error saving validator name", 0, errFields) + utils.SetFlash(w, r, validatorEditFlash, "Error: Db error while updating validator names") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + rowsAffected, _ := res.RowsAffected() + utils.SetFlash(w, r, validatorEditFlash, fmt.Sprintf("Your custom name has been saved for %v validator(s).", rowsAffected)) + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + } else { + _, err := db.WriterDb.Exec(` + INSERT INTO validator_names (publickey, name) + VALUES($2, $1) + ON CONFLICT (publickey) DO UPDATE SET name = excluded.name`, name, pubkeyDecoded) + if err != nil { + utils.LogError(err, "error saving validator name", 0, errFields) + utils.SetFlash(w, r, validatorEditFlash, "Error: Db error while updating validator name") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + return + } + + utils.SetFlash(w, r, validatorEditFlash, "Your custom name has been saved.") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + } + + } else { + utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") + http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) + } +} + +// ValidatorHistory returns a validators history in json +func ValidatorHistory(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + currency := GetCurrency(r) + pageLength := 10 + maxPages := 10 + + vars := mux.Vars(r) + index, err := strconv.ParseUint(vars["index"], 10, 31) + if err != nil { + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + + start, err := strconv.ParseUint(q.Get("start"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) + return + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": index, + "draw": draw, + "start": start} + + var activationAndExitEpoch = struct { + ActivationEpoch uint64 `db:"activationepoch"` + ExitEpoch uint64 `db:"exitepoch"` + }{} + err = db.ReaderDb.Get(&activationAndExitEpoch, "SELECT activationepoch, exitepoch FROM validators WHERE validatorindex = $1", index) + if err != nil { + utils.LogError(err, "error getting activationAndExitEpoch for validator-history from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + totalCount := uint64(0) + + // Every validator is scheduled to issue an attestation once per epoch + // Hence we can calculate the number of attestations using the current epoch and the activation epoch + // Special care needs to be take for exited and pending validators + if activationAndExitEpoch.ExitEpoch != 9223372036854775807 && activationAndExitEpoch.ExitEpoch <= services.LatestFinalizedEpoch() { + totalCount += activationAndExitEpoch.ExitEpoch - activationAndExitEpoch.ActivationEpoch + } else { + totalCount += services.LatestFinalizedEpoch() - activationAndExitEpoch.ActivationEpoch + 1 + } + + if start > uint64((maxPages-1)*pageLength) { + start = uint64((maxPages - 1) * pageLength) + } + + currentEpoch := services.LatestEpoch() + + if currentEpoch != 0 { + currentEpoch = currentEpoch - 1 + } + var postExitEpochs uint64 = 0 + // for an exited validator we show the history until his exit or (in rare cases) until his last sync / propose duties are finished + if activationAndExitEpoch.ExitEpoch != 9223372036854775807 && currentEpoch > (activationAndExitEpoch.ExitEpoch-1) { + currentEpoch = activationAndExitEpoch.ExitEpoch - 1 + + var lastActionDay uint64 + // let's get the last day where validator had duties which can be after the exit epoch of the validator + err = db.ReaderDb.Get(&lastActionDay, ` + SELECT COALESCE(MAX(day), 0) + FROM validator_stats + WHERE + validatorindex = $1 AND + ( missed_sync > 0 + OR orphaned_sync > 0 + OR participated_sync > 0 + OR proposed_blocks > 0 + OR missed_blocks > 0 + OR orphaned_blocks > 0 + );`, index) + if err != nil { + utils.LogError(err, "error getting lastActionDay for validator-history from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + lastActionEpoch := (lastActionDay + 1) * utils.EpochsPerDay() + // if the validator had some duties after the exit epoch we calculate how many epochs we have to check after the exit epoch + if lastActionEpoch > currentEpoch { + postExitEpochs = protomath.MinU64(lastActionEpoch, services.LatestEpoch()-1) - currentEpoch + } + } + + tableData := make([][]interface{}, 0) + if postExitEpochs > 0 { + startEpoch := currentEpoch + 1 + endEpoch := startEpoch + postExitEpochs + withdrawalMap, incomeDetails, err := getWithdrawalAndIncome(index, startEpoch, endEpoch) + if err != nil { + return + } + + // if there are additional epochs with duties we have to go through all of them as there can be gaps (after the exit before the duty) + for i := endEpoch; i >= startEpoch; i-- { + if i > endEpoch { + break + } + + if incomeDetails[index] == nil || incomeDetails[index][i] == nil { + continue + } + totalCount++ + // for paging we skip the first X epochs with duties + if start > 0 { + start-- + continue + } else if len(tableData) >= pageLength { + continue + } + tableData = append(tableData, incomeToTableData(i, incomeDetails[index][i], withdrawalMap[i], currency)) + } + } + + postExitItemLength := len(tableData) + // if we have already found enough items we can skip the 'normal' step. + if postExitItemLength < pageLength { + startEpoch := currentEpoch - start - uint64(pageLength-1-postExitItemLength) // we only get the exact number of epochs to get to the page length + endEpoch := currentEpoch - start + if startEpoch > endEpoch { // handle underflows of startEpoch + startEpoch = 0 + } + + withdrawalMap, incomeDetails, err := getWithdrawalAndIncome(index, startEpoch, endEpoch) + if err != nil { + return + } + + for epoch := endEpoch; epoch >= startEpoch && len(tableData) < pageLength; epoch-- { + + if epoch > endEpoch { + break + } + + if incomeDetails[index] == nil || incomeDetails[index][epoch] == nil { + if epoch <= endEpoch { + rewardsStr := "pending..." + eventStr := template.HTML("") + if epoch < activationAndExitEpoch.ActivationEpoch { + rewardsStr = "" + eventStr = utils.FormatAttestationStatusShort(5) + } + tableData = append(tableData, []interface{}{ + utils.FormatEpoch(epoch), + rewardsStr, + template.HTML(""), + eventStr, + }) + } + continue + } + tableData = append(tableData, incomeToTableData(epoch, incomeDetails[index][epoch], withdrawalMap[epoch], currency)) + + } + } + + if len(tableData) == 0 { + tableData = append(tableData, []interface{}{ + template.HTML("Validator no longer active"), + }) + } + + totalCount = protomath.MinU64(totalCount, uint64(pageLength*maxPages)) + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: totalCount, + RecordsFiltered: totalCount, + Data: tableData, + } + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +func getWithdrawalAndIncome(index uint64, startEpoch uint64, endEpoch uint64) (map[uint64]*types.ValidatorWithdrawal, map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { + g := new(errgroup.Group) + + errFields := map[string]interface{}{ + "index": index, + "startEpoch": startEpoch, + "endEpoch": endEpoch} + + var withdrawals []*types.WithdrawalsByEpoch + g.Go(func() error { + var err error + withdrawals, err = db.GetValidatorsWithdrawalsByEpoch([]uint64{index}, startEpoch, endEpoch) + if err != nil { + utils.LogError(err, "error getting validator withdrawals by epoch", 0, errFields) + return err + } + return nil + }) + + var incomeDetails map[uint64]map[uint64]*itypes.ValidatorEpochIncome + g.Go(func() error { + var err error + incomeDetails, err = db.BigtableClient.GetValidatorIncomeDetailsHistory([]uint64{index}, startEpoch, endEpoch) + if err != nil { + utils.LogError(err, "error getting validator income details history from bigtable", 0, errFields) + return err + } + return nil + }) + + err := g.Wait() + withdrawalMap := make(map[uint64]*types.ValidatorWithdrawal) + for _, withdrawals := range withdrawals { + withdrawalMap[withdrawals.Epoch] = &types.ValidatorWithdrawal{ + Index: withdrawals.ValidatorIndex, + Epoch: withdrawals.Epoch, + Amount: withdrawals.Amount, + Slot: withdrawals.Epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch, + } + } + return withdrawalMap, incomeDetails, err +} + +func incomeToTableData(epoch uint64, income *itypes.ValidatorEpochIncome, withdrawal *types.ValidatorWithdrawal, currency string) []interface{} { + events := template.HTML("") + if income.AttestationSourcePenalty > 0 && income.AttestationTargetPenalty > 0 { + events += utils.FormatAttestationStatusShort(2) + } else { + events += utils.FormatAttestationStatusShort(1) + } + + if income.ProposerAttestationInclusionReward > 0 { + block := utils.FormatBlockStatusShort(1, 0) + events += block + } else if income.ProposalsMissed > 0 { + block := utils.FormatBlockStatusShort(2, 0) + events += block + } + + if withdrawal != nil { + withdrawal := utils.FormatWithdrawalShort(uint64(withdrawal.Slot), withdrawal.Amount) + events += withdrawal + } + + rewards := income.TotalClRewards() + return []interface{}{ + utils.FormatEpoch(epoch), + utils.FormatBalanceChangeFormatted(&rewards, currency, income), + template.HTML(""), + events, + } +} + +// ValidatorSync retrieves one page of sync duties of a specific validator for DataTable. +func ValidatorSync(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + validatorIndex, err := strconv.ParseUint(vars["index"], 10, 64) + if err != nil || validatorIndex > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) + return + } + + q := r.URL.Query() + + draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) + return + } + start, err := strconv.ParseUint(q.Get("start"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) + return + } + length, err := strconv.ParseUint(q.Get("length"), 10, 64) + if err != nil { + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) + return + } + if length > 100 { + length = 100 + } + + errFields := map[string]interface{}{ + "route": r.URL.String(), + "index": validatorIndex, + "draw": draw, + "start": start, + "length": length} + + // retrieve all sync periods for this validator + var syncPeriods []uint64 = []uint64{} + err = db.ReaderDb.Select(&syncPeriods, ` + SELECT distinct period + FROM sync_committees + WHERE validatorindex = $1 + ORDER BY period desc`, validatorIndex) + + if err != nil { + utils.LogError(err, "error getting sync tab count data of sync-assignments from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + tableData := [][]interface{}{} + + data := &types.DataTableResponse{ + Draw: draw, + RecordsTotal: 0, + RecordsFiltered: 0, + Data: tableData, + } + + if len(syncPeriods) == 0 { + // no sync periods for this validator => early exit with empty tableData + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + } + + return + } + + totalCount := uint64(0) // total count of sync duties for this validator + latestProposedSlot := services.LatestProposedSlot() + slots := make([]uint64, 0, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod*utils.Config.Chain.ClConfig.SlotsPerEpoch*uint64(len(syncPeriods))) + + for _, period := range syncPeriods { + firstEpoch := utils.FirstEpochOfSyncPeriod(period) + lastEpoch := firstEpoch + utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod - 1 + + firstSlot := firstEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + lastSlot := (lastEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 + + for slot := lastSlot; slot >= firstSlot && (slot <= lastSlot /* guards against underflows */); slot-- { + if slot > latestProposedSlot || utils.EpochOfSlot(slot) < utils.Config.Chain.ClConfig.AltairForkEpoch { + continue + } + slots = append(slots, slot) + } + } + + totalCount = uint64(len(slots)) + if start >= totalCount { + start = totalCount - 1 + } + + startIndex := start + length - 1 + if startIndex >= totalCount { + startIndex = totalCount - 1 + } + endIndex := start + + // retrieve sync duties and sync participations + syncDuties := make(map[uint64]*types.ValidatorSyncParticipation, length) + participations := make(map[uint64]uint64, length) + + // the slot range for the given table page might contain multiple sync periods and therefore we may need to split the queries to avoid fetching potentially thousands of duties at once + type SlotRange struct { + StartSlot uint64 + EndSlot uint64 + } + var consecutiveSlotRanges []SlotRange + + // slots are sorted in descending order + nextSlotRange := SlotRange{StartSlot: slots[endIndex], EndSlot: slots[endIndex]} + for i := endIndex + 1; i <= startIndex; i++ { + if slots[i] == nextSlotRange.StartSlot-1 { + nextSlotRange.StartSlot = slots[i] + } else { + consecutiveSlotRanges = append(consecutiveSlotRanges, nextSlotRange) + nextSlotRange = SlotRange{StartSlot: slots[i], EndSlot: slots[i]} + } + } + consecutiveSlotRanges = append(consecutiveSlotRanges, nextSlotRange) + + // make individual queries for each consecutive slot range and accumulate results + for _, slotRange := range consecutiveSlotRanges { + sdh, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, slotRange.StartSlot, slotRange.EndSlot) + if err != nil { + errFields["slotRange"] = slotRange + utils.LogError(err, "error getting validator sync duties data from bigtable", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + for slot, duty := range sdh[validatorIndex] { + syncDuties[slot] = duty + } + + par, err := db.GetSyncParticipationBySlotRange(slotRange.StartSlot, slotRange.EndSlot) + if err != nil { + errFields["slotRange"] = slotRange + utils.LogError(err, "error getting validator sync participation data from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + for slot, participation := range par { + participations[slot] = participation + } + } + + // search for the missed slots (status = 2), to see if it was only our validator that missed the slot or if the block was missed + slotsRange := slots[endIndex : startIndex+1] + + missedSlots := []uint64{} + err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, slotsRange) + if err != nil { + utils.LogError(err, "error getting missed slots data from db", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + missedSlotsMap := make(map[uint64]bool, len(missedSlots)) + for _, slot := range missedSlots { + missedSlotsMap[slot] = true + } + + // extract correct slots + tableData = make([][]interface{}, 0, length) + for index := endIndex; index <= startIndex; index++ { + slot := slots[index] + + epoch := utils.EpochOfSlot(slot) + participation := participations[slot] + + status := uint64(0) + if syncDuties[slot] != nil { + status = syncDuties[slot].Status + } + if _, ok := missedSlotsMap[slot]; ok { + status = 3 + } + + tableData = append(tableData, []interface{}{ + fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), + utils.FormatEpoch(epoch), + utils.FormatBlockSlot(slot), + utils.FormatSyncParticipationStatus(status, slot), + utils.FormatSyncParticipations(participation), + }) + } + + // fill and send data + data.RecordsTotal = totalCount + data.RecordsFiltered = totalCount + data.Data = tableData + + err = json.NewEncoder(w).Encode(data) + if err != nil { + utils.LogError(err, "error encoding json response", 0, errFields) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } +} + +// validatorNotFound will print the appropriate error message for when the requested validator cannot be found +func validatorNotFound(data *types.PageData, w http.ResponseWriter, r *http.Request, vars map[string]string, page string) { + validatorNotFoundTemplateFiles := append(layoutTemplateFiles, "validator/validatornotfound.html") + var validatorNotFoundTemplate = templates.GetTemplate(validatorNotFoundTemplateFiles...) + + SetPageDataTitle(data, "Validator not found") + d := InitPageData(w, r, "validators", fmt.Sprintf("/validator/%v%v", vars["index"], page), "", validatorNotFoundTemplateFiles) + + err := handleTemplateError(w, r, "validator.go", "Validator", "GetValidatorDeposits", validatorNotFoundTemplate.ExecuteTemplate(w, "layout", d)) + if err != nil { + return // an error has occurred and was processed + } +} diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/mainnet.chain.yml b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/mainnet.chain.yml new file mode 100644 index 000000000..1cb5197f7 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/mainnet.chain.yml @@ -0,0 +1,307 @@ +# Mainnet config + +# Extends the mainnet preset +PRESET_BASE: "mainnet" + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: "mainnet" + +# Transition +# --------------------------------------------------------------- +# TBD, 2**256-2**10 is a placeholder +TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Dec 1, 2020, 12pm UTC +MIN_GENESIS_TIME: 1606824000 +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x00000000 +# 604800 seconds (7 days) +GENESIS_DELAY: 604800 + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x01000000 +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +# Bellatrix +BELLATRIX_FORK_VERSION: 0x02000000 +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +# Capella +CAPELLA_FORK_VERSION: 0x03000000 +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +# Deneb +DENEB_FORK_VERSION: 0x04000000 +DENEB_FORK_EPOCH: 18446744073709551615 +# Byzantium +BYZANTIUM_FORK_BLOCK: 4370000 +# Constantinople +CONSTANTINOPLE_FORK_BLOCK: 7280000 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum PoW Mainnet +DEPOSIT_CHAIN_ID: 1337 +DEPOSIT_NETWORK_ID: 1337 +DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + +# Mainnet preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 + diff --git a/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/provision-explorer-config-custom.sh b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/provision-explorer-config-custom.sh new file mode 100644 index 000000000..cae8df1e6 --- /dev/null +++ b/docker_images/seedemu-ethexplorer/eth2-beaconchain-explorer/local-deployment/provision-explorer-config-custom.sh @@ -0,0 +1,258 @@ +#! /bin/bash + +POSTGRES_HOST=${POSTGRES_HOST:-postgres} +POSTGRES_PORT=${POSTGRES_PORT:-5432} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pass} +POSTGRES_USER=${POSTGRES_USER:-postgres} +POSTGRES_DB=${POSTGRES_DB:-db} +ALLOY_PORT=${ALLOY_PORT:-5432} +CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-9000} +LBT_PORT=${LBT_PORT:-9000} +RBT_PORT=${RBT_PORT:-9000} +EL_HOST=${EL_HOST:-192.168.254.128} +EL_PORT=${EL_PORT:-8545} +CL_HOST=${CL_HOST:-192.168.254.128} +CL_PORT=${CL_PORT:-8000} +REDIS_PORT=${REDIS_PORT:-6379} +REDIS_SESSIONS_PORT=${REDIS_SESSIONS_PORT:-6379} +SERVER_PORT=${SERVER_PORT:-5000} +CLICKHOUSE_DB=${CLICKHOUSE_DB:-beaconchain} + +export POSTGRES_HOST=$POSTGRES_HOST POSTGRES_PORT=$POSTGRES_PORT POSTGRES_PASSWORD=$POSTGRES_PASSWORD POSTGRES_USER=$POSTGRES_USER POSTGRES_DB=$POSTGRES_DB CLICKHOUSE_DB=$CLICKHOUSE_DB + +######################################## +# Common Retry Function - JSON RPC +######################################## + +retry_rpc_method() { + local rpc_url="$1" + local method="$2" + + local resp + local value + local i=0 + while true; do + i=$((i + 1)) + echo "[$method] Attempt $i..." >&2 + + resp=$(curl -sS --fail -X POST "$rpc_url" \ + -H "Content-Type: application/json" \ + -d "{ + \"jsonrpc\":\"2.0\", + \"method\":\"$method\", + \"params\":[], + \"id\":1 + }" 2>/dev/null) + + value=$(echo "$resp" | jq -er '.result // empty' 2>/dev/null || true) + + if [ -n "$value" ]; then + echo "[$method] Success: $value" >&2 + echo "$value" + + return 0 + fi + + echo "[$method] Failed response:" >&2 + echo "$resp" >&2 + + echo "[$method] Retrying in 30s..." >&2 + sleep 30 + done + + echo "[$method] Failed after retries" >&2 + return 1 +} + +######################################## +# Beacon Genesis +######################################## + +URL="http://${CL_HOST}:${CL_PORT}/eth/v1/beacon/genesis" + +count=0 +while true; do + count=$((count + 1)) + echo "[Genesis] Attempt $count..." + + RESP=$(curl -sS --fail "$URL" 2>/dev/null) + + GENESIS_TIMESTAMP=$(echo "$RESP" | jq -er '.data.genesis_time // empty' 2>/dev/null || true) + GENESIS_VALIDATORSROOT=$(echo "$RESP" | jq -er '.data.genesis_validators_root // empty' 2>/dev/null || true) + + if [ -n "$GENESIS_TIMESTAMP" ] && [ -n "$GENESIS_VALIDATORSROOT" ]; then + echo "[Genesis] Success!" + echo "GENESIS_TIMESTAMP=$GENESIS_TIMESTAMP" + echo "GENESIS_VALIDATORS_ROOT=$GENESIS_VALIDATORSROOT" + break + fi + + echo "[Genesis] Failed response:" + echo "$RESP" + + echo "[Genesis] Retrying in 30s..." + sleep 30 +done + +if [ -z "$GENESIS_TIMESTAMP" ] || [ -z "$GENESIS_VALIDATORSROOT" ]; then + echo "Failed to get GENESIS_TIMESTAMP or GENESIS_VALIDATORSROOT" + exit 1 +fi + +######################################## +# Execution Layer RPC +######################################## + +RPC_URL="http://${EL_HOST}:${EL_PORT}" + +CHAIN_HEX=$( + retry_rpc_method \ + "$RPC_URL" \ + "eth_chainId" +) + +NETWORK_ID=$( + retry_rpc_method \ + "$RPC_URL" \ + "net_version" +) + +if [ -z "$CHAIN_HEX" ]; then + echo "Failed to get CHAIN_HEX" + exit 1 +fi + +if [ -z "$NETWORK_ID" ]; then + echo "Failed to get NETWORK_ID" + exit 1 +fi + +CHAIN_ID=$((CHAIN_HEX)) + +echo "chainId (dec): $CHAIN_ID" +echo "networkId (dec): $NETWORK_ID" + +FILE="/app/local-deployment/mainnet.chain.yml" +sed -i "s/^DEPOSIT_CHAIN_ID:.*/DEPOSIT_CHAIN_ID: ${CHAIN_ID}/" "$FILE" +sed -i "s/^DEPOSIT_NETWORK_ID:.*/DEPOSIT_NETWORK_ID: ${NETWORK_ID}/" "$FILE" + +touch config.yml + +cat >config.yml <= 8 GB) and disk. + +ARG SOLANA_VERSION=v2.2.14 +ARG SOLANA_SHA256_AMD64=b5dadeb39ab7f31062c599ead459b9d809dd9d709365df6cb5943828b7693b36 + +# --------------------------------------------------------------------------- +# Common base image (same image the standard Docker compiler defaults use). +# --------------------------------------------------------------------------- +FROM handsonsecurity/seedemu-base:2.0 AS base + +# --------------------------------------------------------------------------- +# amd64: download the official pre-built release into /out. +# --------------------------------------------------------------------------- +FROM base AS build-amd64 +ARG SOLANA_VERSION +ARG SOLANA_SHA256_AMD64 +ARG DEBIAN_FRONTEND=noninteractive +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl bzip2; \ + rm -rf /var/lib/apt/lists/*; \ + mkdir -p /out/bin; \ + archive="solana-release-x86_64-unknown-linux-gnu.tar.bz2"; \ + url="https://github.com/anza-xyz/agave/releases/download/${SOLANA_VERSION}/${archive}"; \ + curl -L --fail --retry 3 "$url" -o "/tmp/${archive}"; \ + echo "${SOLANA_SHA256_AMD64} /tmp/${archive}" | sha256sum -c -; \ + mkdir -p /tmp/solana; \ + tar -xf "/tmp/${archive}" --strip-components=1 -C /tmp/solana; \ + for bin in solana solana-keygen solana-genesis solana-faucet agave-validator solana-test-validator; do \ + cp "/tmp/solana/bin/${bin}" /out/bin/; \ + done; \ + rm -rf "/tmp/${archive}" /tmp/solana + +# --------------------------------------------------------------------------- +# arm64: build the same pinned version from source into /out, on this base so +# the resulting binaries match the runtime glibc. +# (Anza publishes no aarch64 Linux release binaries.) +# --------------------------------------------------------------------------- +FROM base AS build-arm64 +ARG SOLANA_VERSION +ARG DEBIAN_FRONTEND=noninteractive +ENV CARGO_HOME=/root/.cargo \ + RUSTUP_HOME=/root/.rustup \ + PATH=/root/.cargo/bin:$PATH +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates curl git build-essential \ + libssl-dev libudev-dev pkg-config zlib1g-dev \ + llvm clang cmake make libprotobuf-dev protobuf-compiler; \ + rm -rf /var/lib/apt/lists/* +# Install rustup; the repo's rust-toolchain.toml pins the exact Rust version, +# which rustup installs automatically on first cargo invocation in the tree. +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --no-modify-path --default-toolchain none +RUN set -eux; \ + git clone --depth 1 --branch "${SOLANA_VERSION}" https://github.com/anza-xyz/agave.git /tmp/agave; \ + cd /tmp/agave; \ + cargo build --release \ + --bin solana \ + --bin solana-keygen \ + --bin solana-genesis \ + --bin solana-faucet \ + --bin agave-validator; \ + mkdir -p /out/bin; \ + for bin in solana solana-keygen solana-genesis solana-faucet agave-validator; do \ + cp "/tmp/agave/target/release/${bin}" /out/bin/; \ + done; \ + # solana-test-validator is convenient for single-node debugging; build if possible. + (cargo build --release --bin solana-test-validator \ + && cp /tmp/agave/target/release/solana-test-validator /out/bin/ || true); \ + rm -rf /tmp/agave /root/.cargo/registry /root/.cargo/git + +# --------------------------------------------------------------------------- +# Select the per-architecture binary source. TARGETARCH is set by the builder +# to the target platform's arch (amd64 / arm64), so the matching stage is used. +# --------------------------------------------------------------------------- +FROM build-${TARGETARCH} AS binaries + +# --------------------------------------------------------------------------- +# Final runtime image: the SEED base + the selected Agave binaries. +# --------------------------------------------------------------------------- +FROM base +ARG DEBIAN_FRONTEND=noninteractive +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates bzip2 jq; \ + rm -rf /var/lib/apt/lists/* + +COPY --from=binaries /out/bin/ /opt/solana/bin/ +RUN set -eux; \ + for bin in /opt/solana/bin/*; do ln -sf "$bin" "/usr/local/bin/$(basename "$bin")"; done; \ + /usr/local/bin/solana --version + +# Keep WORKDIR at '/'. seedemu's generated start.sh and interface_setup run from +# the container's WORKDIR and read helper files such as ifinfo.txt by relative +# path (they live at '/'); a non-root WORKDIR would break interface renaming and +# routing on every node. +WORKDIR / + +CMD ["/bin/bash"] diff --git a/docs/ai/ibgp-design-revision.md b/docs/ai/ibgp-design-revision.md new file mode 100644 index 000000000..e54c42a69 --- /dev/null +++ b/docs/ai/ibgp-design-revision.md @@ -0,0 +1,28 @@ +# Design: IBGP with Autonomous System + +## Goal + +Revise the current design of the ibgp layer and the `AutonomousSystem` class + + +## Design Principles + +- Clearly separate the duties of `AutonomousSystem` class (AS) and the ibgp layers. Deciding the ibgp mode withing the AS and roles of routers should be done in the AS, not in the ibgp layer. +- Although IBGP might have many modes, we would like to stick to this principle: simple default model, clean extension points, minimal built-in special cases. +- Avoid adding special cases to core layers unless they represent major research or operational patterns. +- Every new API should have a small example and a test. + + +## Required Behavior + + +- Inside the Autonomous System class, set the following (use the first option as the default) +``` +ibgp_mode = "full-mesh" | "route-reflector" +bgp_scope = "all-routers" | "edge-only" +core_forwarding = "plain-ip" | "mpls" | "sr" | "tunnel" | "redistribute" +``` +- all-routers + full-mesh: All routers participate in ibgp using full-mesh peering +- all-routers + route-reflector: All routers participate in ibgp using route reflectors +- edge-only + full-mesh: Only edge routers participate in ibgp using full-mesh peerin; core routers do not participate in ibgp +- edge-only + route-reflector: Only edge routers participate in ibgp using route reflector; core routers do not participate in ibgp diff --git a/docs/ai/route-reflector-code-details.md b/docs/ai/route-reflector-code-details.md new file mode 100644 index 000000000..d4546c238 --- /dev/null +++ b/docs/ai/route-reflector-code-details.md @@ -0,0 +1,44 @@ +# AS-level iBGP Mode Prompt + +## Background + +Original related code locations: + +- `seedemu/core/AutonomousSystem.py` + - `createBgpCluster(address)` registers an RR cluster ID. + - `_aggregateBgpClusters()` aggregates cluster, RR, and client membership. + - The current implicit default cluster ID is hard-coded as `"10.0.0.0"`. +- `seedemu/core/Node.py` + - `Router.makeRouteReflector()` records whether a router is an RR. + - `Router.joinBgpCluster(cluster_id)` records the cluster a router belongs to. +- `seedemu/layers/Ibgp.py` + - `configure()` currently calls `asobj._aggregateBgpClusters()`. + - It currently uses `has_rr = any(len(rrs) > 0 ...)` to decide whether to use RR mode or full mesh mode. + - `_render_rr_mode()` writes RR/client session intent into routers. + - `_render_full_mesh_mode()` keeps the legacy full mesh behavior. +- `seedemu/layers/_bgp_metadata.py` + - `route_reflector_client` and `route_reflector_cluster_id` are BGP intent fields. + - BIRD renders `rr client` and `rr cluster id ...`. +- `seedemu/layers/Routing.py` + - FRR renders `bgp cluster-id ...` and `neighbor ... route-reflector-client`. + +## Behavior Requirements + +1. `Ibgp.configure()` must no longer infer the mode by itself using `has_rr`. +2. `Ibgp.configure()` should get the effective iBGP mode from `AutonomousSystem`: + - `"full-mesh"` calls `_render_full_mesh_mode()`. + - `"route-reflector"` calls `_render_rr_mode()`. +3. RR cluster aggregation, the default cluster ID, and default RR selection must all be completed inside `AutonomousSystem`. +4. For an AS to enter effective `"route-reflector"` mode, the user must explicitly call `setIbgpMode("route-reflector")`. Only after that should the following APIs be callable: + - `createBgpCluster()`. + - `makeRouteReflector(True)`. + - `joinBgpCluster(cluster_id)`. +5. If the user explicitly sets `"full-mesh"`, that setting should be respected unless RR-specific configuration already exists. + If there is a conflict, do not silently ignore it. Raise a clear error, for example: + `"AS2 has route-reflector cluster/router configuration but ibgp_mode is full-mesh"`. +6. Multi-cluster RR must still validate that every cluster has an RR and clients. The default RR is only used when the user selected `"route-reflector"` and did not provide any RR configuration. +7. Do not change the intent schema in `_bgp_metadata.py`. Continue using the existing fields + `route_reflector_client` and `route_reflector_cluster_id`. +8. Do not change BIRD/FRR rendering semantics. BIRD should still render RR configuration through `_bgp_metadata.py`, and FRR should still render cluster-id and route-reflector-client through `Routing.py`. +9. Existing old RR usage through `createBgpCluster()`, `joinBgpCluster()`, and `makeRouteReflector()` must not be broken. +10. `setIbgpMode()` must validate the input. Invalid values should directly raise `ValueError` or `AssertionError`, and the error message must include the valid values. diff --git a/docs/ai/route-reflector-design.md b/docs/ai/route-reflector-design.md new file mode 100644 index 000000000..248e1dd00 --- /dev/null +++ b/docs/ai/route-reflector-design.md @@ -0,0 +1,31 @@ +# Design: Route Reflector + +## Goal + +Revise the implemenation of the route reflector in the emulator + + +## Design Principles + +- Keep the configuration of route reflector structure inside the `AutonomousSystem` class, not in the ibgp layer. + + +## API design + +- Introduce an API called `completeIbgpSetup()` inside the `AutonomousSystem` class. It inspects the `AutonomousSystem` object and all the routers inside this AS, finding any place where ibgp setup is incomplete, and complete it. +- The reason for this API: Users might not set up the route reflector structure inside the AS completely, we need to automatically complete the setup. +- Who invokes this API? When `ibgp` starts rendering for an AS, it first calls this API to complete the ibgp setup within the AS. +- Principle: We choose to do this within the AS class, not at the ibgp layer, because we want all the ibgp-related setup to be done inside the AS, and the ibgp layer's job only focuses on rendering, i.e., turning the setup into actual system setup. + + + +## Required Behavior + +- Ensure at least one cluster ID is created for the AS. If users do not set one, a deterministic default cluster ID must be created. The first created cluster ID is set as the default. +- Each cluster must have at least one route reflector, if users do not set one, a determininistic router is selected as the RR. If the user does not specify a route reflector for the default single-cluster setup, the emulator deterministically selects one router as the default route reflector. The default rule is: Select the first router in this AS after sorting router names in ascending order.(A cluster may contain multiple route reflectors. Among them, one route reflector is treated as the default route reflector of that cluster.) +- Each router must join one cluster ID; if users do not set one, the router joins the default one of the AS. +- Each router must have one RR; if users do not set one, the default RR in the cluster is used. +- A cluster allows multiple route reflectors, with one being the default. All the route reflectors within the same cluster must peer with one another. +- The default cluster ID must be decided by the AS. Use an ASN-derived IPv4-style string, for example `"10.{asn}.0.1"`, and ensure it does not conflict. +- If the user configures multiple cluster IDs within the same AS, the emulator assumes that the user intends to define a more advanced route reflector topology. In this case, the emulator should not create an additional default cluster ID or automatically select a default route reflector. Instead, the user must explicitly provide all required route reflector information:Every router must explicitly join one cluster ID. Every configured cluster ID must have at least one route reflector. Each router’s cluster ID must exist. Each router must be associated with a valid route reflector in its cluster, unless the router itself is a route reflector. All route reflectors inside the same cluster must peer with one another. If any required information is missing, completeIbgpSetup() should raise a clear error. Examples of error cases: Router r3 in AS 150 is missing a cluster ID; Cluster 10.150.0.2 in AS 150 has no route reflector. + diff --git a/docs/designs/control-plane-extension-design.md b/docs/designs/control-plane-extension-design.md new file mode 100644 index 000000000..a64a0e72e --- /dev/null +++ b/docs/designs/control-plane-extension-design.md @@ -0,0 +1,52 @@ +# Control-Plane Extension Design + +This slice adds a narrow FRR and ExaBGP foundation without changing the default +IPv4/BIRD behavior. + +## Backend Ownership + +`Router` owns the routing daemon choice: + +```python +as2.createRouter("r1") +as2.createRouter("r2", routingBackend="frr") +``` + +Only `bird` and `frr` are valid router backends. `exabgp` and `external` are +intentionally rejected because ExaBGP is not a full router daemon in this model. + +## Intent Versus Rendering + +`Ebgp`, `Ibgp`, and `Ospf` record control-plane intent: + +- BGP peer address, ASN, relationship, import policy, export policy. +- OSPF active/passive interface intent. + +They do not decide whether the router runs BIRD or FRR. `Routing` is the only +layer that renders backend-specific daemon files: + +- BIRD routers get `/etc/bird/bird.conf` and `bird -d`. +- FRR routers get `/etc/frr/frr.conf` and `/frr_start`. + +This keeps protocol semantics stable while allowing backend-specific rendering +to evolve. + +## ExaBGP Shape + +ExaBGP is modeled as `ExaBgpService + Binding`. + +The service installs an ExaBGP speaker on a bound node, resolves the attached +router, emits `/etc/exabgp/exabgp.conf`, and records the router-side BGP peer +intent. `Routing` then renders that peer on the router's selected backend. + +This avoids three legacy traps: + +- ExaBGP is not a router backend. +- ExaBGP is not a `Layer` shim for BGP routing. +- ExaBGP is not mixed with Looking Glass route-state observation. + +## Current Scope + +This review slice is IPv4-only and intentionally excludes IPv6 readiness, +Looking Glass, K8s, CI redesign, and runtime evidence plumbing. Those belong to +separate follow-up work. diff --git a/docs/user_manual/blockchain/metamask.md b/docs/user_manual/blockchain/metamask.md index 7dc3ef205..1588c4a24 100644 --- a/docs/user_manual/blockchain/metamask.md +++ b/docs/user_manual/blockchain/metamask.md @@ -11,7 +11,7 @@ We show how to conduct this step for Firefox. Other browsers have similar steps. Go to Firefox's menu, click the `"Add-ons and themes"` item. Search for `metamask`, and you will find -an extension developed by `danfinlay`. +an extension developed by `MetaMask`. Follow the instruction to install it. @@ -26,26 +26,23 @@ address you get from your emulator may be different from those listed in the following). ``` -$ dockps | grep Eth -e372096bb926 as150h-Ethereum-POA-00-Signer-BootNode-10.150.0.71 -f0ef91ef9e22 as150h-Ethereum-POA-01-10.150.0.72 -3b8c1d191058 as151h-Ethereum-POA-02-Signer-10.151.0.71 -... -aea1106d932d as164h-Ethereum-POA-18-Signer-BootNode-10.164.0.71 -7cd6fa6888b2 as164h-Ethereum-POA-19-10.164.0.72 +$ dockps | grep Geth +2ebb64a85bbc as150h-Ethereum-POS-Geth-1-10.150.0.71 +9b8623f3cb7a as150h-Ethereum-POS-Geth-2-10.150.0.72 +14093c1f7c6f as151h-Ethereum-POS-Geth-3-10.151.0.71 ``` Pick one of the nodes, and then configure MetaMask to connect to -this node. Go to the `Settings` menu inside MetaMask +this node. Go to the menu inside MetaMask (usually in the upper right corner) and follow the instructions below. Replace the `` with the actual IP address of the node that you have selected. ``` -Settings > Networks > Add a network > Add a network manually +Manage > Networks > Add a custom network Network name: pick any name (e.g., SEED emulator) -New RPC URL: http://:8545 +Default RPC URL: http://:8545 Chain ID: 1337 Currency symbol: ETH ``` @@ -103,3 +100,6 @@ networks are worthless. To enable that, turn on the following switch. Select this to show fiat conversion on test networks ``` + +- Problem: MetaMask can connect to our blockchain and the account can be created, but MetaMask gets stuck when sending transactions. +If you are in Mainland China, the possible reason is that your browser does not have a system proxy. You can configure the proxy in the browser settings or directly configure the system proxy for your machine. diff --git a/examples/basic/A00_simple_as/README.md b/examples/basic/A00_simple_as/README.md index 14dbb4377..12dd2f065 100644 --- a/examples/basic/A00_simple_as/README.md +++ b/examples/basic/A00_simple_as/README.md @@ -169,3 +169,49 @@ See [this manual](/docs/user_manual/manual.md#rendering). Generate the emulation files. See [this manual](/docs/user_manual/compiler.md). +## Standard Arguments + +The example accepts both the legacy platform argument and the newer named +arguments: + +```sh +python examples/basic/A00_simple_as/simple_as.py amd +python examples/basic/A00_simple_as/simple_as.py --platform amd --output examples/basic/A00_simple_as/output +python examples/basic/A00_simple_as/simple_as.py --dumpfile examples/basic/A00_simple_as/simple_as.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +This example includes an `example.yaml` manifest for `seedemu.testing`. Run +these commands from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/basic/A00_simple_as/example.yaml +python seedemu/testing/cli.py compile examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +python seedemu/testing/cli.py build examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +python seedemu/testing/cli.py up examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +python seedemu/testing/cli.py probe examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +python seedemu/testing/cli.py test examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +python seedemu/testing/cli.py down examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/basic/A00_simple_as/example.yaml --artifact-dir ci-artifacts/a00 +``` + +The manifest declares `runner: internet` because it uses Internet-style probes. +The readiness stage checks that the three web hosts and three border routers are +running. The probe stage checks cross-AS reachability, and `test_runtime.py` +demonstrates the same runtime validation as a custom Python test program. + diff --git a/examples/basic/A00_simple_as/example.yaml b/examples/basic/A00_simple_as/example.yaml new file mode 100644 index 000000000..f3da94ee0 --- /dev/null +++ b/examples/basic/A00_simple_as/example.yaml @@ -0,0 +1,68 @@ +id: basic-a00-simple-as +name: Simple Autonomous Systems +description: Three stub autonomous systems peered through IX100 with one web host in each AS. +runner: internet +script: simple_as.py +platform: amd +features: + - ipv4-default + - web-service + - ebgp-route-server + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A00 services are running + type: compose-ps + services: + - hnode_150_web + - hnode_151_web + - hnode_152_web + - brdnode_150_router0 + - brdnode_151_router0 + - brdnode_152_router0 + retries: 30 + interval: 2 + +probes: + - name: AS151 web host reaches AS150 web service + type: exec + service: hnode_151_web + command: curl -fsS http://10.150.0.71 >/dev/null + expect_exit: 0 + retries: 20 + interval: 3 + + - name: AS152 web host reaches AS151 web service + type: exec + service: hnode_152_web + command: curl -fsS http://10.151.0.71 >/dev/null + expect_exit: 0 + retries: 20 + interval: 3 + + - name: AS150 can ping AS152 web host + type: ping + service: hnode_150_web + target: 10.152.0.71 + count: 3 + retries: 10 + interval: 3 + +test_programs: + - name: A00 custom runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A00_simple_as/simple_as.py b/examples/basic/A00_simple_as/simple_as.py index 159f38cb1..d19daea37 100755 --- a/examples/basic/A00_simple_as/simple_as.py +++ b/examples/basic/A00_simple_as/simple_as.py @@ -1,118 +1,106 @@ #!/usr/bin/env python3 # encoding: utf-8 -from seedemu.layers import Base, Routing, Ebgp -from seedemu.services import WebService -from seedemu.compiler import Docker, Platform -from seedemu.core import Emulator, Binding, Filter -import sys, os +from __future__ import annotations -def run(dumpfile = None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - - # Initialize the emulator and layers - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - web = WebService() +import argparse +from pathlib import Path +import sys - ############################################################################### - # Create an Internet Exchange - base.createInternetExchange(100) - ############################################################################### - # Create and set up AS-150 - - # Create an autonomous system - as150 = base.createAutonomousSystem(150) - - # Create a network - as150.createNetwork('net0') +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) - # Create a router and connect it to two networks - as150.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Routing +from seedemu.services import WebService - # Create a host called web and connect it to a network - as150.createHost('web').joinNetwork('net0') - # Create a web service on virtual node, give it a name - # This will install the web service on this virtual node - web.install('web150') +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A00 simple AS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args - # Bind the virtual node to a physical node - emu.addBinding(Binding('web150', filter = Filter(nodeName = 'web', asn = 150))) +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 - ############################################################################### - # Create and set up AS-151 - # It is similar to what is done to AS-150 - as151 = base.createAutonomousSystem(151) - as151.createNetwork('net0') - as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') +def build_emulator() -> Emulator: + emu = Emulator() - as151.createHost('web').joinNetwork('net0') - web.install('web151') - emu.addBinding(Binding('web151', filter = Filter(nodeName = 'web', asn = 151))) + base = Base() + routing = Routing() + ebgp = Ebgp() + web = WebService() ############################################################################### - # Create and set up AS-152 - # It is similar to what is done to AS-150 - - as152 = base.createAutonomousSystem(152) - as152.createNetwork('net0') - as152.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - - as152.createHost('web').joinNetwork('net0') - web.install('web152') - emu.addBinding(Binding('web152', filter = Filter(nodeName = 'web', asn = 152))) - + # Create an Internet Exchange + base.createInternetExchange(100) ############################################################################### - # Peering these ASes at Internet Exchange IX-100 + # Create and set up AS150, AS151, and AS152 - ebgp.addRsPeer(100, 150) - ebgp.addRsPeer(100, 151) - ebgp.addRsPeer(100, 152) + for asn in [150, 151, 152]: + current_as = base.createAutonomousSystem(asn) + current_as.createNetwork("net0") + current_as.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + current_as.createHost("web").joinNetwork("net0") - - ############################################################################### - # Rendering + vnode = "web{}".format(asn) + web.install(vnode) + emu.addBinding(Binding(vnode, filter=Filter(nodeName="web", asn=asn))) + ebgp.addRsPeer(100, asn) emu.addLayer(base) emu.addLayer(routing) emu.addLayer(ebgp) emu.addLayer(web) + return emu + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() if dumpfile is not None: emu.dump(dumpfile) - else: + return + + if render: emu.render() - # Attach the Internet Map container to the emulator - docker = Docker(platform=platform) + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 - ############################################################################### - # Compilation - emu.compile(docker, './output', override=True) -if __name__ == '__main__': - run() +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A00_simple_as/test_runtime.py b/examples/basic/A00_simple_as/test_runtime.py new file mode 100644 index 000000000..d026a92e7 --- /dev/null +++ b/examples/basic/A00_simple_as/test_runtime.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web150 = test.require_service(150, "web") + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + + if web150 and web151 and web152: + test.exec_check("AS151 fetches AS150 web service", web151, "curl -fsS http://{} >/dev/null".format(web150.address)) + test.exec_check("AS152 fetches AS151 web service", web152, "curl -fsS http://{} >/dev/null".format(web151.address)) + test.exec_check("AS150 reaches AS152 by ICMP", web150, "ping -c 3 {} >/dev/null".format(web152.address)) + + test.write_summary("a00-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A01_transit_as/README.md b/examples/basic/A01_transit_as/README.md index 8006bc473..7030a3197 100644 --- a/examples/basic/A01_transit_as/README.md +++ b/examples/basic/A01_transit_as/README.md @@ -91,3 +91,35 @@ emu.dump('base_component.bin') It should be noted that `dump()` must be called before the `render()`. +## TestRunner lifecycle + +This example also includes an `example.yaml` manifest for +`seedemu.testing`. The manifest lets CI, agents, and developers drive the +example through the same lifecycle: + +```sh +python seedemu/testing/cli.py clean examples/basic/A01_transit_as/example.yaml +python seedemu/testing/cli.py compile examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +python seedemu/testing/cli.py build examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +python seedemu/testing/cli.py up examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +python seedemu/testing/cli.py probe examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +python seedemu/testing/cli.py down examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +``` + +The full lifecycle can be run with: + +```sh +python seedemu/testing/cli.py all examples/basic/A01_transit_as/example.yaml --artifact-dir ci-artifacts/a01 +``` + +The runtime probes check that AS2 is actually providing transit between the +stub ASes: + +- AS151 reaches AS152 through AS2 from IX100 to IX101. +- AS151 reaches AS153 through AS2 from IX100 to IX101. +- AS152 reaches AS151 through AS2 from IX101 to IX100. + +The readiness check uses Docker Compose service names generated by the Docker +compiler. Routers connected to an Internet Exchange are border routers and use +`brdnode_*` service names, while internal transit routers use `rnode_*` names. + diff --git a/examples/basic/A01_transit_as/example.yaml b/examples/basic/A01_transit_as/example.yaml new file mode 100644 index 000000000..fb4e68a17 --- /dev/null +++ b/examples/basic/A01_transit_as/example.yaml @@ -0,0 +1,75 @@ +id: basic-a01-transit-as +name: Transit AS +description: A transit-AS example with AS2 carrying traffic between stub ASes on IX100 and IX101. +runner: internet +script: transit_as.py +platform: amd +features: + - ipv4-default + - transit-as + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + - base_component.bin + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: transit AS services are running + type: compose-ps + services: + - brdnode_2_r1 + - rnode_2_r2 + - rnode_2_r3 + - brdnode_2_r4 + - brdnode_151_router0 + - brdnode_152_router0 + - brdnode_153_router0 + - hnode_151_host0 + - hnode_152_host0 + - hnode_153_host0 + retries: 30 + interval: 2 + +probes: + - name: AS151 reaches AS152 through transit AS2 + type: ping + service: hnode_151_host0 + target: 10.152.0.71 + count: 3 + retries: 30 + interval: 5 + + - name: AS151 reaches AS153 through transit AS2 + type: ping + service: hnode_151_host0 + target: 10.153.0.71 + count: 3 + retries: 30 + interval: 5 + + - name: AS152 reaches AS151 through transit AS2 + type: ping + service: hnode_152_host0 + target: 10.151.0.71 + count: 3 + retries: 30 + interval: 5 + +test_programs: + - name: A01 transit AS runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A01_transit_as/old_transit_as.py b/examples/basic/A01_transit_as/old_transit_as.py deleted file mode 100755 index c25519c66..000000000 --- a/examples/basic/A01_transit_as/old_transit_as.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - -from seedemu.layers import Base, Routing, Ebgp, PeerRelationship, Ibgp, Ospf -from seedemu.services import WebService -from seedemu.core import Emulator, Binding, Filter -from seedemu.compiler import Docker - -emu = Emulator() - -base = Base() -routing = Routing() -ebgp = Ebgp() -ibgp = Ibgp() -ospf = Ospf() -web = WebService() - -############################################################################### - -base.createInternetExchange(100) -base.createInternetExchange(101) - -############################################################################### -# Create and set up the transit AS (AS-150) - -as150 = base.createAutonomousSystem(150) - -# Create 3 internal networks -as150.createNetwork('net0') -as150.createNetwork('net1') -as150.createNetwork('net2') - -# Create four routers and link them in a linear structure: -# ix100 <--> r1 <--> r2 <--> r3 <--> r4 <--> ix101 -# r1 and r2 are BGP routers because they are connected to internet exchanges -as150.createRouter('r1').joinNetwork('net0').joinNetwork('ix100') -as150.createRouter('r2').joinNetwork('net0').joinNetwork('net1') -as150.createRouter('r3').joinNetwork('net1').joinNetwork('net2') -as150.createRouter('r4').joinNetwork('net2').joinNetwork('ix101') - -############################################################################### -# Create and set up the stub AS (AS-151) - -as151 = base.createAutonomousSystem(151) - -# Create an internal network and a router -as151.createNetwork('net0') -as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - -# Create a web-service node -as151.createHost('web').joinNetwork('net0') -web.install('web151') -emu.addBinding(Binding('web151', filter = Filter(nodeName = 'web', asn = 151))) - - -############################################################################### -# Create and set up the stub AS (AS-152) - -as152 = base.createAutonomousSystem(152) -as152.createNetwork('net0') -as152.createRouter('router0').joinNetwork('net0').joinNetwork('ix101') - -as152_web = as152.createHost('web').joinNetwork('net0') -web.install('web152') -emu.addBinding(Binding('web152', filter = Filter(nodeName = 'web', asn = 152))) - - -############################################################################### -# Add BGP peering - -# Make AS-150 the internet service provider for AS-151 and AS-152 -ebgp.addPrivatePeering(100, 150, 151, abRelationship = PeerRelationship.Provider) -ebgp.addPrivatePeering(101, 150, 152, abRelationship = PeerRelationship.Provider) - -############################################################################### - -emu.addLayer(base) -emu.addLayer(routing) -emu.addLayer(ebgp) -emu.addLayer(ibgp) -emu.addLayer(ospf) -emu.addLayer(web) - -############################################################################### -# Save the emulation as a component (can be reused by other emulation) - -emu.dump('base_component.bin') - -############################################################################### -# Generate the docker file - -emu.render() -emu.compile(Docker(), './output', override=True) diff --git a/examples/basic/A01_transit_as/test_runtime.py b/examples/basic/A01_transit_as/test_runtime.py new file mode 100644 index 000000000..988da21e7 --- /dev/null +++ b/examples/basic/A01_transit_as/test_runtime.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + host151 = test.require_service(151, "host0") + host152 = test.require_service(152, "host0") + host153 = test.require_service(153, "host0") + test.require_service(2, "r1") + test.require_service(2, "r2") + test.require_service(2, "r3") + test.require_service(2, "r4") + + if host151 and host152: + test.exec_check("AS151 reaches AS152 through transit AS2", host151, "ping -c 3 {} >/dev/null".format(host152.address)) + if host151 and host153: + test.exec_check("AS151 reaches AS153 through transit AS2", host151, "ping -c 3 {} >/dev/null".format(host153.address)) + if host152 and host151: + test.exec_check("AS152 reaches AS151 through transit AS2", host152, "ping -c 3 {} >/dev/null".format(host151.address)) + + test.write_summary("a01-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A01_transit_as/transit_as.py b/examples/basic/A01_transit_as/transit_as.py index 48ad89eaf..9c9af1beb 100755 --- a/examples/basic/A01_transit_as/transit_as.py +++ b/examples/basic/A01_transit_as/transit_as.py @@ -1,39 +1,50 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu import * -import sys, os -def run(dumpfile = None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A01 transit AS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: ############################################################################### # Create the base layer - base = Base() + base = Base() # Create two Internet Exchanges, where BGP routers peer with one another. base.createInternetExchange(100) base.createInternetExchange(101) ############################################################################### - # Create and configure a transit autonomous system + # Create and configure a transit autonomous system as2 = base.createAutonomousSystem(2) @@ -59,10 +70,9 @@ def run(dumpfile = None): as151.createNetwork('net0') as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - # Create a host node + # Create a host node as151.createHost('host0').joinNetwork('net0') - ############################################################################### # Create and set up the stub AS (AS-152) as152 = base.createAutonomousSystem(152) @@ -70,7 +80,6 @@ def run(dumpfile = None): as152.createRouter('router0').joinNetwork('net0').joinNetwork('ix101') as152.createHost('host0').joinNetwork('net0') - ############################################################################### # Create and set up the stub AS (AS-153) as153 = base.createAutonomousSystem(153) @@ -78,31 +87,38 @@ def run(dumpfile = None): as153.createRouter('router0').joinNetwork('net0').joinNetwork('ix101') as153.createHost('host0').joinNetwork('net0') - - ############################################################################### # Create the EBGP layer, conduct peering - ebgp = Ebgp() + ebgp = Ebgp() # Peer AS-2 with ASes 151, 152, and 153 (AS-2 is the Internet service provider) - ebgp.addPrivatePeering(100, 2, 151, abRelationship = PeerRelationship.Provider) - ebgp.addPrivatePeering(101, 2, 152, abRelationship = PeerRelationship.Provider) - ebgp.addPrivatePeering(101, 2, 153, abRelationship = PeerRelationship.Provider) + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 153, abRelationship=PeerRelationship.Provider) # Peer AS-152 and AS-153 (as equal peers for mutual benefit) - ebgp.addPrivatePeering(101, 152, 153, abRelationship = PeerRelationship.Peer) - + ebgp.addPrivatePeering(101, 152, 153, abRelationship=PeerRelationship.Peer) ############################################################################### - # Add all the necessary layers - emu = Emulator() + # Add all the necessary layers + emu = Emulator() emu.addLayer(base) - emu.addLayer(Routing()) + emu.addLayer(Routing()) emu.addLayer(ebgp) emu.addLayer(Ibgp()) emu.addLayer(Ospf()) + return emu + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() if dumpfile is not None: ############################################################################### # Save the emulation if needed (can be reused by other emulation) @@ -112,12 +128,26 @@ def run(dumpfile = None): else: ############################################################################### # Render the emulation - emu.render() + if render: + emu.render() ############################################################################### # Final step: Generate the docker files docker = Docker(internetMapEnabled=True, platform=platform) - emu.compile(docker, './output', override = True) + emu.compile(docker, output or './output', override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + if __name__ == "__main__": - run() + raise SystemExit(main()) diff --git a/examples/basic/A02_transit_as_mpls/transit_as_mpls.py b/examples/basic/A02_transit_as_mpls/transit_as_mpls.py deleted file mode 100755 index a66b8b48e..000000000 --- a/examples/basic/A02_transit_as_mpls/transit_as_mpls.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - -from seedemu.layers import Base, Routing, Ebgp, PeerRelationship, Mpls -from seedemu.services import WebService -from seedemu.core import Emulator, Binding, Filter -from seedemu.compiler import Docker, Platform -import sys, os - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -emu = Emulator() - -base = Base() -routing = Routing() -ebgp = Ebgp() -mpls = Mpls() -web = WebService() - -############################################################################### - -base.createInternetExchange(100) -base.createInternetExchange(101) - -############################################################################### -# Create and set up the transit AS (AS-150) - -as150 = base.createAutonomousSystem(150) - -# Create 3 internal networks -as150.createNetwork('net0') -as150.createNetwork('net1') -as150.createNetwork('net2') - -# Create four routers and link them in a linear structure: -# ix100 <--> r1 <--> r2 <--> r3 <--> r4 <--> ix101 -# r1 and r2 are BGP routers because they are connected to internet exchanges -as150.createRouter('r1').joinNetwork('net0').joinNetwork('ix100') -as150.createRouter('r2').joinNetwork('net0').joinNetwork('net1') -as150.createRouter('r3').joinNetwork('net1').joinNetwork('net2') -as150.createRouter('r4').joinNetwork('net2').joinNetwork('ix101') - -# Enable MPLS -mpls.enableOn(150) - -############################################################################### -# Create and set up the stub AS (AS-151) - -as151 = base.createAutonomousSystem(151) - -# Create an internal network and a router -as151.createNetwork('net0') -as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - -# Create a web-service node -as151.createHost('web').joinNetwork('net0') -web.install('web151') -emu.addBinding(Binding('web151', filter = Filter(nodeName = 'web', asn = 151))) - - -############################################################################### -# Create and set up the stub AS (AS-152) - -as152 = base.createAutonomousSystem(152) -as152.createNetwork('net0') -as152.createRouter('router0').joinNetwork('net0').joinNetwork('ix101') - -as152_web = as152.createHost('web').joinNetwork('net0') -web.install('web152') -emu.addBinding(Binding('web152', filter = Filter(nodeName = 'web', asn = 152))) - - -############################################################################### -# Make AS-150 the internet service provider for AS-151 and AS-152 -ebgp.addPrivatePeering(100, 150, 151, abRelationship = PeerRelationship.Provider) -ebgp.addPrivatePeering(101, 150, 152, abRelationship = PeerRelationship.Provider) - -############################################################################### - -emu.addLayer(base) -emu.addLayer(routing) -emu.addLayer(ebgp) -emu.addLayer(mpls) -emu.addLayer(web) - -emu.render() - -############################################################################### - -emu.compile(Docker(platform=platform), './output', override=True) diff --git a/examples/basic/A02_transit_as_mpls/README.md b/examples/basic/A02a_transit_as_mpls/README.md similarity index 62% rename from examples/basic/A02_transit_as_mpls/README.md rename to examples/basic/A02a_transit_as_mpls/README.md index a11119395..6b9b4f89f 100644 --- a/examples/basic/A02_transit_as_mpls/README.md +++ b/examples/basic/A02a_transit_as_mpls/README.md @@ -38,7 +38,7 @@ mpls = Mpls() Unlike OSPF and IBGP, MPLS needs to be explicitly enabled for an autonomous system. This can be done by `Mpls::enableOn`: ```python -mpls.enableOn(150) +mpls.enableOn(2) ``` The `enableOn` call takes on parameter, the ASN to enable MPLS on. @@ -46,10 +46,10 @@ The `enableOn` call takes on parameter, the ASN to enable MPLS on. Here, only `r1` and `r4` are edge routers; thus, IBGP session will only be set up between them. `r2` and `r3` will only participate in OSPF and LDP. The topology looks like this: ``` - | AS150's MPLS backbone | + | AS2's MPLS backbone | | ____________ ibgp ___________ | | / \ | -as151 -|- as150_r1 -- as150_r2 -- as150_r3 -- as150_r4 -|- as152 +as151 -|- as2_r1 -- as2_r2 -- as2_r3 -- as2_r4 -|- as152 | | ``` @@ -58,10 +58,10 @@ Since `r2` and `r3` don't carry the tables from AS151 and AS152, traceroute will ``` HOST: 0e58e675b98b Loss% Snt Last Avg Best Wrst StDev 1.|-- 10.152.0.254 0.0% 10 0.1 0.1 0.1 0.1 0.0 - 2.|-- 10.101.0.150 0.0% 10 0.1 0.1 0.1 0.2 0.0 + 2.|-- 10.101.0.2 0.0% 10 0.1 0.1 0.1 0.2 0.0 3.|-- ??? 100.0 10 0.0 0.0 0.0 0.0 0.0 4.|-- ??? 100.0 10 0.0 0.0 0.0 0.0 0.0 - 5.|-- 10.150.0.254 0.0% 10 0.1 0.1 0.1 0.2 0.0 + 5.|-- 10.2.0.254 0.0% 10 0.1 0.1 0.1 0.2 0.0 6.|-- 10.100.0.151 0.0% 10 0.3 0.2 0.1 0.3 0.1 7.|-- 10.151.0.71 0.0% 10 0.2 0.2 0.1 0.3 0.0 ``` @@ -81,3 +81,36 @@ listening on net1, link-type EN10MB (Ethernet), capture size 262144 bytes 4 packets received by filter 0 packets dropped by kernel ``` + +## Standardized TestRunner lifecycle + +This example also includes an `example.yaml` manifest so it can be compiled, +built, started, probed, tested, and stopped by `seedemu.testing`. +Run these commands from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/basic/A02a_transit_as_mpls/example.yaml +python seedemu/testing/cli.py compile examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +python seedemu/testing/cli.py build examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +python seedemu/testing/cli.py up examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +python seedemu/testing/cli.py probe examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +python seedemu/testing/cli.py test examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +python seedemu/testing/cli.py down examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/basic/A02a_transit_as_mpls/example.yaml --artifact-dir ci-artifacts/a02-transit-as-mpls +``` + +The manifest uses declarative probes for simple runtime checks: + +- AS151 fetches the AS152 web service through AS2. +- AS152 fetches the AS151 web service through AS2. +- AS151 can ping AS152 through AS2. + +The `test_runtime.py` program demonstrates a custom test for cases where YAML +would become awkward. It checks the same end-to-end reachability and also +verifies that the AS2 transit routers have MPLS/LDP configuration files for +the expected internal links. diff --git a/examples/basic/A02a_transit_as_mpls/example.yaml b/examples/basic/A02a_transit_as_mpls/example.yaml new file mode 100644 index 000000000..a895fa4da --- /dev/null +++ b/examples/basic/A02a_transit_as_mpls/example.yaml @@ -0,0 +1,89 @@ +id: basic-a02a-transit-as-mpls +name: Transit AS MPLS With LDP +description: A transit-AS example where AS2 carries traffic between AS151 and AS152 over MPLS with LDP. +runner: internet +script: transit_as_mpls.py +platform: amd +features: + - ipv4-default + - transit-as + - mpls + - ebgp-private-peering + - web-service + +compile: + enabled: true + output: output + clean: + - output + - base_component.bin + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: MPLS transit AS services are running + type: compose-ps + services: + - brdnode_2_r1 + - rnode_2_r2 + - rnode_2_r3 + - brdnode_2_r4 + - brdnode_151_router0 + - brdnode_152_router0 + - hnode_151_web + - hnode_152_web + retries: 30 + interval: 2 + + - name: AS151 local web service is ready + type: exec + service: hnode_151_web + command: curl -fsS http://127.0.0.1 >/dev/null + expect_exit: 0 + retries: 30 + interval: 2 + + - name: AS152 local web service is ready + type: exec + service: hnode_152_web + command: curl -fsS http://127.0.0.1 >/dev/null + expect_exit: 0 + retries: 30 + interval: 2 + +probes: + - name: AS151 reaches AS152 web service through MPLS transit AS2 + type: exec + service: hnode_151_web + command: curl -fsS http://10.152.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS152 reaches AS151 web service through MPLS transit AS2 + type: exec + service: hnode_152_web + command: curl -fsS http://10.151.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS151 can ping AS152 through MPLS transit AS2 + type: ping + service: hnode_151_web + target: 10.152.0.71 + count: 3 + retries: 30 + interval: 5 + +test_programs: + - name: MPLS runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A02a_transit_as_mpls/test_runtime.py b/examples/basic/A02a_transit_as_mpls/test_runtime.py new file mode 100644 index 000000000..ec26852bb --- /dev/null +++ b/examples/basic/A02a_transit_as_mpls/test_runtime.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + r1 = test.require_service(2, "r1") + r2 = test.require_service(2, "r2") + r3 = test.require_service(2, "r3") + r4 = test.require_service(2, "r4") + + if web151 and web152: + test.exec_check("AS151 fetches AS152 web service", web151, "curl -fsS http://{} >/dev/null".format(web152.address)) + test.exec_check("AS152 fetches AS151 web service", web152, "curl -fsS http://{} >/dev/null".format(web151.address)) + + if r1: + test.exec_check( + "r1 has MPLS/LDP enabled on the internal transit network", + r1, + "test -s /mpls_ifaces.txt && grep -q '^net0$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r2: + test.exec_check( + "r2 is an MPLS non-edge router with both internal links enabled", + r2, + "grep -q '^net0$' /mpls_ifaces.txt && grep -q '^net1$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r3: + test.exec_check( + "r3 is an MPLS non-edge router with both internal links enabled", + r3, + "grep -q '^net1$' /mpls_ifaces.txt && grep -q '^net2$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r4: + test.exec_check( + "r4 has MPLS/LDP enabled on the internal transit network", + r4, + "test -s /mpls_ifaces.txt && grep -q '^net2$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + + test.write_summary("a02-mpls-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A02a_transit_as_mpls/transit_as_mpls.py b/examples/basic/A02a_transit_as_mpls/transit_as_mpls.py new file mode 100644 index 000000000..bda023af8 --- /dev/null +++ b/examples/basic/A02a_transit_as_mpls/transit_as_mpls.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Mpls, Ospf, PeerRelationship, Routing +from seedemu.services import WebService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A02a transit AS MPLS-with-LDP example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + ebgp = Ebgp() + mpls = Mpls() + web = WebService() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + ############################################################################### + # Create and set up the transit AS (AS2) + + as2 = base.createAutonomousSystem(2) + + # Create 3 internal networks + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createNetwork("net2") + + # Create four routers and link them in a linear structure: + # ix100 <--> r1 <--> r2 <--> r3 <--> r4 <--> ix101 + # r1 and r4 are BGP routers because they are connected to Internet exchanges. + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") + as2.createRouter("r2").joinNetwork("net0").joinNetwork("net1") + as2.createRouter("r3").joinNetwork("net1").joinNetwork("net2") + as2.createRouter("r4").joinNetwork("net2").joinNetwork("ix101") + + # Enable MPLS in the transit AS. + mpls.enableOn(2) + + ############################################################################### + # Create and set up the stub AS (AS151) + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as151.createHost("web").joinNetwork("net0") + web.install("web151") + emu.addBinding(Binding("web151", filter=Filter(nodeName="web", asn=151))) + + ############################################################################### + # Create and set up the stub AS (AS152) + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + as152.createHost("web").joinNetwork("net0") + web.install("web152") + emu.addBinding(Binding("web152", filter=Filter(nodeName="web", asn=152))) + + ############################################################################### + # Make AS2 the Internet service provider for AS151 and AS152. + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + emu.addLayer(mpls) + emu.addLayer(web) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + ############################################################################### + # Save the emulation if needed. This must happen before rendering. + emu.dump(dumpfile) + ############################################################################### + return + + ############################################################################### + # Render the emulation. + if render: + emu.render() + + ############################################################################### + # Generate the Docker files. + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A02b_manual_mpls/README.md b/examples/basic/A02b_manual_mpls/README.md new file mode 100644 index 000000000..4d846570a --- /dev/null +++ b/examples/basic/A02b_manual_mpls/README.md @@ -0,0 +1,108 @@ +# Transit AS Manual MPLS + +This example demonstrates manual MPLS label-table setup. It is the companion to +`A02a_transit_as_mpls`, which uses LDP to distribute labels automatically. + +The topology has one transit AS and three stub ASes. `AS2` has three BGP edge +routers that also act as MPLS provider-edge routers, plus three internal MPLS +core routers: + +```text + AS151 + | + IX101 + | + e1 + | + r1 + / \ + / \ + r2 ---- r3 + | | + e2 e3 + | | + IX102 IX103 + | | + AS152 AS153 +``` + +`e1`, `e2`, and `e3` are AS2's BGP edge routers and MPLS ingress/egress +routers: + +- `e1`: `IX101`, connected to `AS151`. +- `e2`: `IX102`, connected to `AS152`. +- `e3`: `IX103`, connected to `AS153`. + +`r1`, `r2`, and `r3` are internal AS2 routers. They form the triangle MPLS +core and do not connect directly to any IX: + +- `e1 -- r1` +- `e2 -- r2` +- `e3 -- r3` +- `r1 -- r2` +- `r2 -- r3` +- `r3 -- r1` + +The example uses normal BGP/OSPF routing, but it does not use the SEED Emulator +`Mpls` layer and does not use LDP. Instead, AS2's edge and core routers install +static MPLS routes from `/manual_mpls_setup.sh`. The edge routers push labels +when traffic enters the provider backbone and pop labels when traffic leaves it. +The internal routers only swap labels across the provider core. + +## Manual Labels + +The labels are directional: + +| Traffic | Edge push | Core swap | Edge pop | +| --- | --- | --- | +| AS151 to AS152 | `e1` pushes `200` to `r1` | `r1`: `200 -> 201`; `r2`: `201 -> 202` | `e2` pops `202`, forwards to AS152 | +| AS151 to AS153 | `e1` pushes `210` to `r1` | `r1`: `210 -> 211`; `r3`: `211 -> 212` | `e3` pops `212`, forwards to AS153 | +| AS152 to AS151 | `e2` pushes `300` to `r2` | `r2`: `300 -> 301`; `r1`: `301 -> 302` | `e1` pops `302`, forwards to AS151 | +| AS152 to AS153 | `e2` pushes `310` to `r2` | `r2`: `310 -> 311`; `r3`: `311 -> 312` | `e3` pops `312`, forwards to AS153 | +| AS153 to AS151 | `e3` pushes `400` to `r3` | `r3`: `400 -> 401`; `r1`: `401 -> 402` | `e1` pops `402`, forwards to AS151 | +| AS153 to AS152 | `e3` pushes `410` to `r3` | `r3`: `410 -> 411`; `r2`: `411 -> 412` | `e2` pops `412`, forwards to AS152 | + +The Docker host must support Linux MPLS: + +```sh +sudo modprobe mpls_router +sudo modprobe mpls_iptunnel +sudo modprobe mpls_gso +``` + +GitHub-hosted runners may not provide these modules, so this example is not +included in the default CI workflow. + +## Standard Arguments + +```sh +python examples/basic/A02b_manual_mpls/manual_mpls.py amd +python examples/basic/A02b_manual_mpls/manual_mpls.py --platform amd --output examples/basic/A02b_manual_mpls/output +python examples/basic/A02b_manual_mpls/manual_mpls.py --dumpfile examples/basic/A02b_manual_mpls/manual_mpls.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +```sh +python seedemu/testing/cli.py all examples/basic/A02b_manual_mpls/example.yaml --artifact-dir ci-artifacts/a02b-manual-mpls +``` + +The runtime test uses `ComposeRuntimeTest`. It verifies cross-AS reachability, +checks that edge routers install the expected push/pop labels, checks that core +routers install the expected swap labels, and confirms that LDP is not used. + +## Learning Activities + +- Trace AS151 to AS152 and identify labels `200`, `201`, and `202`. +- Trace AS151 to AS153 and identify labels `210`, `211`, and `212`. +- Break one label in `/manual_mpls_setup.sh` and observe which destination fails. +- Compare this example with `A02a_transit_as_mpls` to see what LDP automates. diff --git a/examples/basic/A02b_manual_mpls/example.yaml b/examples/basic/A02b_manual_mpls/example.yaml new file mode 100644 index 000000000..28ffd0b7e --- /dev/null +++ b/examples/basic/A02b_manual_mpls/example.yaml @@ -0,0 +1,110 @@ +id: basic-a02b-manual-mpls +name: Transit AS Manual MPLS +description: A transit AS with MPLS provider-edge routers and a triangle core configured manually instead of using LDP. +runner: internet +script: manual_mpls.py +platform: amd +features: + - ipv4-default + - transit-as + - manual-mpls + - ebgp-private-peering + - ibgp + - ospf + - web-service + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: manual MPLS triangle services are running + type: compose-ps + services: + - brdnode_2_e1 + - brdnode_2_e2 + - brdnode_2_e3 + - rnode_2_r1 + - rnode_2_r2 + - rnode_2_r3 + - brdnode_151_router0 + - brdnode_152_router0 + - brdnode_153_router0 + - hnode_151_web + - hnode_152_web + - hnode_153_web + retries: 30 + interval: 2 + + - name: AS151 local web service is ready + type: exec + service: hnode_151_web + command: curl -fsS http://127.0.0.1 >/dev/null + expect_exit: 0 + retries: 30 + interval: 2 + + - name: AS152 local web service is ready + type: exec + service: hnode_152_web + command: curl -fsS http://127.0.0.1 >/dev/null + expect_exit: 0 + retries: 30 + interval: 2 + + - name: AS153 local web service is ready + type: exec + service: hnode_153_web + command: curl -fsS http://127.0.0.1 >/dev/null + expect_exit: 0 + retries: 30 + interval: 2 + +probes: + - name: AS151 reaches AS152 through manual MPLS + type: exec + service: hnode_151_web + command: curl -fsS http://10.152.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS151 reaches AS153 through manual MPLS + type: exec + service: hnode_151_web + command: curl -fsS http://10.153.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS152 reaches AS151 through manual MPLS + type: ping + service: hnode_152_web + target: 10.151.0.71 + count: 3 + retries: 30 + interval: 5 + + - name: AS153 reaches AS151 through manual MPLS + type: ping + service: hnode_153_web + target: 10.151.0.71 + count: 3 + retries: 30 + interval: 5 + +test_programs: + - name: A02b manual MPLS runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A02b_manual_mpls/manual_mpls.py b/examples/basic/A02b_manual_mpls/manual_mpls.py new file mode 100755 index 000000000..510652673 --- /dev/null +++ b/examples/basic/A02b_manual_mpls/manual_mpls.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.services import WebService + + +MANUAL_MPLS = { + "e1": { + "ifaces": ["net_e1_r1"], + "routes": [ + "ip route replace 10.152.0.0/24 encap mpls 200 via inet 10.2.101.12 dev net_e1_r1", + "ip route replace 10.153.0.0/24 encap mpls 210 via inet 10.2.101.12 dev net_e1_r1", + "ip -f mpls route replace 302 via inet 10.101.0.151 dev ix101", + "ip -f mpls route replace 402 via inet 10.101.0.151 dev ix101", + ], + }, + "e2": { + "ifaces": ["net_e2_r2"], + "routes": [ + "ip route replace 10.151.0.0/24 encap mpls 300 via inet 10.2.102.12 dev net_e2_r2", + "ip route replace 10.153.0.0/24 encap mpls 310 via inet 10.2.102.12 dev net_e2_r2", + "ip -f mpls route replace 202 via inet 10.102.0.152 dev ix102", + "ip -f mpls route replace 412 via inet 10.102.0.152 dev ix102", + ], + }, + "e3": { + "ifaces": ["net_e3_r3"], + "routes": [ + "ip route replace 10.151.0.0/24 encap mpls 400 via inet 10.2.103.12 dev net_e3_r3", + "ip route replace 10.152.0.0/24 encap mpls 410 via inet 10.2.103.12 dev net_e3_r3", + "ip -f mpls route replace 212 via inet 10.103.0.153 dev ix103", + "ip -f mpls route replace 312 via inet 10.103.0.153 dev ix103", + ], + }, + "r1": { + "ifaces": ["net_e1_r1", "net12", "net31"], + "routes": [ + "ip -f mpls route replace 200 as 201 via inet 10.2.12.12 dev net12", + "ip -f mpls route replace 210 as 211 via inet 10.2.31.13 dev net31", + "ip -f mpls route replace 301 as 302 via inet 10.2.101.11 dev net_e1_r1", + "ip -f mpls route replace 401 as 402 via inet 10.2.101.11 dev net_e1_r1", + ], + }, + "r2": { + "ifaces": ["net_e2_r2", "net12", "net23"], + "routes": [ + "ip -f mpls route replace 201 as 202 via inet 10.2.102.11 dev net_e2_r2", + "ip -f mpls route replace 300 as 301 via inet 10.2.12.11 dev net12", + "ip -f mpls route replace 310 as 311 via inet 10.2.23.13 dev net23", + "ip -f mpls route replace 411 as 412 via inet 10.2.102.11 dev net_e2_r2", + ], + }, + "r3": { + "ifaces": ["net_e3_r3", "net23", "net31"], + "routes": [ + "ip -f mpls route replace 211 as 212 via inet 10.2.103.11 dev net_e3_r3", + "ip -f mpls route replace 311 as 312 via inet 10.2.103.11 dev net_e3_r3", + "ip -f mpls route replace 400 as 401 via inet 10.2.31.11 dev net31", + "ip -f mpls route replace 410 as 411 via inet 10.2.23.12 dev net23", + ], + }, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A02b manual MPLS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def manual_mpls_script(router_name: str) -> str: + config = MANUAL_MPLS[router_name] + lines = [ + "#!/bin/bash", + "set -e", + "mount -o remount rw /proc/sys 2> /dev/null || true", + "test -d /proc/sys/net/mpls", + "echo 1048575 > /proc/sys/net/mpls/platform_labels", + "sleep 5", + ] + for iface in config["ifaces"]: + lines.append('echo 1 > "/proc/sys/net/mpls/conf/{}/input"'.format(iface)) + lines.extend(config["routes"]) + lines.append("ip -f mpls route show > /manual_mpls_table.txt") + lines.append("ip route show > /manual_ip_routes.txt") + return "\n".join(lines) + "\n" + + +def install_manual_mpls(router, router_name: str) -> None: + router.setPrivileged(True) + router.setFile("/manual_mpls_setup.sh", manual_mpls_script(router_name)) + router.appendStartCommand("chmod +x /manual_mpls_setup.sh", isPostConfigCommand=True) + router.appendStartCommand("/manual_mpls_setup.sh", isPostConfigCommand=True) + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + ebgp = Ebgp() + web = WebService() + + base.createInternetExchange(101) + base.createInternetExchange(102) + base.createInternetExchange(103) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net_e1_r1", prefix="10.2.101.0/24") + as2.createNetwork("net_e2_r2", prefix="10.2.102.0/24") + as2.createNetwork("net_e3_r3", prefix="10.2.103.0/24") + as2.createNetwork("net12", prefix="10.2.12.0/24") + as2.createNetwork("net23", prefix="10.2.23.0/24") + as2.createNetwork("net31", prefix="10.2.31.0/24") + + e1 = as2.createRouter("e1") + e1.joinNetwork("ix101", "10.101.0.2").joinNetwork("net_e1_r1", "10.2.101.11") + + e2 = as2.createRouter("e2") + e2.joinNetwork("ix102", "10.102.0.2").joinNetwork("net_e2_r2", "10.2.102.11") + + e3 = as2.createRouter("e3") + e3.joinNetwork("ix103", "10.103.0.2").joinNetwork("net_e3_r3", "10.2.103.11") + + r1 = as2.createRouter("r1") + r1.joinNetwork("net_e1_r1", "10.2.101.12").joinNetwork("net12", "10.2.12.11").joinNetwork("net31", "10.2.31.11") + + r2 = as2.createRouter("r2") + r2.joinNetwork("net_e2_r2", "10.2.102.12").joinNetwork("net12", "10.2.12.12").joinNetwork("net23", "10.2.23.12") + + r3 = as2.createRouter("r3") + r3.joinNetwork("net_e3_r3", "10.2.103.12").joinNetwork("net23", "10.2.23.13").joinNetwork("net31", "10.2.31.13") + + for router, router_name in ((e1, "e1"), (e2, "e2"), (e3, "e3"), (r1, "r1"), (r2, "r2"), (r3, "r3")): + install_manual_mpls(router, router_name) + + for asn, ix in ((151, 101), (152, 102), (153, 103)): + current_as = base.createAutonomousSystem(asn) + current_as.createNetwork("net0") + current_as.createRouter("router0").joinNetwork("net0").joinNetwork("ix{}".format(ix), "10.{}.0.{}".format(ix, asn)) + current_as.createHost("web").joinNetwork("net0") + web.install("web{}".format(asn)) + emu.addBinding(Binding("web{}".format(asn), filter=Filter(nodeName="web", asn=asn))) + ebgp.addPrivatePeering(ix, 2, asn, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + emu.addLayer(web) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A02b_manual_mpls/test_runtime.py b/examples/basic/A02b_manual_mpls/test_runtime.py new file mode 100644 index 000000000..359b8dc9e --- /dev/null +++ b/examples/basic/A02b_manual_mpls/test_runtime.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + web153 = test.require_service(153, "web") + e1 = test.require_service(2, "e1") + e2 = test.require_service(2, "e2") + e3 = test.require_service(2, "e3") + r1 = test.require_service(2, "r1") + r2 = test.require_service(2, "r2") + r3 = test.require_service(2, "r3") + + if web151 and web152: + test.exec_check("AS151 fetches AS152 web service", web151, "curl -fsS http://{} >/dev/null".format(web152.address)) + test.exec_check("AS152 reaches AS151 by ICMP", web152, "ping -c 3 {} >/dev/null".format(web151.address)) + if web151 and web153: + test.exec_check("AS151 fetches AS153 web service", web151, "curl -fsS http://{} >/dev/null".format(web153.address)) + test.exec_check("AS153 reaches AS151 by ICMP", web153, "ping -c 3 {} >/dev/null".format(web151.address)) + + if e1: + test.exec_check( + "e1 pushes labels toward the core and pops labels for AS151", + e1, + "test -x /manual_mpls_setup.sh && " + "grep -q 'encap mpls 200' /manual_mpls_setup.sh && " + "grep -q 'encap mpls 210' /manual_mpls_setup.sh && " + "grep -q 'route replace 302 via inet 10.101.0.151 dev ix101' /manual_mpls_setup.sh && " + "grep -q 'route replace 402 via inet 10.101.0.151 dev ix101' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + if e2: + test.exec_check( + "e2 pushes labels toward the core and pops labels for AS152", + e2, + "test -x /manual_mpls_setup.sh && " + "grep -q 'encap mpls 300' /manual_mpls_setup.sh && " + "grep -q 'encap mpls 310' /manual_mpls_setup.sh && " + "grep -q 'route replace 202 via inet 10.102.0.152 dev ix102' /manual_mpls_setup.sh && " + "grep -q 'route replace 412 via inet 10.102.0.152 dev ix102' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + if e3: + test.exec_check( + "e3 pushes labels toward the core and pops labels for AS153", + e3, + "test -x /manual_mpls_setup.sh && " + "grep -q 'encap mpls 400' /manual_mpls_setup.sh && " + "grep -q 'encap mpls 410' /manual_mpls_setup.sh && " + "grep -q 'route replace 212 via inet 10.103.0.153 dev ix103' /manual_mpls_setup.sh && " + "grep -q 'route replace 312 via inet 10.103.0.153 dev ix103' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + + if r1: + test.exec_check( + "r1 swaps labels between e1 and the core triangle", + r1, + "test -x /manual_mpls_setup.sh && " + "grep -q 'route replace 200 as 201' /manual_mpls_setup.sh && " + "grep -q 'route replace 210 as 211' /manual_mpls_setup.sh && " + "grep -q 'route replace 301 as 302' /manual_mpls_setup.sh && " + "grep -q 'route replace 401 as 402' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + if r2: + test.exec_check( + "r2 swaps labels between e2 and the core triangle", + r2, + "test -x /manual_mpls_setup.sh && " + "grep -q 'route replace 201 as 202' /manual_mpls_setup.sh && " + "grep -q 'route replace 300 as 301' /manual_mpls_setup.sh && " + "grep -q 'route replace 310 as 311' /manual_mpls_setup.sh && " + "grep -q 'route replace 411 as 412' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + if r3: + test.exec_check( + "r3 swaps labels between e3 and the core triangle", + r3, + "test -x /manual_mpls_setup.sh && " + "grep -q 'route replace 211 as 212' /manual_mpls_setup.sh && " + "grep -q 'route replace 311 as 312' /manual_mpls_setup.sh && " + "grep -q 'route replace 400 as 401' /manual_mpls_setup.sh && " + "grep -q 'route replace 410 as 411' /manual_mpls_setup.sh && " + "test -s /manual_mpls_table.txt && test ! -e /mpls_ifaces.txt", + ) + + for router in (e1, e2, e3, r1, r2, r3): + if router: + test.exec_check( + "{} does not use LDP".format(router.name), + router, + "test ! -e /etc/frr/frr.conf || ! grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + + test.write_summary("a02b-manual-mpls-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A03_real_world/README.md b/examples/basic/A03_real_world/README.md deleted file mode 100644 index b707052f6..000000000 --- a/examples/basic/A03_real_world/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Real-world interaction - -This example demonstrates two features related to the real world. -The first feature allows outside machines to connect to the -emulator, so they can participate in the emulation. The way -to achieve this is through VPN. - -The second feature allows an emulation to include real-world -autonomous systems. A real-world AS included in the emulation -will collect the real network prefixes from the real Internet, -and announce them inside the emulator. Packets reaching this AS -will exit the emulator and be routed to the real destination. -Responses from the outside will come back to this real-world AS -and be routed to the final destination in the emulator. - - -## Import and Create Required Components - -Most of this part is the same as that in `01-transit-as`. The only difference is the following: - -```python -ovpn = OpenVpnRemoteAccessProvider() -``` - -This creates an OpenVPN remote access provider. A remote access provider contains the logic to enable remote access to a network. It takes the following options: - -- `startPort`: The starting port number to assign to the OpenVPN servers. Default to `65000`. -- `naddrs`: Number of addresses to reserve from the network. Default to `8`. -- `ovpnCa`: CA to use for the OpenVPN server. Default to `None` (uses bulletin CA). -- `ovpnCert`: Server certificate to use for the OpenVPN server. Default to `None` (uses bulletin certificate). -- `ovpnKey`: Server key to use for the OpenVPN server. Default to `None` (uses bulletin key). - -For details on how to connect to the OpenVPN server with bulletin CA/cert/key, -see [misc/openvpn-remote-access](/misc/openvpn-remote-access). - -The way a remote access provider works is that the emulator -will provide the remote access provider with the followings: - -- The network to enable remote access on. -- A for-service bridging network: this network is not a part of the emulation. Instead, it was used by special services like this to communicate with the emulator host. -- A bridging node: this node is not a part of emulation. It is a special node that will have access to both the for-service bridging network and the network to enable remote access on. - -Then, what a remote access provider usually does is: - -- Start a VPN server, listen for incoming connections on the for-service bridge network, so the emulator host can port-forward to the VPN server and allow hosts in the real world to connect. -- Add the VPN server interface and the network to enable remote access on to the same bridge so that clients can access the network. - -In the OpenVPN access provider, besides the two steps above, also reserve some IP addresses (8 by default) for the client IP address pool. - -However, note that a remote access provider does not necessarily create a VPN server - they can also be a VPN client that connect to another emulator or just a regular Linux bridge, to connect to, say, a network created by some other virtualization software. - - -## Enable Remote Access - -To allow remote access on a network, we just need to call the enabling API: - -```python -as151.createNetwork('net0').enableRemoteAccess(ovpn) -``` - -The `Network::enableRemoteAccess` call enables remote access to a network. `enableRemoteAccess` takes only one parameter, the remote access provider. - - -## Create a Real-World Stub AS - -A real-world AS is an AS, so we will first create an AS: - -```python -as11872 = base.createAutonomousSystem(11872) -``` - -We will create a special router in this AS: - -```python -as11872.createRealWorldRouter('rw') - -``` - -The `createRealWorldRouter` call takes three parameters: - -- `name`: name of the node. -- `hideHops`: enable hide hops feature. When `True`, the router will hide real world hops from traceroute. This works by setting TTL = 64 to all real world destinations on `POSTROUTING`. Default to `True`. -- `prefixes`: list of prefix. Can be a list of prefixes or `None`. When set to `None`, the router will automatically fetch the list of prefixes announced by the autonomous system in the real world. Default to `None`. - - -We will connect this autonomous system to an internet exchange. -Here, we picked `IX101`. We need to override the auto address assignment, -as 11872 is out of the 2~254 range: - -```python -as11872.createRealWorldRouter('rw').joinNetwork('ix101', '10.101.0.118') -``` diff --git a/examples/basic/A03_real_world/real_world.py b/examples/basic/A03_real_world/real_world.py deleted file mode 100755 index dbd4db33d..000000000 --- a/examples/basic/A03_real_world/real_world.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - -from seedemu.layers import Base, Routing, Ebgp, PeerRelationship, Ibgp, Ospf -from seedemu.services import WebService -from seedemu.core import Emulator, Binding, Filter -from seedemu.raps import OpenVpnRemoteAccessProvider -from seedemu.compiler import Docker, Platform -import os, sys - -def run(dumpfile = None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - ibgp = Ibgp() - ospf = Ospf() - web = WebService() - ovpn = OpenVpnRemoteAccessProvider() - - ############################################################################### - - base.createInternetExchange(100) - base.createInternetExchange(101) - - ############################################################################### - # Create a transit AS (AS-150) - - as150 = base.createAutonomousSystem(150) - - as150.createNetwork('net0') - as150.createNetwork('net1') - as150.createNetwork('net2') - - # Create 4 routers: r1 and r4 are BGP routers (connected to internet exchange) - as150.createRouter('r1').joinNetwork('ix100').joinNetwork('net0') - as150.createRouter('r2').joinNetwork('net0').joinNetwork('net1') - as150.createRouter('r3').joinNetwork('net1').joinNetwork('net2') - as150.createRouter('r4').joinNetwork('net2').joinNetwork('ix101') - - - ############################################################################### - # Create AS-151 - - as151 = base.createAutonomousSystem(151) - - # Create a network and enable the access from real world - as151.createNetwork('net0').enableRemoteAccess(ovpn) - as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - - # Create a web host - as151.createHost('web').joinNetwork('net0') - web.install('web1') - emu.addBinding(Binding('web1', filter = Filter(asn = 151, nodeName = 'web'))) - - - ############################################################################### - # Create AS-152 - - as152 = base.createAutonomousSystem(152) - - # Create a network and enable the access from real world - as152.createNetwork('net0').enableRemoteAccess(ovpn) - as152.createRouter('router0').joinNetwork('net0').joinNetwork('ix101') - - # Create a web host - as152.createHost('web').joinNetwork('net0') - web.install('web2') - emu.addBinding(Binding('web2', filter = Filter(asn = 152, nodeName = 'web'))) - - - ############################################################################### - # Create a real-world AS. - # AS11872 is the Syracuse University's autonomous system - # The network prefixes announced by this AS will be collected from the real Internet - # Packets coming into this AS will be routed out to the real world. - - as11872 = base.createAutonomousSystem(11872) - as11872.createRealWorldRouter('rw').joinNetwork('ix101', '10.101.0.118') - - - ############################################################################### - # BGP peering - - ebgp.addPrivatePeering(100, 150, 151, abRelationship = PeerRelationship.Provider) - ebgp.addPrivatePeering(101, 150, 152, abRelationship = PeerRelationship.Provider) - ebgp.addPrivatePeering(101, 150, 11872, abRelationship = PeerRelationship.Unfiltered) - - - ############################################################################### - # Rendering - - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.addLayer(ibgp) - emu.addLayer(ospf) - emu.addLayer(web) - - if dumpfile is not None: - emu.dump(dumpfile) - else: - emu.render() - - ############################################################################### - # Compilation - - emu.compile(Docker(platform=platform), './output', override=True) - -if __name__ == "__main__": - run() \ No newline at end of file diff --git a/examples/basic/A03a_out_to_real_world/README.md b/examples/basic/A03a_out_to_real_world/README.md new file mode 100644 index 000000000..1caa040fa --- /dev/null +++ b/examples/basic/A03a_out_to_real_world/README.md @@ -0,0 +1,65 @@ +# Out To The Real World + +This example demonstrates traffic going from the emulator out to the real +world. It intentionally does not include OpenVPN remote access. + +The topology has: + +- `AS2`: transit AS between `IX100` and `IX101`. +- `AS151`: stub AS at `IX100` with a web host. +- `AS152`: stub AS at `IX101` with a web host. +- `AS20940`: Akamai real-world AS at `IX101`. + +`example.com` is commonly served from Akamai infrastructure, so this example +uses Akamai to demonstrate real-world prefix injection. By default, the example +uses a deterministic Akamai prefix: + +```text +23.192.228.0/24 +``` + +Pass `--live-prefixes` to fetch live AS20940 prefixes instead. + +## Standard Arguments + +```sh +python examples/basic/A03a_out_to_real_world/out_to_real_world.py amd +python examples/basic/A03a_out_to_real_world/out_to_real_world.py --platform amd --output examples/basic/A03a_out_to_real_world/output +python examples/basic/A03a_out_to_real_world/out_to_real_world.py --dumpfile examples/basic/A03a_out_to_real_world/out_to_real_world.bin +python examples/basic/A03a_out_to_real_world/out_to_real_world.py --live-prefixes +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--live-prefixes`: fetch live AS20940 prefixes. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +```sh +python seedemu/testing/cli.py all examples/basic/A03a_out_to_real_world/example.yaml --artifact-dir ci-artifacts/a03a +``` + +The automatic tests avoid depending on live Internet availability. They check +emulated reachability, Akamai real-world router configuration, and that OpenVPN +remote access is not part of this example. + +## Manual Real-World Reachability Test + +The automatic CI tests do not require live Internet access. To manually verify +that an emulated host can reach the outside world, start the compiled emulation, +go to this example folder, and run: + +```sh +sh test_real_world_reachability.sh +``` + +The script runs one command from `hnode_151_web`: ping `23.192.228.80` over +IPv4. This address is inside the deterministic Akamai prefix used by this +example, so the check does not depend on DNS or a web server. If the command +succeeds, the emulator can reach the real world from inside. diff --git a/examples/basic/A03a_out_to_real_world/example.yaml b/examples/basic/A03a_out_to_real_world/example.yaml new file mode 100644 index 000000000..276636a06 --- /dev/null +++ b/examples/basic/A03a_out_to_real_world/example.yaml @@ -0,0 +1,66 @@ +id: basic-a03a-out-to-real-world +name: Out To The Real World +description: Demonstrates outbound real-world routing through Akamai AS20940. +runner: internet +script: out_to_real_world.py +platform: amd +features: + - ipv4-default + - real-world-router + - web-service + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A03a representative services are running + type: compose-ps + services: + - brdnode_2_r1 + - rnode_2_r2 + - rnode_2_r3 + - brdnode_2_r4 + - brdnode_151_router0 + - brdnode_152_router0 + - hnode_151_web + - hnode_152_web + - brdnode_20940_rw + retries: 30 + interval: 3 + +probes: + - name: AS151 reaches AS152 through AS2 + type: exec + service: hnode_151_web + command: curl -fsS http://10.152.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS152 reaches AS151 through AS2 + type: exec + service: hnode_152_web + command: curl -fsS http://10.151.0.71 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + +test_programs: + - name: A03a real-world structural validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A03a_out_to_real_world/out_to_real_world.py b/examples/basic/A03a_out_to_real_world/out_to_real_world.py new file mode 100755 index 000000000..6ed3a9cc3 --- /dev/null +++ b/examples/basic/A03a_out_to_real_world/out_to_real_world.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.services import WebService + + +AKAMAI_ASN = 20940 +AKAMAI_EXAMPLE_PREFIXES = ["23.192.228.0/24"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A03a out-to-real-world example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--live-prefixes", + action="store_true", + help="Fetch live prefixes for AS20940 instead of using deterministic example.com-related prefixes.", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator(use_live_prefixes=False) -> Emulator: + emu = Emulator() + base = Base() + ebgp = Ebgp() + web = WebService() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createNetwork("net2") + as2.createRouter("r1").joinNetwork("ix100").joinNetwork("net0") + as2.createRouter("r2").joinNetwork("net0").joinNetwork("net1") + as2.createRouter("r3").joinNetwork("net1").joinNetwork("net2") + as2.createRouter("r4").joinNetwork("net2").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as151.createHost("web").joinNetwork("net0") + web.install("web151") + emu.addBinding(Binding("web151", filter=Filter(asn=151, nodeName="web"))) + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + as152.createHost("web").joinNetwork("net0") + web.install("web152") + emu.addBinding(Binding("web152", filter=Filter(asn=152, nodeName="web"))) + + as20940 = base.createAutonomousSystem(AKAMAI_ASN) + prefixes = None if use_live_prefixes else AKAMAI_EXAMPLE_PREFIXES + as20940.createRealWorldRouter("rw", prefixes=prefixes).joinNetwork("ix101", "10.101.0.209") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, AKAMAI_ASN, abRelationship=PeerRelationship.Unfiltered) + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + emu.addLayer(web) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, + use_live_prefixes=False, +): + emu = build_emulator(use_live_prefixes=use_live_prefixes) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + use_live_prefixes=args.live_prefixes, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A03a_out_to_real_world/test_real_world_reachability.sh b/examples/basic/A03a_out_to_real_world/test_real_world_reachability.sh new file mode 100755 index 000000000..5056dbde5 --- /dev/null +++ b/examples/basic/A03a_out_to_real_world/test_real_world_reachability.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +docker compose -f output/docker-compose.yml exec -T hnode_151_web \ + ping -4 -c 5 -W 5 23.192.228.80 diff --git a/examples/basic/A03a_out_to_real_world/test_runtime.py b/examples/basic/A03a_out_to_real_world/test_runtime.py new file mode 100644 index 000000000..4e758a3f5 --- /dev/null +++ b/examples/basic/A03a_out_to_real_world/test_runtime.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + akamai = test.require_service(20940, "rw", "Akamai real-world router is generated") + router151 = test.require_service(151, "router0") + + if web151 and web152: + test.exec_check("AS151 fetches AS152 web service", web151, "curl -fsS http://{} >/dev/null".format(web152.address)) + + if akamai: + test.exec_check( + "Akamai real-world router has deterministic example prefix", + akamai, + "grep -q '23.192.228.0/24' /etc/bird/bird.conf", + ) + test.exec_check( + "Akamai real-world router has service-network route setup", + akamai, + "test -s /rw_configure_script && grep -q 'MASQUERADE' /rw_configure_script && grep -q 'sed -i' /rw_configure_script", + ) + + if router151: + test.exec_check("OpenVPN remote access is absent from out-to-real-world example", router151, "test ! -e /ovpn-server.conf") + + test.write_summary("a03a-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A03b_from_real_world/README.md b/examples/basic/A03b_from_real_world/README.md new file mode 100644 index 000000000..6a65a2a20 --- /dev/null +++ b/examples/basic/A03b_from_real_world/README.md @@ -0,0 +1,60 @@ +# From The Real World + +This example demonstrates real-world machines coming into the emulator with +OpenVPN remote access. It intentionally does not include a real-world AS router. + +The topology has: + +- `AS2`: transit AS between `IX100` and `IX101`. +- `AS151`: stub AS at `IX100`, with OpenVPN remote access enabled on `net0`. +- `AS152`: stub AS at `IX101`, with OpenVPN remote access enabled on `net0`. + +## OpenVPN Remote Access + +The OpenVPN remote access provider is created with: + +```python +ovpn = OpenVpnRemoteAccessProvider() +``` + +Remote access is enabled on selected networks: + +```python +as151.createNetwork("net0").enableRemoteAccess(ovpn) +as152.createNetwork("net0").enableRemoteAccess(ovpn) +``` + +For details on how to connect to the OpenVPN server with the built-in +CA/certificate/key, see: + +```text +misc/openvpn-remote-access +``` + +## Standard Arguments + +```sh +python examples/basic/A03b_from_real_world/from_real_world.py amd +python examples/basic/A03b_from_real_world/from_real_world.py --platform amd --output examples/basic/A03b_from_real_world/output +python examples/basic/A03b_from_real_world/from_real_world.py --dumpfile examples/basic/A03b_from_real_world/from_real_world.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +```sh +python seedemu/testing/cli.py all examples/basic/A03b_from_real_world/example.yaml --artifact-dir ci-artifacts/a03b +``` + +The automatic tests check emulated reachability and verify that OpenVPN bridge +nodes are generated. Manual lab activities can additionally connect an external +OpenVPN client into AS151 or AS152 and access the web hosts from outside the +emulator. diff --git a/examples/basic/A03b_from_real_world/example.yaml b/examples/basic/A03b_from_real_world/example.yaml new file mode 100644 index 000000000..d79a61222 --- /dev/null +++ b/examples/basic/A03b_from_real_world/example.yaml @@ -0,0 +1,65 @@ +id: basic-a03b-from-real-world +name: From The Real World +description: Demonstrates OpenVPN remote access from outside machines into emulated AS151 and AS152. +runner: internet +script: from_real_world.py +platform: amd +features: + - ipv4-default + - openvpn-remote-access + - web-service + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A03b representative services are running + type: compose-ps + services: + - brdnode_2_r1 + - rnode_2_r2 + - rnode_2_r3 + - brdnode_2_r4 + - brdnode_151_router0 + - brdnode_152_router0 + - hnode_151_web + - hnode_152_web + retries: 30 + interval: 3 + +probes: + - name: AS151 reaches AS152 through AS2 + type: exec + service: hnode_151_web + command: curl -fsS http://10.152.0.79 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS152 reaches AS151 through AS2 + type: exec + service: hnode_152_web + command: curl -fsS http://10.151.0.79 >/dev/null + expect_exit: 0 + retries: 30 + interval: 5 + +test_programs: + - name: A03b OpenVPN structural validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A03b_from_real_world/from_real_world.py b/examples/basic/A03b_from_real_world/from_real_world.py new file mode 100755 index 000000000..085bcb2c6 --- /dev/null +++ b/examples/basic/A03b_from_real_world/from_real_world.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.raps import OpenVpnRemoteAccessProvider +from seedemu.services import WebService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A03b from-real-world example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + emu = Emulator() + base = Base() + ebgp = Ebgp() + web = WebService() + ovpn = OpenVpnRemoteAccessProvider() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createNetwork("net2") + as2.createRouter("r1").joinNetwork("ix100").joinNetwork("net0") + as2.createRouter("r2").joinNetwork("net0").joinNetwork("net1") + as2.createRouter("r3").joinNetwork("net1").joinNetwork("net2") + as2.createRouter("r4").joinNetwork("net2").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0").enableRemoteAccess(ovpn) + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as151.createHost("web").joinNetwork("net0") + web.install("web151") + emu.addBinding(Binding("web151", filter=Filter(asn=151, nodeName="web"))) + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0").enableRemoteAccess(ovpn) + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + as152.createHost("web").joinNetwork("net0") + web.install("web152") + emu.addBinding(Binding("web152", filter=Filter(asn=152, nodeName="web"))) + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + emu.addLayer(web) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A03b_from_real_world/test_runtime.py b/examples/basic/A03b_from_real_world/test_runtime.py new file mode 100644 index 000000000..a8553e7b9 --- /dev/null +++ b/examples/basic/A03b_from_real_world/test_runtime.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def dockerfiles_containing(test: ComposeRuntimeTest, *patterns: str) -> list[str]: + matches = [] + for dockerfile in test.compose_file.parent.glob("*/Dockerfile"): + text = dockerfile.read_text(encoding="utf-8", errors="replace") + if all(pattern in text for pattern in patterns): + matches.append(dockerfile.parent.name) + return sorted(matches) + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + + if web151 and web152: + test.exec_check("AS151 fetches AS152 web service", web151, "curl -fsS http://{} >/dev/null".format(web152.address)) + test.exec_check("AS152 fetches AS151 web service", web152, "curl -fsS http://{} >/dev/null".format(web151.address)) + + openvpn_outputs = dockerfiles_containing(test, "/ovpn-server.conf", "/ovpn_startup") + test.structural_check( + "OpenVPN bridge nodes are generated", + len(openvpn_outputs) >= 2, + ", ".join(openvpn_outputs), + ) + + real_world_outputs = dockerfiles_containing(test, "/rw_configure_script") + test.structural_check( + "Real-world router is absent from from-real-world example", + not real_world_outputs, + ", ".join(real_world_outputs), + ) + + test.write_summary("a03b-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A05_components/README.md b/examples/basic/A05_components/README.md index f5069ca5d..2aa1426fb 100644 --- a/examples/basic/A05_components/README.md +++ b/examples/basic/A05_components/README.md @@ -4,6 +4,24 @@ In Example `A01-transit-as`, we have saved the emulator in a file as a component. In this example, we demonstrate how we can load it into another emulation. +## Standard Arguments + +```sh +python examples/basic/A05_components/components.py amd +python examples/basic/A05_components/components.py --platform amd --output examples/basic/A05_components/output +python examples/basic/A05_components/components.py --dumpfile examples/basic/A05_components/components.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--component-file PATH`: location for the temporary component loaded from A01. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + ## Loading a Pre-Built Component @@ -64,4 +82,14 @@ and place it inside this IX, if the AS wants to peer with others in this IX. base.createInternetExchange(102) ``` +## TestRunner Lifecycle + +```sh +python seedemu/testing/cli.py all examples/basic/A05_components/example.yaml --artifact-dir ci-artifacts/a05 +``` + +The runtime test dynamically inspects the generated Compose labels instead of +hard-coding service names. It verifies that the added AS151 web host, AS154, and +IX102 are generated, and that the new web services can reach each other. + diff --git a/examples/basic/A05_components/components.py b/examples/basic/A05_components/components.py index 4d44f5124..a08c76464 100755 --- a/examples/basic/A05_components/components.py +++ b/examples/basic/A05_components/components.py @@ -1,99 +1,136 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu.core import Emulator, Binding, Filter from seedemu.layers import Base, Ebgp, PeerRelationship from seedemu.services import WebService from seedemu.compiler import Docker, Platform from examples.basic.A01_transit_as import transit_as -import sys, os -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A05 component-reuse example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--component-file", default=str(SCRIPT_DIR / "base_component.bin")) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + -############################################################################### -# Load the pre-built component from example 01-transit-as -transit_as.run(dumpfile='./base_component.bin') +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 -emu = Emulator() -emu.load('./base_component.bin') +def build_emulator(component_file: str | Path = SCRIPT_DIR / "base_component.bin") -> Emulator: + ############################################################################### + # Load the pre-built component from example 01-transit-as + component_path = Path(component_file).resolve() + component_path.parent.mkdir(parents=True, exist_ok=True) + transit_as.run(dumpfile=str(component_path)) -############################################################################### -# Demonstrating how to get the layers of the component. -# To make changes to any existing layer, we need to get the layer reference + emu = Emulator() + emu.load(str(component_path)) -base: Base = emu.getLayer('Base') -ebgp: Ebgp = emu.getLayer('Ebgp') + ############################################################################### + # Demonstrating how to get the layers of the component. + # To make changes to any existing layer, we need to get the layer reference -web: WebService = WebService() + base: Base = emu.getLayer("Base") + ebgp: Ebgp = emu.getLayer("Ebgp") + web: WebService = WebService() + ############################################################################### + # Add a new host to AS-151, which is from the pre-built component -############################################################################### -# Add a new host to AS-151, which is from the pre-built component + as151 = base.getAutonomousSystem(151) + as151.createHost("web-2").joinNetwork("net0") + web.install("web151-2") + emu.addBinding(Binding("web151-2", filter=Filter(nodeName="web-2", asn=151))) -as151 = base.getAutonomousSystem(151) -as151.createHost('web-2').joinNetwork('net0') -web.install('web151-2') -emu.addBinding(Binding('web151-2', filter = Filter(nodeName = 'web-2', asn = 151))) + ############################################################################### + # Add a new autonomous system (AS-154) + # This requires making changes to the base and ebgp layers. + as154 = base.createAutonomousSystem(154) + as154.createNetwork("net0") -############################################################################### -# Add a new autonomous system (AS-154) -# This requires making changes to the base and ebgp layers. + as154.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as154.createRouter("router1").joinNetwork("net0").joinNetwork("ix101") -as154 = base.createAutonomousSystem(154) -as154.createNetwork('net0') + as154.createHost("web").joinNetwork("net0") + web.install("web154") + emu.addBinding(Binding("web154", filter=Filter(nodeName="web", asn=154))) -as154.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') -as154.createRouter('router1').joinNetwork('net0').joinNetwork('ix101') + # Peer with AS-151 and AS-152 + ebgp.addPrivatePeering(100, 151, 154, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 152, 154, abRelationship=PeerRelationship.Peer) -as154.createHost('web').joinNetwork('net0') -web.install('web154') -emu.addBinding(Binding('web154', filter = Filter(nodeName = 'web', asn = 154))) + ############################################################################### + # Add a new internet exchange (IX-102) and peer AS-154 and AS-152 there. -# Peer with AS-151 and AS-152 -ebgp.addPrivatePeering(100, 151, 154, abRelationship = PeerRelationship.Provider) -ebgp.addPrivatePeering(101, 152, 154, abRelationship = PeerRelationship.Peer) + base.createInternetExchange(102) + as152 = base.getAutonomousSystem(152) + as152.createRouter("router1").joinNetwork("net0").joinNetwork("ix102") + as154.createRouter("router2").joinNetwork("net0").joinNetwork("ix102") -############################################################################### -# Add a new internet exchange (IX-102) and peer AS-153 and AS-152 there. + ebgp.addPrivatePeering(102, 152, 154, abRelationship=PeerRelationship.Peer) + emu.addLayer(web) + return emu -# Create an internet exchange -base.createInternetExchange(102) -# Add a BGP router to AS-152 and connect it to IX-102 -as152 = base.getAutonomousSystem(152) -as152.createRouter('router1').joinNetwork('net0').joinNetwork('ix102') +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, + component_file: str | Path = SCRIPT_DIR / "base_component.bin", +): + emu = build_emulator(component_file) + if dumpfile is not None: + emu.dump(dumpfile) + return -# Add a BGP router to AS-153 and connect it to IX-102 -as154.createRouter('router2').joinNetwork('net0').joinNetwork('ix102') + if render: + emu.render() -# Peer them -ebgp.addPrivatePeering(102, 152, 154, abRelationship = PeerRelationship.Peer) + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + component_file=args.component_file, + ) + return 0 -############################################################################### -# Render and compile -emu.render() -emu.compile(Docker(platform=platform), './output', override=True) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A05_components/example.yaml b/examples/basic/A05_components/example.yaml new file mode 100644 index 000000000..1b13f6973 --- /dev/null +++ b/examples/basic/A05_components/example.yaml @@ -0,0 +1,34 @@ +id: basic-a05-components +name: Using a Pre-built Component +description: Loads the A01 transit-AS component, extends AS151, adds AS154, and adds IX102 peering. +runner: internet +script: components.py +platform: amd +features: + - component-reuse + - ebgp-private-peering + - web-service + - internet-exchange + +compile: + enabled: true + output: output + clean: + - output + - base_component.bin + expected: + - output/docker-compose.yml + - base_component.bin + timeout: 600 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + +test_programs: + - name: A05 component extension runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A05_components/test_runtime.py b/examples/basic/A05_components/test_runtime.py new file mode 100644 index 000000000..abec1df06 --- /dev/null +++ b/examples/basic/A05_components/test_runtime.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + # A05 demonstrates modifying a loaded component: adding web-2 to AS151, + # adding AS154, and adding an IX102 peering point. + web151 = test.require_service(151, "web-2") + web154 = test.require_service(154, "web") + test.require_service(154, "router0") + test.require_service(154, "router1") + test.require_service(154, "router2") + test.require_service(102, "ix102") + + if web151 and web154: + test.exec_check("AS151 added web service is ready", web151, "curl -fsS http://127.0.0.1 >/dev/null") + test.exec_check("AS154 added web service is ready", web154, "curl -fsS http://127.0.0.1 >/dev/null") + test.exec_check( + "AS154 reaches added AS151 web service", + web154, + "curl -fsS http://{} >/dev/null".format(web151.address), + ) + test.exec_check( + "AS151 added web service reaches AS154", + web151, + "curl -fsS http://{} >/dev/null".format(web154.address), + ) + + test.write_summary("a05-components-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A06_merge_emulation/README.md b/examples/basic/A06_merge_emulation/README.md index 18885489f..061ca30e2 100644 --- a/examples/basic/A06_merge_emulation/README.md +++ b/examples/basic/A06_merge_emulation/README.md @@ -8,3 +8,29 @@ and then we merge them. ``` emu_merged = emuA.merge(emuB, DEFAULT_MERGERS) ``` + +## Standard Arguments + +```sh +python examples/basic/A06_merge_emulation/merge_emulation.py amd +python examples/basic/A06_merge_emulation/merge_emulation.py --platform amd --output examples/basic/A06_merge_emulation/output +python examples/basic/A06_merge_emulation/merge_emulation.py --dumpfile examples/basic/A06_merge_emulation/merge_emulation.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +```sh +python seedemu/testing/cli.py all examples/basic/A06_merge_emulation/example.yaml --artifact-dir ci-artifacts/a06 +``` + +The runtime test verifies that the merged AS150 and AS151 nodes are generated +and can reach each other through the merged IX100 topology. diff --git a/examples/basic/A06_merge_emulation/example.yaml b/examples/basic/A06_merge_emulation/example.yaml new file mode 100644 index 000000000..190e7ea44 --- /dev/null +++ b/examples/basic/A06_merge_emulation/example.yaml @@ -0,0 +1,31 @@ +id: basic-a06-merge-emulation +name: Merging Two Emulations +description: Builds two small emulations separately, merges them, and verifies the merged IX100 topology. +runner: internet +script: merge_emulation.py +platform: amd +features: + - emulation-merge + - ebgp-route-server + - ipv4-default + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + +test_programs: + - name: A06 merged emulation runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/basic/A06_merge_emulation/merge_emulation.py b/examples/basic/A06_merge_emulation/merge_emulation.py index ea2411903..59f39ad5d 100755 --- a/examples/basic/A06_merge_emulation/merge_emulation.py +++ b/examples/basic/A06_merge_emulation/merge_emulation.py @@ -1,89 +1,119 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path + from seedemu.core import Emulator from seedemu.mergers import DEFAULT_MERGERS from seedemu.layers import Base, Routing, Ebgp from seedemu.compiler import Docker, Platform -import os, sys - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -############################################################################### -# Create Emulation A -# We can also load this emulation for a pre-built component - -emuA = Emulator() -baseA = Base() -ebgpA = Ebgp() -routingA = Routing() - -# Create an internet exchange ix100 -baseA.createInternetExchange(100) - -# Create an autonomous system AS-150 -as150 = baseA.createAutonomousSystem(150) -as150.createNetwork('net0') -as150.createHost('host0').joinNetwork('net0') -as150.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - -# Peer with others at ix100 -ebgpA.addRsPeer(100, 150) - -# Add these layers to the emulation A -emuA.addLayer(baseA) -emuA.addLayer(routingA) -emuA.addLayer(ebgpA) - -############################################################################### -# Create Emulation B -# We can also load this emulation for a pre-built component - -baseB = Base() -ebgpB = Ebgp() -routingB = Routing() - -# Create an autonomous system AS-150 -as151 = baseB.createAutonomousSystem(151) -as151.createNetwork('net0') -as151.createHost('host0').joinNetwork('net0') -as151.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - -# Peer with others at ix100 -ebgpB.addRsPeer(100, 151) - -# Add these layers to the emulation B -emuB = Emulator() -emuB.addLayer(baseB) -emuB.addLayer(routingB) -emuB.addLayer(ebgpB) - - -############################################################################### -# Merge these two emulations - -emu_merged = emuA.merge(emuB, DEFAULT_MERGERS) - - -############################################################################### -# Generate the final emulation files - -emu_merged.render() -emu_merged.compile(Docker(platform=platform), './output', override=True) + + +SCRIPT_DIR = Path(__file__).resolve().parent + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A06 merge-emulation example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + ############################################################################### + # Create Emulation A + # We can also load this emulation for a pre-built component + + emu_a = Emulator() + base_a = Base() + ebgp_a = Ebgp() + routing_a = Routing() + + base_a.createInternetExchange(100) + + as150 = base_a.createAutonomousSystem(150) + as150.createNetwork("net0") + as150.createHost("host0").joinNetwork("net0") + as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp_a.addRsPeer(100, 150) + + emu_a.addLayer(base_a) + emu_a.addLayer(routing_a) + emu_a.addLayer(ebgp_a) + + ############################################################################### + # Create Emulation B + # We can also load this emulation for a pre-built component + + base_b = Base() + ebgp_b = Ebgp() + routing_b = Routing() + + as151 = base_b.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createHost("host0").joinNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp_b.addRsPeer(100, 151) + + emu_b = Emulator() + emu_b.addLayer(base_b) + emu_b.addLayer(routing_b) + emu_b.addLayer(ebgp_b) + + ############################################################################### + # Merge these two emulations + + return emu_a.merge(emu_b, DEFAULT_MERGERS) + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu_merged = build_emulator() + if dumpfile is not None: + emu_merged.dump(dumpfile) + return + + if render: + emu_merged.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu_merged.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A06_merge_emulation/test_runtime.py b/examples/basic/A06_merge_emulation/test_runtime.py new file mode 100644 index 000000000..b31bacf01 --- /dev/null +++ b/examples/basic/A06_merge_emulation/test_runtime.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + # A06 demonstrates merging two emulations that share IX100. + test.require_service(100, "ix100") + host150 = test.require_service(150, "host0") + test.require_service(150, "router0") + host151 = test.require_service(151, "host0") + test.require_service(151, "router0") + + if host150 and host151: + test.exec_check( + "AS150 host reaches merged AS151 host", + host150, + "ping -c 3 {}".format(host151.address), + ) + test.exec_check( + "AS151 host reaches merged AS150 host", + host151, + "ping -c 3 {}".format(host150.address), + ) + + test.write_summary("a06-merge-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A12_bgp_mixed_backend/README.md b/examples/basic/A12_bgp_mixed_backend/README.md new file mode 100644 index 000000000..da4e7f4c1 --- /dev/null +++ b/examples/basic/A12_bgp_mixed_backend/README.md @@ -0,0 +1,32 @@ +# A12 BGP Mixed Backend + +`A12` demonstrates mixed BIRD and FRRouting control planes in one IPv4 SEED topology. + +The default router backend remains BIRD. Only routers created with +`routingBackend="frr"` receive FRR packages, `/etc/frr/frr.conf`, and `/frr_start`. + +## What It Proves + +- `Router` owns the routing backend choice. +- `Ebgp`, `Ibgp`, and `Ospf` describe routing intent. +- `Routing` renders BIRD or FRR from the same intent model. +- BIRD and FRR routers can coexist without changing old default examples. + +## Topology + +- `AS2/r1` uses the default BIRD backend and peers at `ix100`. +- `AS2/r2` uses FRR and peers at `ix101`. +- `AS151/router0` uses FRR. +- `AS152/router0` uses the default BIRD backend. + +## Test Runner + +```bash +python3 -m seedemu.testing.cli clean examples/basic/A12_bgp_mixed_backend/example.yaml +python3 -m seedemu.testing.cli compile examples/basic/A12_bgp_mixed_backend/example.yaml +python3 -m seedemu.testing.cli build examples/basic/A12_bgp_mixed_backend/example.yaml +COMPOSE_PROJECT_NAME=seedemu-a12 python3 -m seedemu.testing.cli all examples/basic/A12_bgp_mixed_backend/example.yaml +``` + +The runtime test checks backend-specific daemons, generated config, BGP session +state, and learned route state on BIRD and FRR routers. diff --git a/examples/basic/A12_bgp_mixed_backend/bgp_mixed_backend.py b/examples/basic/A12_bgp_mixed_backend/bgp_mixed_backend.py new file mode 100644 index 000000000..c7b7e673d --- /dev/null +++ b/examples/basic/A12_bgp_mixed_backend/bgp_mixed_backend.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.services import WebService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a mixed BIRD/FRR BGP control-plane example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + ebgp = Ebgp() + web = WebService() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") + as2.createRouter("r2", routingBackend="frr").joinNetwork("net0").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix100") + as151.createHost("web").joinNetwork("net0") + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + as152.createHost("web").joinNetwork("net0") + + web.install("web151") + emu.addBinding(Binding("web151", filter=Filter(nodeName="web", asn=151))) + web.install("web152") + emu.addBinding(Binding("web152", filter=Filter(nodeName="web", asn=152))) + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(ebgp) + emu.addLayer(web) + return emu + + +def main() -> int: + args = parse_args() + emu = build_emulator() + + if args.dumpfile: + emu.dump(args.dumpfile) + print("Saved A12 emulator to {}".format(args.dumpfile)) + return 0 + + if args.render: + emu.render() + + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=resolve_platform(args.platform)), str(output_dir), override=args.override) + print("Generated A12 Docker output in {}".format(output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A12_bgp_mixed_backend/example.yaml b/examples/basic/A12_bgp_mixed_backend/example.yaml new file mode 100644 index 000000000..58fa44af2 --- /dev/null +++ b/examples/basic/A12_bgp_mixed_backend/example.yaml @@ -0,0 +1,80 @@ +id: basic-a12-bgp-mixed-backend +name: BGP Mixed BIRD and FRR Backends +description: A transit-AS topology that runs BIRD and FRR routers from the same BGP/OSPF intent model. +runner: internet +script: bgp_mixed_backend.py +platform: amd +features: + - ipv4-default + - routing-backend + - bird + - frr + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A12 mixed backend services are running + type: compose-ps + services: + - brdnode_2_r1 + - brdnode_2_r2 + - brdnode_151_router0 + - brdnode_152_router0 + - hnode_151_web + - hnode_152_web + retries: 30 + interval: 2 + +probes: + - name: AS2 BIRD router establishes iBGP to AS2 FRR router + type: exec + service: brdnode_2_r1 + command: birdc show protocols | grep -q 'ibgp1.*Established' + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS2 FRR router learns AS151 route from AS2 BIRD router + type: exec + service: brdnode_2_r2 + command: vtysh -c "show ip bgp" | grep -q '10.151.0.0/24' + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS151 FRR router learns AS152 route + type: exec + service: brdnode_151_router0 + command: vtysh -c "show ip bgp" | grep -q '10.152.0.0/24' + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS152 BIRD router learns AS151 route + type: exec + service: brdnode_152_router0 + command: birdc show route | grep -q '10.151.0.0/24' + expect_exit: 0 + retries: 30 + interval: 5 + +test_programs: + - name: A12 mixed backend runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/basic/A12_bgp_mixed_backend/test_runtime.py b/examples/basic/A12_bgp_mixed_backend/test_runtime.py new file mode 100644 index 000000000..ad96d0ab7 --- /dev/null +++ b/examples/basic/A12_bgp_mixed_backend/test_runtime.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest, ComposeService + + +ROUTING_BACKEND_LABEL = "org.seedsecuritylabs.seedemu.meta.seedemu_routing_backend" +ASN_LABEL = "org.seedsecuritylabs.seedemu.meta.asn" +NODE_LABEL = "org.seedsecuritylabs.seedemu.meta.nodename" + + +def require_backend(test: ComposeRuntimeTest, service: ComposeService, backend: str) -> None: + actual = service.labels.get(ROUTING_BACKEND_LABEL) + label = "AS{} {} uses {} routing backend".format( + service.labels.get(ASN_LABEL), + service.labels.get(NODE_LABEL), + backend, + ) + test.structural_check( + label, + actual == backend, + "expected {}, found {}".format(backend, actual), + ) + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + as2_r1 = test.require_service(2, "r1") + as2_r2 = test.require_service(2, "r2") + as151_router = test.require_service(151, "router0") + as152_router = test.require_service(152, "router0") + + if as2_r1: + require_backend(test, as2_r1, "bird") + test.exec_check("AS2 r1 starts BIRD", as2_r1, "pgrep -x bird >/dev/null") + test.exec_check("AS2 r1 has BIRD BGP intent", as2_r1, "grep -q 'neighbor 10.100.0.151 as 151' /etc/bird/bird.conf") + test.exec_check("AS2 r1 iBGP session is established", as2_r1, "birdc show protocols | grep -q 'ibgp1.*Established'") + test.exec_check("AS2 r1 learns AS152 route", as2_r1, "birdc show route | grep -q '10.152.0.0/24'") + test.exec_check("AS2 r1 does not carry FRR config", as2_r1, "test ! -e /etc/frr/frr.conf") + + if as2_r2: + require_backend(test, as2_r2, "frr") + test.exec_check("AS2 r2 starts FRR bgpd", as2_r2, "pgrep -x bgpd >/dev/null") + test.exec_check("AS2 r2 renders FRR BGP", as2_r2, "grep -q 'router bgp 2' /etc/frr/frr.conf") + test.exec_check("AS2 r2 renders FRR OSPF", as2_r2, "grep -q 'router ospf' /etc/frr/frr.conf") + test.exec_check( + "AS2 r2 learns AS151 route through iBGP from r1", + as2_r2, + "vtysh -c 'show ip bgp 10.151.0.0/24' | grep -q '10.0.0.1'", + ) + test.exec_check("AS2 r2 learns AS151 route", as2_r2, "vtysh -c 'show ip bgp' | grep -q '10.151.0.0/24'") + test.exec_check("AS2 r2 does not start BIRD", as2_r2, "! pgrep -x bird >/dev/null") + + if as151_router: + require_backend(test, as151_router, "frr") + test.exec_check("AS151 router starts FRR bgpd", as151_router, "pgrep -x bgpd >/dev/null") + test.exec_check( + "AS151 FRR router has AS2 peer", + as151_router, + "grep -q 'neighbor 10.100.0.2 remote-as 2' /etc/frr/frr.conf", + ) + test.exec_check( + "AS151 FRR router learns AS152 route", + as151_router, + "vtysh -c 'show ip bgp' | grep -q '10.152.0.0/24'", + ) + + if as152_router: + require_backend(test, as152_router, "bird") + test.exec_check("AS152 router starts BIRD", as152_router, "pgrep -x bird >/dev/null") + test.exec_check("AS152 BIRD router has AS2 peer", as152_router, "grep -q 'neighbor 10.101.0.2 as 2' /etc/bird/bird.conf") + test.exec_check("AS152 BIRD router learns AS151 route", as152_router, "birdc show route | grep -q '10.151.0.0/24'") + + test.write_summary("a12-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A13_exabgp/README.md b/examples/basic/A13_exabgp/README.md new file mode 100644 index 000000000..de42f5660 --- /dev/null +++ b/examples/basic/A13_exabgp/README.md @@ -0,0 +1,87 @@ +# A13 ExaBGP + +`A13` installs ExaBGP as a SEED service speaker through `ExaBgpService` and +`Binding`. + +ExaBGP is not a `Routing` backend. The peer router keeps its normal BIRD or FRR +daemon, and the service records a router-facing BGP peer intent that `Routing` +renders for that router. + +## What It Proves + +- ExaBGP is installed on a bound host node. +- The ExaBGP node joins the IX peering LAN directly. +- `Routing` renders the peer session on the real router. +- Static IPv4 announcements are emitted from `/etc/exabgp/exabgp.conf`. + +## Topology + +- `AS2/router0` is a normal BIRD router on `ix100`. +- `AS180/exabgp` is a host on `ix100` at `10.100.0.180`. +- `AS180/exabgp` announces `198.51.100.0/24` to `AS2/router0`. + +The ExaBGP peer is declared by IX and AS number: + +```python +exabgp_speaker = exabgp.install("as180_exabgp").setLocalAsn(180) +# Prefer IX-based peering: the service resolves the AS2 router on IX100 automatically. +exabgp_speaker.addPeer(ix=100, peer_asn=2, router_relationship="customer") +# Use router-based peering when multiple AS2 routers share IX100 and one must be selected explicitly. +# exabgp_speaker.addPeerByRouter("router0", router_asn=2, router_relationship="customer") +exabgp_speaker.addAnnouncement("198.51.100.0/24") +``` + +The active `addPeer()` call is the preferred form for most examples. It says +that AS180 should peer with AS2 at IX100, and the service finds the AS2 router +attached to that IX. The commented `addPeerByRouter()` form is useful when the +target AS has more than one router on the same IX and the example needs to +choose a specific one. + +## Manual BGP Updates + +The ExaBGP service also creates a manual control FIFO inside the ExaBGP +container: + +```text +/run/exabgp/manual.in +``` + +Use `exabgpctl.sh` from this directory to send live BGP updates after the +emulator is running: + +```bash +sh ./exabgpctl.sh announce 203.0.113.0/24 self +sh ./exabgpctl.sh withdraw 203.0.113.0/24 self +``` + +You can also send a raw ExaBGP command: + +```bash +sh ./exabgpctl.sh command "announce route 203.0.113.0/24 next-hop self" +``` + +To inspect ExaBGP logs: + +```bash +sh ./exabgpctl.sh log +``` + +The script defaults to `output/docker-compose.yml` and service +`hnode_180_exabgp`. Override them if needed: + +```bash +COMPOSE_FILE=/path/to/docker-compose.yml EXABGP_SERVICE=hnode_180_exabgp sh ./exabgpctl.sh announce 203.0.113.0/24 self +``` + +## Test Runner + +```bash +python3 -m seedemu.testing.cli clean examples/basic/A13_exabgp/example.yaml +python3 -m seedemu.testing.cli compile examples/basic/A13_exabgp/example.yaml +python3 -m seedemu.testing.cli build examples/basic/A13_exabgp/example.yaml +COMPOSE_PROJECT_NAME=seedemu-a13 python3 -m seedemu.testing.cli all examples/basic/A13_exabgp/example.yaml +``` + +The runtime test checks `/etc/exabgp/exabgp.conf`, the ExaBGP process, IX100 +reachability, the manual control FIFO, and the generated BGP peer on +`AS2/router0`. diff --git a/examples/basic/A13_exabgp/exabgp.py b/examples/basic/A13_exabgp/exabgp.py new file mode 100755 index 000000000..0b9dc381e --- /dev/null +++ b/examples/basic/A13_exabgp/exabgp.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Routing +from seedemu.services import ExaBgpService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build an ExaBGP service speaker example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as180 = base.createAutonomousSystem(180) + as180.createHost("exabgp").joinNetwork("ix100", address="10.100.0.180") + + exabgp_speaker = exabgp.install("as180_exabgp").setLocalAsn(180) + # Prefer IX-based peering: the service resolves the AS2 router on IX100 automatically. + exabgp_speaker.addPeer(ix=100, peer_asn=2, router_relationship="customer") + # Use router-based peering when multiple AS2 routers share IX100 and one must be selected explicitly. + # exabgp_speaker.addPeerByRouter("router0", router_asn=2, router_relationship="customer") + exabgp_speaker.addAnnouncement("198.51.100.0/24") + emu.addBinding(Binding("as180_exabgp", filter=Filter(asn=180, nodeName="exabgp"))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(exabgp) + return emu + + +def main() -> int: + args = parse_args() + emu = build_emulator() + + if args.dumpfile: + emu.dump(args.dumpfile) + print("Saved A13 emulator to {}".format(args.dumpfile)) + return 0 + + if args.render: + emu.render() + + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=resolve_platform(args.platform)), str(output_dir), override=args.override) + print("Generated A13 Docker output in {}".format(output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A13_exabgp/exabgpctl.sh b/examples/basic/A13_exabgp/exabgpctl.sh new file mode 100755 index 000000000..104b36ebe --- /dev/null +++ b/examples/basic/A13_exabgp/exabgpctl.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +COMPOSE_FILE=${COMPOSE_FILE:-"$SCRIPT_DIR/output/docker-compose.yml"} +SERVICE=${EXABGP_SERVICE:-hnode_180_exabgp} + +usage() { + cat <<'EOF' +Usage: + exabgpctl.sh announce PREFIX [NEXT_HOP] + exabgpctl.sh withdraw PREFIX [NEXT_HOP] + exabgpctl.sh command "RAW EXABGP COMMAND" + exabgpctl.sh log + +Examples: + ./exabgpctl.sh announce 203.0.113.0/24 self + ./exabgpctl.sh withdraw 203.0.113.0/24 self + ./exabgpctl.sh command "announce route 203.0.113.0/24 next-hop self" + +Environment: + COMPOSE_FILE Path to docker-compose.yml. Defaults to output/docker-compose.yml. + EXABGP_SERVICE Compose service name. Defaults to hnode_180_exabgp. +EOF +} + +require_compose() { + if [ ! -f "$COMPOSE_FILE" ]; then + echo "compose file not found: $COMPOSE_FILE" >&2 + exit 1 + fi +} + +send_command() { + command_text=$1 + require_compose + printf 'Sending to %s: %s\n' "$SERVICE" "$command_text" + docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE" sh -lc \ + 'test -p /run/exabgp/manual.in && printf "%s\n" "$1" > /run/exabgp/manual.in' \ + sh "$command_text" +} + +case "${1:-}" in + announce|withdraw) + action=$1 + prefix=${2:-} + next_hop=${3:-self} + if [ -z "$prefix" ]; then + usage + exit 1 + fi + send_command "$action route $prefix next-hop $next_hop" + ;; + command) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + send_command "$2" + ;; + log) + require_compose + docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE" sh -lc \ + 'tail -n 80 /var/log/exabgp/exabgp.log' + ;; + ""|-h|--help|help) + usage + ;; + *) + usage + exit 1 + ;; +esac diff --git a/examples/basic/A13_exabgp/example.yaml b/examples/basic/A13_exabgp/example.yaml new file mode 100644 index 000000000..f2c5bbd7d --- /dev/null +++ b/examples/basic/A13_exabgp/example.yaml @@ -0,0 +1,50 @@ +id: basic-a13-exabgp +name: ExaBGP Service +description: A router-facing ExaBGP service speaker installed with Service and Binding APIs. +runner: internet +script: exabgp.py +platform: amd +features: + - ipv4-default + - exabgp-service + - binding + - ebgp-private-peering + - bird + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A13 ExaBGP topology is running + type: compose-ps + services: + - brdnode_2_router0 + - hnode_180_exabgp + retries: 30 + interval: 2 + +probes: + - name: ExaBGP speaker reaches AS2 router on IX100 + type: ping + service: hnode_180_exabgp + target: 10.100.0.2 + count: 3 + retries: 20 + interval: 3 + +test_programs: + - name: A13 ExaBGP runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/basic/A13_exabgp/test_runtime.py b/examples/basic/A13_exabgp/test_runtime.py new file mode 100644 index 000000000..83bd9d9da --- /dev/null +++ b/examples/basic/A13_exabgp/test_runtime.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + router = test.require_service(2, "router0") + speaker = test.require_service(180, "exabgp") + + if router: + test.exec_check("AS2 router starts BIRD", router, "pgrep -x bird >/dev/null") + test.exec_check( + "AS2 router peers with ExaBGP speaker", + router, + "grep -q 'neighbor 10.100.0.180 as 180' /etc/bird/bird.conf", + ) + test.exec_check( + "AS2 router keeps ExaBGP out of routing backend", + router, + "test ! -e /etc/frr/frr.conf", + ) + + if speaker: + test.exec_check("ExaBGP speaker process is running", speaker, "pgrep -f 'exabgp /etc/exabgp/exabgp.conf' >/dev/null") + test.exec_check("ExaBGP manual control FIFO is available", speaker, "test -p /run/exabgp/manual.in") + test.exec_check( + "ExaBGP config peers with AS2 router", + speaker, + "grep -q 'neighbor 10.100.0.2' /etc/exabgp/exabgp.conf", + ) + test.exec_check( + "ExaBGP config enables manual-control process", + speaker, + "grep -q 'processes \\[ manual-control \\]' /etc/exabgp/exabgp.conf", + ) + test.exec_check( + "ExaBGP config announces static IPv4 route", + speaker, + "grep -q 'route 198.51.100.0/24 next-hop self' /etc/exabgp/exabgp.conf", + ) + test.exec_check("ExaBGP speaker reaches AS2 router", speaker, "ping -c 3 10.100.0.2 >/dev/null") + + test.write_summary("a13-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A15_toplogy_generator/README.md b/examples/basic/A15_toplogy_generator/README.md new file mode 100644 index 000000000..ca210e012 --- /dev/null +++ b/examples/basic/A15_toplogy_generator/README.md @@ -0,0 +1,241 @@ +# A15 Autonomous System Topology Generator + +This example demonstrates `AutonomousSystemTopologyGenerator`, a NetworkX-based +helper for generating the internal topology of an autonomous system. + +The example by default creates: + +```text +AS2: generated transit AS +IX100, IX101, IX102: external peering LANs +AS150, AS151, AS152: stub ASes connected through AS2 +``` + +The generated AS2 topology has eBGP routers such as `r0`, `r1`, and `r2`, plus +internal routers such as `core0` and `core1`. The example code manually +connects those eBGP routers to IXes. +Internal link network names are kept compact, such as `n_c10_c5`, because Linux +interface names inside containers must be no longer than 15 characters. + +## Basic Run + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py --seed 42 +``` + +The script writes two topology artifacts into the output folder: + +```text +output/topology.json +output/topology.txt +``` + +These files record the generated routers, links, eBGP-to-internal attachment +mapping, seed, graph model, graph parameters, internal routing mode, and router +locations. + +## Command-Line Controls + +Important options: + +- `--seed N`: make the random topology reproducible. +- `--asn N`: transit AS number, default `2`. +- `--ebgp-routers N`: number of eBGP routers generated for the transit AS. +- `--ixes 100,101,102`: IXes where this example manually connects the generated eBGP routers. +- `--stub-asns 150,151,152`: stub ASes connected to those IXes (by default, one stuf asn for each ix). +- `--internal-routers N`: number of generated internal routers. +- `--graph-model MODEL`: NetworkX model or alias. +- `--graph-param KEY=VALUE`: parameter passed to the graph model. Can be repeated. +- `--ebgp-attach-policy spread|round_robin|random|degree`: how eBGP routers attach to internal routers. +- `--internal-routing full-mesh|rr`: iBGP design inside the generated AS (full mesh or route reflector). +- `--route-reflector ROUTER`: route reflector router name when using `--internal-routing rr`. +- `--router-location ROUTER=LAT,LON`: known router location used to anchor generated positions. Can be repeated. +- `--no-locations`: skip location generation. + +Example using a scale-free-style core: + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --seed 7 \ + --internal-routers 6 \ + --graph-model barabasi_albert \ + --graph-param m=2 \ + --ebgp-attach-policy degree +``` + +Example using an Erdos-Renyi graph: + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --seed 12 \ + --internal-routers 6 \ + --graph-model erdos_renyi \ + --graph-param p=0.4 \ + --ebgp-attach-policy random +``` + +Example with a larger AS: + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --ixes 100,101,102,103,104 \ + --stub-asns 150,151,152,153,154 \ + --ebgp-routers 5 \ + --asn 3 \ + --internal-routers 20 \ + --hosts-per-stub 2 +``` + +Without extra options, this example keeps the default SEED `Ibgp()` behavior, +which creates full-mesh iBGP among routers in the generated transit AS. + +## Internal Routing Modes + +A15 can use the same generated physical topology with two different iBGP +control-plane designs. + +### Full-Mesh iBGP + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --internal-routing full-mesh +``` + +This is the default mode. Every router in the generated AS participates in the +SEED `Ibgp()` full mesh. It is easy to understand and useful for small +topologies, but the number of iBGP sessions grows quickly as the AS gets larger. + +### Route Reflector + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --internal-routing rr +``` + +In this mode, the example creates one BGP cluster. One generated internal router +is selected as the route reflector, and all routers in the generated AS join the +cluster as route-reflector clients. By default, the selected reflector is a +high-degree internal router, because that is usually a natural place to +centralize control-plane sessions in a generated topology. + +You can choose the reflector explicitly: + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --internal-routing rr \ + --route-reflector core0 +``` + +The selected mode is recorded in `output/topology.json` and +`output/topology.txt`, so CI logs and manual experiments can be compared later. + +## Location Generation + +A15 keeps topology generation and location generation separate. The topology +generator first creates routers and links. After that, the example uses +`AutonomousSystemLocationGenerator` to assign positions to all routers. + +Known router locations are fixed. Missing router locations are generated with +NetworkX `spring_layout`, using the fixed routers as anchors. This means the +known eBGP locations stay where the user put them, while internal routers are +placed in positions that are consistent with the graph structure. + +By default, this example anchors `r0`, `r1`, and `r2` to sample city locations: + +```text +r0: New York +r1: Chicago +r2: Los Angeles +``` + +You can override or add anchors: + +```sh +python examples/basic/A15_toplogy_generator/topology_generator.py \ + --router-location r0=40.7128,-74.0060 \ + --router-location r1=41.8781,-87.6298 \ + --router-location r2=34.0522,-118.2437 +``` + +The generated locations are written to `output/topology.json` under +`locations`. They are metadata only; they do not change links, routing, BGP +sessions, OSPF, or Docker network generation. + +The example also stores the coordinates on each generated router node with +`setGeo()` and `setLabel()`. The Docker compiler writes those labels into +`output/docker-compose.yml`: + +```yaml +org.seedsecuritylabs.seedemu.meta.geo.lat: "40.712800" +org.seedsecuritylabs.seedemu.meta.geo.lon: "-74.006000" +org.seedsecuritylabs.seedemu.meta.geo.source: "fixed" +org.seedsecuritylabs.seedemu.meta.geo.label: "New York" +``` + +## Code Pattern + +The core pattern is the following. The generator first creates a reusable topology object. The topology can be validated, exported, and inspected. + +```python +topology = AutonomousSystemTopologyGenerator( + ebgp_router_count=args.ebgp_routers, + internal_router_count=args.internal_routers, + graph_model=args.graph_model, + graph_params=args.graph_params, + ebgp_attach_policy=args.ebgp_attach_policy, + seed=args.seed, +).generate() + +transit_as = base.createAutonomousSystem(args.asn) +routers = {name: transit_as.createRouter(name) for name in topology.routers()} + +for ebgp_router, ix in zip(topology.ebgp_routers(), args.ixes): + routers[ebgp_router].joinNetwork(f"ix{ix}") + +for left, right, network in topology.link_networks(): + transit_as.createNetwork(network) + routers[left].joinNetwork(network) + routers[right].joinNetwork(network) + +if args.internal_routing == "rr": + cluster_id = "10.2.0.1" + transit_as.createBgpCluster(cluster_id) + for router_name in topology.routers(): + router = transit_as.getRouter(router_name).joinBgpCluster(cluster_id) + if router_name == "core0": + router.makeRouteReflector() + +locations = AutonomousSystemLocationGenerator( + fixed_locations={ + "r0": {"lat": 40.7128, "lon": -74.0060}, + "r1": {"lat": 41.8781, "lon": -87.6298}, + "r2": {"lat": 34.0522, "lon": -118.2437}, + }, + seed=args.seed, +).generate(topology) + +for router_name, location in locations.items(): + router = transit_as.getRouter(router_name) + router.setGeo(location["lat"], location["lon"]) + router.setLabel("geo.lat", str(location["lat"])) + router.setLabel("geo.lon", str(location["lon"])) + router.setLabel("geo.source", location["source"]) +``` + + + + +## TestRunner Lifecycle + +Run from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/basic/A15_toplogy_generator/example.yaml +python seedemu/testing/cli.py compile examples/basic/A15_toplogy_generator/example.yaml +python seedemu/testing/cli.py build examples/basic/A15_toplogy_generator/example.yaml +python seedemu/testing/cli.py all examples/basic/A15_toplogy_generator/example.yaml +``` + +The runtime test checks that generated AS2 routers start BIRD, that the edge +routers receive generated eBGP peerings, and that traffic can cross the +generated AS2 transit topology from AS150 to AS152. diff --git a/examples/basic/A15_toplogy_generator/example.yaml b/examples/basic/A15_toplogy_generator/example.yaml new file mode 100644 index 000000000..b0fb0d5da --- /dev/null +++ b/examples/basic/A15_toplogy_generator/example.yaml @@ -0,0 +1,60 @@ +id: basic-a15-topology-generator +name: Autonomous System Topology Generator +description: Demonstrates NetworkX-based random autonomous-system topology generation with CLI controls. +runner: internet +script: topology_generator.py +platform: amd +features: + - ipv4-default + - topology-generator + - networkx + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + - output/topology.json + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A15 generated topology services are running + type: compose-ps + services: + - brdnode_2_r0 + - brdnode_2_r1 + - brdnode_2_r2 + - rnode_2_core0 + - rnode_2_core1 + - brdnode_150_router0 + - brdnode_151_router0 + - brdnode_152_router0 + - hnode_150_host_0 + - hnode_152_host_0 + retries: 30 + interval: 3 + +probes: + - name: AS150 reaches AS152 through generated AS2 transit topology + type: ping + service: hnode_150_host_0 + target: 10.152.0.71 + count: 3 + retries: 30 + interval: 5 + +test_programs: + - name: A15 generated topology runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/basic/A15_toplogy_generator/run.sh b/examples/basic/A15_toplogy_generator/run.sh new file mode 100755 index 000000000..a2e429f5a --- /dev/null +++ b/examples/basic/A15_toplogy_generator/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +python ./topology_generator.py \ + --ixes 100,101,102,103,104 \ + --stub-asns 150,151,152,153,154 \ + --ebgp-routers 5 \ + --asn 3 \ + --internal-routers 20 \ + --hosts-per-stub 2 \ + --graph-model connected_watts_strogatz \ + --internal-routing rr diff --git a/examples/basic/A15_toplogy_generator/test_runtime.py b/examples/basic/A15_toplogy_generator/test_runtime.py new file mode 100644 index 000000000..55789003e --- /dev/null +++ b/examples/basic/A15_toplogy_generator/test_runtime.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + r0 = test.require_service(2, "r0") + r1 = test.require_service(2, "r1") + core0 = test.require_service(2, "core0") + core1 = test.require_service(2, "core1") + host150 = test.require_service(150, "host_0") + host152 = test.require_service(152, "host_0") + + for name, router in [("r0", r0), ("r1", r1), ("core0", core0), ("core1", core1)]: + if router: + test.exec_check("AS2 {} starts BIRD".format(name), router, "pgrep -x bird >/dev/null") + + if r0: + test.exec_check( + "AS2 r0 peers with AS150 on IX100", + r0, + "grep -q 'neighbor 10.100.0.150 as 150' /etc/bird/bird.conf", + ) + + if r1: + test.exec_check( + "AS2 r1 peers with AS151 on IX101", + r1, + "grep -q 'neighbor 10.101.0.151 as 151' /etc/bird/bird.conf", + ) + + if host150 and host152: + test.exec_check( + "AS150 reaches AS152 through generated AS2 topology", + host150, + "ping -c 3 {} >/dev/null".format(host152.address), + ) + + test.write_summary("a15-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A15_toplogy_generator/topology_generator.py b/examples/basic/A15_toplogy_generator/topology_generator.py new file mode 100755 index 000000000..8b6d5fff4 --- /dev/null +++ b/examples/basic/A15_toplogy_generator/topology_generator.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Any, Dict, List + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Emulator +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.utilities import Makers, AutonomousSystemLocationGenerator, AutonomousSystemTopologyGenerator + + +DEFAULT_IXES = [100, 101, 102] +DEFAULT_STUB_ASNS = [150, 151, 152] + +# These locations are only anchors for visualization and metadata. They do not +# affect the physical links or routing configuration generated by SEED. +DEFAULT_ROUTER_LOCATIONS = { + "r0": {"lat": 40.7128, "lon": -74.0060, "label": "New York"}, + "r1": {"lat": 41.8781, "lon": -87.6298, "label": "Chicago"}, + "r2": {"lat": 34.0522, "lon": -118.2437, "label": "Los Angeles"}, + "r3": {"lat": 39.9042, "lon": 116.4074, "label": "Beijing"}, + "r4": {"lat": 51.5072, "lon": 0.1276, "label": "London"}, +} + + +def parse_graph_param(value: str) -> tuple[str, Any]: + """Parse --graph-param KEY=VALUE and preserve basic numeric/bool types.""" + if "=" not in value: + raise argparse.ArgumentTypeError("graph parameters must use KEY=VALUE") + + key, raw_value = value.split("=", 1) + key = key.strip() + raw_value = raw_value.strip() + if not key: + raise argparse.ArgumentTypeError("graph parameter key cannot be empty") + + if raw_value.lower() in {"true", "false"}: + return key, raw_value.lower() == "true" + + try: + return key, int(raw_value) + except ValueError: + pass + + try: + return key, float(raw_value) + except ValueError: + return key, raw_value + + +def parse_csv_ints(value: str) -> List[int]: + items = [item.strip() for item in value.split(",") if item.strip()] + if not items: + raise argparse.ArgumentTypeError("value must contain at least one integer") + return [int(item) for item in items] + + +def parse_router_location(value: str): + """Parse --router-location ROUTER=LAT,LON into the location format used below.""" + if "=" not in value: + raise argparse.ArgumentTypeError("router locations must use ROUTER=LAT,LON") + + router, raw_location = value.split("=", 1) + router = router.strip() + coordinates = [item.strip() for item in raw_location.split(",") if item.strip()] + if not router or len(coordinates) != 2: + raise argparse.ArgumentTypeError("router locations must use ROUTER=LAT,LON") + + try: + return router, {"lat": float(coordinates[0]), "lon": float(coordinates[1])} + except ValueError as exc: + raise argparse.ArgumentTypeError("router location coordinates must be numbers") from exc + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a transit AS with a generated internal topology.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--asn", type=int, default=2) + parser.add_argument("--ixes", type=parse_csv_ints, default=DEFAULT_IXES) + parser.add_argument("--stub-asns", type=parse_csv_ints, default=DEFAULT_STUB_ASNS) + parser.add_argument("--ebgp-routers", type=int) + parser.add_argument("--hosts-per-stub", type=int, default=1) + parser.add_argument("--internal-routers", type=int, default=4) + parser.add_argument("--graph-model", default="small_world") + parser.add_argument("--graph-param", action="append", type=parse_graph_param, default=[]) + parser.add_argument( + "--ebgp-attach-policy", + choices=["spread", "round_robin", "random", "degree"], + default="spread", + ) + parser.add_argument( + "--internal-routing", + choices=["full-mesh", "rr", "route-reflector"], + default="full-mesh", + help="iBGP design inside the generated AS.", + ) + parser.add_argument( + "--route-reflector", + help="Router name to use as the route reflector in rr mode. Defaults to a high-degree internal router.", + ) + parser.add_argument( + "--router-location", + action="append", + type=parse_router_location, + default=[], + metavar="ROUTER=LAT,LON", + help="Known router location used to anchor generated positions. Can be repeated.", + ) + parser.add_argument( + "--no-locations", + dest="locations", + action="store_false", + default=True, + help="Do not generate router location artifacts.", + ) + + args = parser.parse_args() + + # Keep compatibility with older examples that accepted the platform as the + # first positional argument, while allowing the newer --platform option. + args.platform = args.platform or args.legacy_platform or "amd" + + # Convert repeated parser options into dictionaries that are easier to pass + # into the topology and location generators. + args.graph_params = dict(args.graph_param) + args.router_locations = dict(args.router_location) + + # A15 maps each generated eBGP router to one IX, so when the user does not + # provide --ebgp-routers, the IX count defines the eBGP-router count. + args.ebgp_routers = args.ebgp_routers or len(args.ixes) + if args.internal_routing == "route-reflector": + args.internal_routing = "rr" + if len(args.ixes) != len(args.stub_asns): + parser.error("--ixes and --stub-asns must contain the same number of entries") + if args.ebgp_routers != len(args.ixes): + parser.error("this example maps one eBGP router to one IX, so --ebgp-routers must match --ixes") + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator(args: argparse.Namespace): + emu = Emulator() + base = Base() + ebgp = Ebgp() + + # IXes are created first because the generated border routers and stub ASes + # join these peering LANs later. + for ix in args.ixes: + base.createInternetExchange(ix) + + # Generate the abstract AS topology first. This object knows routers, links, + # and eBGP-to-internal attachments, but it is still independent from SEED's + # Base layer until apply_topology() below. + topology = AutonomousSystemTopologyGenerator( + ebgp_router_count=args.ebgp_routers, + internal_router_count=args.internal_routers, + graph_model=args.graph_model, + graph_params=args.graph_params, + ebgp_attach_policy=args.ebgp_attach_policy, + seed=args.seed, + ebgp_router_prefix="r", + ).generate() + + # Materialize the generated topology into the SEED Base layer. After this + # step, routers and networks exist in the emulator. + transit_as = apply_topology(base, topology, args.asn, args.ixes) + + # Configure the iBGP control plane independently from the physical topology. + # Full mesh needs no extra metadata; route-reflector mode marks one router + # as the reflector and puts all AS routers into the same BGP cluster. + internal_routing = configure_internal_routing( + transit_as, + topology, + mode=args.internal_routing, + route_reflector=args.route_reflector, + ) + + # Locations are generated after the topology is known. They are then written + # both to topology artifacts and to Docker Compose labels via setLabel(). + locations = generate_locations(topology, args) + if locations is not None: + apply_location_metadata(transit_as, locations) + + # Stub ASes are created after the transit AS. Each stub joins the matching IX + # and peers privately with the generated transit AS. + for stub_asn, ix in zip(args.stub_asns, args.ixes): + Makers.makeStubAsWithHosts(emu, base, stub_asn, ix, args.hosts_per_stub) + ebgp.addPrivatePeering(ix, args.asn, stub_asn, PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + return emu, topology, internal_routing, locations + + +def apply_topology(base: Base, topology, asn: int, ixes: List[int]): + transit_as = base.createAutonomousSystem(asn) + + # Create every router before joining networks, so link creation can refer to + # both endpoints regardless of edge ordering in the generated graph. + routers = {name: transit_as.createRouter(name) for name in topology.routers()} + + # The topology generator only says which routers are eBGP routers. This + # example decides how to map those routers onto concrete IX numbers. + for ebgp_router, ix in zip(topology.ebgp_routers(), ixes): + routers[ebgp_router].joinNetwork("ix{}".format(ix)) + + # Each graph edge becomes one point-to-point SEED network. The generator + # provides compact network names that fit Linux's interface-name limit. + for left, right, network in topology.link_networks(): + transit_as.createNetwork(network) + routers[left].joinNetwork(network) + routers[right].joinNetwork(network) + + return transit_as + + +def configure_internal_routing(transit_as, topology, mode: str, route_reflector: str = None) -> Dict[str, Any]: + mode = mode.lower() + if mode == "full-mesh": + # SEED's Ibgp layer creates a full mesh by default when no BGP clusters + # are configured, so this mode only records metadata for artifacts. + return { + "mode": "full-mesh", + "route_reflector": None, + "description": "default SEED Ibgp() full mesh among AS routers", + } + + if mode != "rr": + raise ValueError("unsupported internal routing mode: {}".format(mode)) + + # Use the provided router when the user wants a specific reflector; otherwise + # choose a central internal router based on graph degree. + rr_name = route_reflector or choose_route_reflector(topology) + if rr_name not in set(topology.routers()): + raise ValueError("unknown route reflector router: {}".format(rr_name)) + + # All routers join one cluster. The selected router is marked as the route + # reflector, and the rest become route-reflector clients. + cluster_id = route_reflector_cluster_id(transit_as.getAsn()) + transit_as.createBgpCluster(cluster_id) + for router_name in topology.routers(): + router = transit_as.getRouter(router_name).joinBgpCluster(cluster_id) + if router_name == rr_name: + router.makeRouteReflector() + + return { + "mode": "rr", + "route_reflector": rr_name, + "cluster_id": cluster_id, + "description": "one generated router is the route reflector; all other AS routers are clients", + } + + +def choose_route_reflector(topology) -> str: + # Prefer internal routers as RR candidates. The highest-degree router is a + # reasonable default because it is central in the generated graph. + graph = topology.graph() + candidates = topology.internal_routers() or topology.routers() + return sorted(candidates, key=lambda router: (-graph.degree(router), router))[0] + + +def route_reflector_cluster_id(asn: int) -> str: + # BIRD expects a route-reflector cluster ID in IPv4-address form. Deriving it + # from ASN keeps it deterministic and avoids another command-line option. + return "10.{}.{}.1".format((int(asn) // 256) % 256, int(asn) % 256) + + +def generate_locations(topology, args: argparse.Namespace): + if not args.locations: + return None + + # Start with demonstration anchors, then let command-line anchors override + # them. This keeps the default example useful while making experiments easy. + fixed_locations = default_router_locations(topology) + fixed_locations.update(args.router_locations) + locations = AutonomousSystemLocationGenerator( + fixed_locations=fixed_locations, + seed=args.seed, + ).generate(topology) + + # Preserve human-readable labels such as city names for fixed anchors. The + # spring-layout generator only needs coordinates, but compose labels can use + # this extra metadata. + for router, fixed_location in fixed_locations.items(): + if isinstance(fixed_location, dict): + label = fixed_location.get("label") or fixed_location.get("address") + if label and router in locations: + locations[router]["label"] = str(label) + + return locations + + +def default_router_locations(topology) -> Dict[str, Dict[str, Any]]: + routers = set(topology.routers()) + return { + router: location + for router, location in DEFAULT_ROUTER_LOCATIONS.items() + if router in routers + } + + +def apply_location_metadata(transit_as, locations: Dict[str, Any]) -> None: + # setGeo() keeps location metadata on the SEED node. setLabel() makes the + # same metadata visible in docker-compose.yml for agents, tools, and tests. + for router_name, location in locations.items(): + router = transit_as.getRouter(router_name) + lat = float(location["lat"]) + lon = float(location["lon"]) + label = str(location.get("label", "")) + + router.setGeo(lat, lon, label) + router.setLabel("geo.lat", "{:.6f}".format(lat)) + router.setLabel("geo.lon", "{:.6f}".format(lon)) + router.setLabel("geo.source", str(location.get("source", "generated"))) + if label: + router.setLabel("geo.label", label) + + +def write_topology_artifacts( + topology, + output_dir: Path, + internal_routing: Dict[str, Any], + locations: Dict[str, Any] = None, +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + + # topology.json is intended for tools and agents. It includes the generated + # graph, routing mode, and optional location metadata in structured form. + data = topology.to_dict() + data["internal_routing"] = internal_routing + if locations is not None: + data["locations"] = locations + with open(output_dir / "topology.json", "w", encoding="utf-8") as file: + json.dump(data, file, indent=2, sort_keys=True) + + # topology.txt is a short human-readable summary for logs and quick checks. + with open(output_dir / "topology.txt", "w", encoding="utf-8") as file: + file.write(topology.summary()) + file.write("\n") + file.write("internal routing: {}".format(internal_routing["mode"])) + if internal_routing.get("route_reflector"): + file.write(" via {} cluster {}".format( + internal_routing["route_reflector"], + internal_routing["cluster_id"], + )) + file.write("\n") + if locations is not None: + file.write("locations: {} routers\n".format(len(locations))) + + +def main() -> int: + args = parse_args() + emu, topology, internal_routing, locations = build_emulator(args) + + # A dumpfile is useful for debugging or composing emulators; it skips render + # and Docker compilation by design. + if args.dumpfile: + emu.dump(args.dumpfile) + return 0 + + if args.render: + emu.render() + + # Docker compilation happens last, after topology, routing metadata, and + # location labels have all been attached to the emulator objects. + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=resolve_platform(args.platform)), str(output_dir), override=args.override) + write_topology_artifacts(topology, output_dir, internal_routing, locations) + print(topology.summary()) + print("Internal routing mode: {}".format(internal_routing["mode"])) + if internal_routing.get("route_reflector"): + print("Route reflector: {} ({})".format( + internal_routing["route_reflector"], + internal_routing["cluster_id"], + )) + if locations is not None: + print("Generated router locations for {} routers".format(len(locations))) + print("Generated A15 Docker output in {}".format(output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A62_route_reflector/README.md b/examples/basic/A62_route_reflector/README.md new file mode 100644 index 000000000..22d994034 --- /dev/null +++ b/examples/basic/A62_route_reflector/README.md @@ -0,0 +1,124 @@ +# Route Reflector Mini Internet + +This example extends the `B00_mini_internet` topology with iBGP Route Reflector +configuration. It keeps the same IX, stub AS, and eBGP peering shape, then +uses selected transit ASes to demonstrate three iBGP modes: + +- `AS2`: no RR metadata, rendered as legacy full-mesh iBGP. +- `AS12`: one RR cluster with `r101` as the RR and `r104` as the client. +- `AS3`: a larger transit AS with two RR clusters. `r100` reflects for the + west-side clients `r105`, `r110`, and `r111`; `r103` reflects for the + east-side clients `r104`, `r112`, and `r113`. The two RRs are meshed with + each other. + +## Why Route Reflection Is Better + +In a transit AS, all BGP-speaking routers need a way to share routes learned +from external peers. Without route reflectors, iBGP usually uses a full mesh: +every router peers with every other router inside the same AS. For the eight +routers in `AS3`, that design would need: + +```text +8 * 7 / 2 = 28 iBGP sessions +``` + +This example uses route reflection instead. The same eight-router AS is +organized as two clusters: + +```text +West cluster: + r100 = route reflector + r105, r110, r111 = clients + +East cluster: + r103 = route reflector + r104, r112, r113 = clients + +Inter-cluster: + r100 <-> r103 +``` + +With this design, each client only needs an iBGP session to its local route +reflector, and the route reflectors peer with each other. The control plane is +smaller and easier to manage, while BGP routes can still be distributed across +the whole AS. Readers can compare the full-mesh model in `AS2`, the minimal +one-RR model in `AS12`, and the larger two-cluster model in `AS3`. + +## Files + +- `route_reflector.py`: topology entrypoint. Supports the standard example + arguments used by `seedemu/testing/cli.py`. +- `example.yaml`: test manifest for compile, build, runtime readiness, probes, + and the custom runtime test. +- `test_runtime.py`: custom runtime validation using `ComposeRuntimeTest`. +- `output/`: generated Docker compiler output, removed by the clean command. + +## Run + +From the repository root: + +```sh +python examples/basic/A62_route_reflector/route_reflector.py --platform amd --output examples/basic/A62_route_reflector/output +``` + +The legacy platform argument is also accepted: + +```sh +python examples/basic/A62_route_reflector/route_reflector.py amd +``` + +## Test Runner + +Use the standardized runner from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/basic/A62_route_reflector/example.yaml +python seedemu/testing/cli.py compile examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +python seedemu/testing/cli.py build examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +python seedemu/testing/cli.py up examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +python seedemu/testing/cli.py probe examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +python seedemu/testing/cli.py test examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +python seedemu/testing/cli.py down examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/basic/A62_route_reflector/example.yaml --artifact-dir ci-artifacts/a62 +``` + +## Runtime Checks + +The manifest validates simple fixed runtime checks: + +- all router services and ping endpoint services used by the runtime checks are + running; +- representative hosts can communicate across AS boundaries in both tested + directions. + +The custom `test_runtime.py` validates dynamic or example-specific behavior: + +- generated routers have healthy BIRD protocols, with BGP sessions established; +- every generated router service is discovered from Docker Compose metadata, + including both border routers (`brdnode_*`) and internal routers (`rnode_*`); +- routers in RR-enabled ASes (`AS3` and `AS12`) expose iBGP protocol names that + contain `rr`. + +The `AS3` runtime checks include all eight routers, so the test verifies both +the route reflectors and the additional internal RR clients. + +## RR API Pattern + +Use RR mode by registering a cluster on the AS, assigning routers to it, and +marking the RR router: + +```python +as12 = base.getAutonomousSystem(12) +as12.setIbgpMode("route-reflector") +as12.createBgpCluster("10.12.0.1") +as12.getRouter("r101").joinBgpCluster("10.12.0.1").makeRouteReflector() +as12.getRouter("r104").joinBgpCluster("10.12.0.1") +``` + +If an AS has no RR and only the implicit default cluster, `Ibgp` keeps the +legacy full-mesh behavior. diff --git a/examples/basic/A62_route_reflector/example.yaml b/examples/basic/A62_route_reflector/example.yaml new file mode 100644 index 000000000..76d228a64 --- /dev/null +++ b/examples/basic/A62_route_reflector/example.yaml @@ -0,0 +1,132 @@ +id: basic-a62-route-reflector +name: Route Reflector Mini Internet +description: A B00-style mini Internet that demonstrates legacy full-mesh iBGP, one-RR, and a larger two-cluster Route Reflector transit AS. +runner: internet +script: route_reflector.py +platform: amd +features: + - ipv4-default + - mini-internet + - ebgp-route-server + - ebgp-private-peering + - ibgp + - ibgp-route-reflector + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A62 services used by runtime checks are running + type: compose-ps + services: + - brdnode_2_r100 + - brdnode_2_r101 + - brdnode_2_r102 + - brdnode_2_r105 + - brdnode_3_r100 + - brdnode_3_r103 + - brdnode_3_r104 + - brdnode_3_r105 + - rnode_3_r110 + - rnode_3_r111 + - rnode_3_r112 + - rnode_3_r113 + - brdnode_4_r100 + - brdnode_4_r102 + - brdnode_4_r104 + - brdnode_11_r102 + - brdnode_11_r105 + - brdnode_12_r101 + - brdnode_12_r104 + - brdnode_150_router0 + - brdnode_151_router0 + - brdnode_152_router0 + - brdnode_153_router0 + - brdnode_154_router0 + - brdnode_160_router0 + - brdnode_161_router0 + - brdnode_162_router0 + - brdnode_163_router0 + - brdnode_164_router0 + - brdnode_170_router0 + - brdnode_171_router0 + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_154_host_0 + - hnode_160_host_0 + - hnode_171_host_0 + retries: 40 + interval: 3 + +probes: + - name: AS150 reaches AS152 across the RR mini Internet + type: ping + service: hnode_150_host_0 + target: 10.152.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS152 reaches AS150 across the RR mini Internet + type: ping + service: hnode_152_host_0 + target: 10.150.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS150 reaches AS160 through AS3 RR clusters + type: ping + service: hnode_150_host_0 + target: 10.160.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS160 reaches AS150 through AS3 RR clusters + type: ping + service: hnode_160_host_0 + target: 10.150.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS171 reaches AS154 across transit ASes + type: ping + service: hnode_171_host_0 + target: 10.154.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS154 reaches AS171 across transit ASes + type: ping + service: hnode_154_host_0 + target: 10.171.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + +test_programs: + - name: A62 custom runtime validation + script: test_runtime.py + timeout: 1200 diff --git a/examples/basic/A62_route_reflector/route_reflector.py b/examples/basic/A62_route_reflector/route_reflector.py new file mode 100644 index 000000000..0ea87c310 --- /dev/null +++ b/examples/basic/A62_route_reflector/route_reflector.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# Purpose: build the A62 IPv4/BIRD Route Reflector mini-Internet example. +# Inputs are standard TestRunner CLI arguments. Outputs are Docker compiler +# files under --output. Run from the repository root or this example directory. + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Emulator +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing +from seedemu.utilities import Makers + + +AS12_CLUSTER_ID = "10.12.0.1" +AS3_WEST_CLUSTER_ID = "10.3.0.1" +AS3_EAST_CLUSTER_ID = "10.3.0.2" +AS3_WEST_CLIENTS = ["r105", "r110", "r111"] +AS3_EAST_CLIENTS = ["r104", "r112", "r113"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A62 Route Reflector mini Internet example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def make_as3_route_reflector_transit(base: Base): + """Create the larger AS3 topology used to demonstrate RR scaling.""" + as3 = base.createAutonomousSystem(3) + + border_routers = { + 100: as3.createRouter("r100").joinNetwork("ix100"), + 103: as3.createRouter("r103").joinNetwork("ix103"), + 104: as3.createRouter("r104").joinNetwork("ix104"), + 105: as3.createRouter("r105").joinNetwork("ix105"), + } + routers = { + "r100": border_routers[100], + "r103": border_routers[103], + "r104": border_routers[104], + "r105": border_routers[105], + "r110": as3.createRouter("r110"), + "r111": as3.createRouter("r111"), + "r112": as3.createRouter("r112"), + "r113": as3.createRouter("r113"), + } + + for left, right in [ + ("r100", "r105"), + ("r100", "r110"), + ("r100", "r111"), + ("r100", "r103"), + ("r103", "r104"), + ("r103", "r112"), + ("r103", "r113"), + ("r105", "r103"), + ("r111", "r112"), + ]: + network = "net_{}_{}".format(left[1:], right[1:]) + as3.createNetwork(network) + routers[left].joinNetwork(network) + routers[right].joinNetwork(network) + + return as3 + + +def build_mini_internet(emu: Emulator, base: Base, ebgp: Ebgp, hosts_per_as: int): + """Create the B00-style mini Internet and add selected RR metadata.""" + ix100 = base.createInternetExchange(100) + ix101 = base.createInternetExchange(101) + ix102 = base.createInternetExchange(102) + ix103 = base.createInternetExchange(103) + ix104 = base.createInternetExchange(104) + ix105 = base.createInternetExchange(105) + + ix100.getPeeringLan().setDisplayName("NYC-100") + ix101.getPeeringLan().setDisplayName("San Jose-101") + ix102.getPeeringLan().setDisplayName("Chicago-102") + ix103.getPeeringLan().setDisplayName("Miami-103") + ix104.getPeeringLan().setDisplayName("Boston-104") + ix105.getPeeringLan().setDisplayName("Houston-105") + + Makers.makeTransitAs( + base, + 2, + [100, 101, 102, 105], + [(100, 101), (101, 102), (100, 105)], + ) + make_as3_route_reflector_transit(base) + Makers.makeTransitAs( + base, + 4, + [100, 102, 104], + [(100, 104), (102, 104)], + ) + + Makers.makeTransitAs(base, 11, [102, 105], [(102, 105)]) + Makers.makeTransitAs(base, 12, [101, 104], [(101, 104)]) + + # AS2 keeps the legacy full-mesh iBGP behavior. + + # AS12 demonstrates a single Route Reflector and one client. + as12 = base.getAutonomousSystem(12) + as12.setIbgpMode("route-reflector") + as12.createBgpCluster(AS12_CLUSTER_ID) + as12.getRouter("r101").joinBgpCluster(AS12_CLUSTER_ID).makeRouteReflector() + as12.getRouter("r104").joinBgpCluster(AS12_CLUSTER_ID) + + # AS3 demonstrates two RR clusters plus an RR-to-RR mesh. + as3 = base.getAutonomousSystem(3) + as3.setIbgpMode("route-reflector") + as3.createBgpCluster(AS3_WEST_CLUSTER_ID) + as3.createBgpCluster(AS3_EAST_CLUSTER_ID) + as3.getRouter("r100").joinBgpCluster(AS3_WEST_CLUSTER_ID).makeRouteReflector() + for router in AS3_WEST_CLIENTS: + as3.getRouter(router).joinBgpCluster(AS3_WEST_CLUSTER_ID) + as3.getRouter("r103").joinBgpCluster(AS3_EAST_CLUSTER_ID).makeRouteReflector() + for router in AS3_EAST_CLIENTS: + as3.getRouter(router).joinBgpCluster(AS3_EAST_CLUSTER_ID) + + Makers.makeStubAsWithHosts(emu, base, 150, 100, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 151, 100, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 152, 101, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 153, 101, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 154, 102, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 160, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 161, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 162, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 163, 104, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 164, 104, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 170, 105, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 171, 105, hosts_per_as) + + ebgp.addRsPeers(100, [2, 3, 4]) + ebgp.addRsPeers(102, [2, 4]) + ebgp.addRsPeers(104, [3, 4]) + ebgp.addRsPeers(105, [2, 3]) + + ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) + ebgp.addPrivatePeerings(100, [3], [150], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(101, [12], [152, 153], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) + ebgp.addPrivatePeerings(102, [11], [154], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(103, [3], [160, 161, 162], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(104, [3, 4], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [12], [164], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) + ebgp.addPrivatePeerings(105, [11], [171], PeerRelationship.Provider) + + +def build_emulator(hosts_per_as: int = 2) -> Emulator: + emu = Emulator() + base = Base() + ebgp = Ebgp() + + build_mini_internet(emu, base, ebgp, hosts_per_as) + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + return emu + + +def run( + dumpfile=None, + hosts_per_as: int = 2, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, +): + emu = build_emulator(hosts_per_as=hosts_per_as) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + emu.compile(Docker(platform=platform), output or "./output", override=override) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(output_dir), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A62_route_reflector/test_runtime.py b/examples/basic/A62_route_reflector/test_runtime.py new file mode 100644 index 000000000..c0f9557d5 --- /dev/null +++ b/examples/basic/A62_route_reflector/test_runtime.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# +# Purpose: validate the running A62 IPv4/BIRD Route Reflector example after +# TestRunner starts Docker Compose. Inputs come from TestRunner environment +# variables and generated docker-compose.yml labels. Outputs are JSON runtime +# summaries. Side effects are limited to read-only protocol checks. + +from __future__ import annotations + +from pathlib import Path +import sys +from typing import List + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.testing import ComposeRuntimeTest, ComposeService +from seedemu.testing.runtime import ADDRESS_LABEL, ASN_LABEL, NODE_LABEL + + +EXPECTED_ROUTER_COUNT = 31 + +BIRD_PROTOCOL_HEALTH_COMMAND = ( + "birdc show protocols | awk '" + "NR <= 2 { next } " + "$4 != \"up\" { print; bad=1 } " + "$2 == \"BGP\" && $NF != \"Established\" { print; bad=1 } " + "END { exit bad }" + "'" +) + +RR_IBGP_NAME_COMMAND = ( + "birdc show protocols | awk '" + "NR <= 2 { next } " + "$2 == \"BGP\" && $1 ~ /^Ibgp/ { seen=1; if (tolower($1) !~ /rr/) { print; bad=1 } } " + "END { if (!seen) { print \"no iBGP protocols found\"; exit 1 } exit bad }" + "'" +) + + +def discover_routers(test: ComposeRuntimeTest) -> List[ComposeService]: + """Return generated router services sorted by AS and node name.""" + services: List[ComposeService] = [] + + for name, service in test.compose.get("services", {}).items(): + if not (str(name).startswith("brdnode_") or str(name).startswith("rnode_")): + continue + + labels = dict(service.get("labels", {})) + address = str(labels.get(ADDRESS_LABEL, "")).split("/", 1)[0] + services.append(ComposeService(name=str(name), address=address, labels=labels)) + + services.sort( + key=lambda service: ( + int(str(service.labels.get(ASN_LABEL, "0"))), + str(service.labels.get(NODE_LABEL, service.name)), + ) + ) + return services + + +def service_label(service: ComposeService) -> str: + """Return a compact AS/node label for a generated service.""" + return "AS{} {}".format(service.labels.get(ASN_LABEL, "?"), service.labels.get(NODE_LABEL, service.name)) + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + routers = discover_routers(test) + test.structural_check( + "A62 generates all router services", + len(routers) == EXPECTED_ROUTER_COUNT, + "found {} router services".format(len(routers)), + ) + + for router in routers: + test.exec_check( + "{} BIRD protocols are healthy".format(service_label(router)), + router, + BIRD_PROTOCOL_HEALTH_COMMAND, + retries=40, + interval=3, + timeout=60, + ) + + rr_targets = [ + (3, "r100"), + (3, "r103"), + (3, "r104"), + (3, "r105"), + (3, "r110"), + (3, "r111"), + (3, "r112"), + (3, "r113"), + (12, "r101"), + (12, "r104"), + ] + for asn, node in rr_targets: + router = test.find_service(asn, node) + if router: + test.exec_check( + "{} RR iBGP protocol names contain rr".format(service_label(router)), + router, + RR_IBGP_NAME_COMMAND, + retries=20, + interval=3, + timeout=45, + ) + else: + test.structural_check( + "AS{} {} is available for RR protocol checks".format(asn, node), + False, + "service for AS{} node {} not found".format(asn, node), + ) + + test.write_summary("a62-route-reflector-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A63_control_plane_regression/README.md b/examples/basic/A63_control_plane_regression/README.md new file mode 100644 index 000000000..8b87b06cd --- /dev/null +++ b/examples/basic/A63_control_plane_regression/README.md @@ -0,0 +1,26 @@ +# A63 Control-Plane Regression + +This example is the compact runtime regression entry for the control-plane +foundation work. It intentionally keeps several independent slices in one +emulation so a reviewer can run one example after changes to Routing, BGP +intent metadata, FRR rendering, ExaBGP service binding, or MPLS readiness. + +Covered slices: + +- IX100 uses the legacy BIRD route server path with AS150/AS151 clients. +- AS2 mixes a BIRD router and an FRR router from the same OSPF/iBGP/eBGP intent. +- AS3 uses an FRR route reflector and an FRR RR client. +- AS180 installs ExaBGP through `ExaBgpService + Binding` and peers with an FRR + router in AS4. ExaBGP is not a router backend. +- AS20 enables MPLS/LDP readiness on a small transit chain. The default runtime + test checks generated MPLS interface and FRR LDP config. Full MPLS dataplane + validation remains host-gated because GitHub hosted runners may not provide + MPLS kernel modules. + +Run it locally: + +```bash +python -m seedemu.testing.cli clean examples/basic/A63_control_plane_regression/example.yaml +python -m seedemu.testing.cli compile examples/basic/A63_control_plane_regression/example.yaml +COMPOSE_PROJECT_NAME=seedemu-a63-control-plane python -m seedemu.testing.cli all examples/basic/A63_control_plane_regression/example.yaml +``` diff --git a/examples/basic/A63_control_plane_regression/control_plane_regression.py b/examples/basic/A63_control_plane_regression/control_plane_regression.py new file mode 100644 index 000000000..91ac5a695 --- /dev/null +++ b/examples/basic/A63_control_plane_regression/control_plane_regression.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Mpls, Ospf, PeerRelationship, Routing +from seedemu.services import ExaBgpService, WebService + + +AS3_CLUSTER_ID = "10.3.0.1" +EXABGP_PREFIX = "198.51.100.0/24" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the A63 control-plane regression example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_route_server_slice(emu: Emulator, base: Base, ebgp: Ebgp, web: WebService) -> None: + """Keep the legacy BIRD route-server path visible in the regression topology.""" + + base.createInternetExchange(100) + + for asn in [150, 151]: + current_as = base.createAutonomousSystem(asn) + current_as.createNetwork("net0") + current_as.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + current_as.createHost("web").joinNetwork("net0") + + vnode = "web{}".format(asn) + web.install(vnode) + emu.addBinding(Binding(vnode, filter=Filter(nodeName="web", asn=asn))) + ebgp.addRsPeer(100, asn) + + +def build_frr_route_server_slice(emu: Emulator, base: Base, ebgp: Ebgp, web: WebService) -> None: + """Exercise a FRR route-server IX without changing the legacy BIRD RS default.""" + + ix = base.createInternetExchange(107) + ix.getRouteServerNode().setRoutingBackend("frr") + + for asn in [157, 158]: + current_as = base.createAutonomousSystem(asn) + current_as.createNetwork("net0") + current_as.createRouter("router0").joinNetwork("net0").joinNetwork("ix107") + current_as.createHost("web").joinNetwork("net0") + + vnode = "web{}".format(asn) + web.install(vnode) + emu.addBinding(Binding(vnode, filter=Filter(nodeName="web", asn=asn))) + ebgp.addRsPeer(107, asn) + + +def build_mixed_backend_slice(emu: Emulator, base: Base, ebgp: Ebgp, web: WebService) -> None: + """Exercise one transit AS with BIRD and FRR routers from shared BGP/OSPF intent.""" + + base.createInternetExchange(101) + base.createInternetExchange(102) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix101") + as2.createRouter("r2", routingBackend="frr").joinNetwork("net0").joinNetwork("ix102") + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix101") + as152.createHost("web").joinNetwork("net0") + web.install("web152") + emu.addBinding(Binding("web152", filter=Filter(nodeName="web", asn=152))) + + as153 = base.createAutonomousSystem(153) + as153.createNetwork("net0") + as153.createRouter("router0").joinNetwork("net0").joinNetwork("ix102") + as153.createHost("web").joinNetwork("net0") + web.install("web153") + emu.addBinding(Binding("web153", filter=Filter(nodeName="web", asn=153))) + + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(102, 2, 153, abRelationship=PeerRelationship.Provider) + + +def build_frr_route_reflector_slice(base: Base, ebgp: Ebgp) -> None: + """Validate FRR route-reflector rendering and runtime route propagation.""" + + base.createInternetExchange(103) + + as3 = base.createAutonomousSystem(3) + as3.createNetwork("net0") + as3.setIbgpMode("route-reflector") + as3.createBgpCluster(AS3_CLUSTER_ID) + as3.createRouter("rr", routingBackend="frr").joinNetwork("net0").joinNetwork("ix103").joinBgpCluster(AS3_CLUSTER_ID).makeRouteReflector() + as3.createRouter("client", routingBackend="frr").joinNetwork("net0").joinBgpCluster(AS3_CLUSTER_ID) + + as154 = base.createAutonomousSystem(154) + as154.createNetwork("net0") + as154.createRouter("router0").joinNetwork("net0").joinNetwork("ix103") + as154.createHost("host").joinNetwork("net0") + + ebgp.addPrivatePeering(103, 3, 154, abRelationship=PeerRelationship.Provider) + + +def build_exabgp_slice(base: Base, exabgp: ExaBgpService, emu: Emulator) -> None: + """Install ExaBGP as a Service + Binding speaker, never as a router backend.""" + + base.createInternetExchange(104) + + as4 = base.createAutonomousSystem(4) + as4.createNetwork("net0") + as4.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix104") + + as180 = base.createAutonomousSystem(180) + as180.createHost("exabgp").joinNetwork("ix104", address="10.104.0.180") + + exabgp.install("as180_exabgp") \ + .setLocalAsn(180) \ + .addPeer("router0", router_asn=4, router_relationship="customer") \ + .addAnnouncement(EXABGP_PREFIX) + emu.addBinding(Binding("as180_exabgp", filter=Filter(asn=180, nodeName="exabgp"))) + + +def build_mpls_readiness_slice(base: Base, ebgp: Ebgp, mpls: Mpls) -> None: + """Keep MPLS config/readiness in scope while leaving dataplane checks host-gated.""" + + base.createInternetExchange(105) + base.createInternetExchange(106) + + as20 = base.createAutonomousSystem(20) + as20.createNetwork("net0") + as20.createNetwork("net1") + as20.createNetwork("net2") + as20.createRouter("r1").joinNetwork("net0").joinNetwork("ix105") + as20.createRouter("r2").joinNetwork("net0").joinNetwork("net1") + as20.createRouter("r3").joinNetwork("net1").joinNetwork("net2") + as20.createRouter("r4").joinNetwork("net2").joinNetwork("ix106") + mpls.enableOn(20) + + as155 = base.createAutonomousSystem(155) + as155.createNetwork("net0") + as155.createRouter("router0").joinNetwork("net0").joinNetwork("ix105") + + as156 = base.createAutonomousSystem(156) + as156.createNetwork("net0") + as156.createRouter("router0").joinNetwork("net0").joinNetwork("ix106") + + ebgp.addPrivatePeering(105, 20, 155, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(106, 20, 156, abRelationship=PeerRelationship.Provider) + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + ebgp = Ebgp() + mpls = Mpls() + exabgp = ExaBgpService() + web = WebService() + + build_route_server_slice(emu, base, ebgp, web) + build_frr_route_server_slice(emu, base, ebgp, web) + build_mixed_backend_slice(emu, base, ebgp, web) + build_frr_route_reflector_slice(base, ebgp) + build_exabgp_slice(base, exabgp, emu) + build_mpls_readiness_slice(base, ebgp, mpls) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(ebgp) + emu.addLayer(mpls) + emu.addLayer(exabgp) + emu.addLayer(web) + return emu + + +def main() -> int: + args = parse_args() + emu = build_emulator() + + if args.dumpfile: + emu.dump(args.dumpfile) + print("Saved A63 emulator to {}".format(args.dumpfile)) + return 0 + + if args.render: + emu.render() + + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=resolve_platform(args.platform)), str(output_dir), override=args.override) + print("Generated A63 Docker output in {}".format(output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/basic/A63_control_plane_regression/example.yaml b/examples/basic/A63_control_plane_regression/example.yaml new file mode 100644 index 000000000..51dda2304 --- /dev/null +++ b/examples/basic/A63_control_plane_regression/example.yaml @@ -0,0 +1,94 @@ +id: basic-a63-control-plane-regression +name: Control-Plane Regression +description: A compact regression topology for route-server, mixed BIRD/FRR, FRR route-reflector, ExaBGP service, and MPLS readiness coverage. +runner: internet +script: control_plane_regression.py +platform: amd +features: + - ipv4-default + - ebgp-route-server + - ebgp-private-peering + - routing-backend + - bird + - frr + - ibgp + - ibgp-route-reflector + - ospf + - exabgp-service + - mpls-readiness + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: A63 control-plane services are running + type: compose-ps + services: + - rs_ix_ix100 + - brdnode_150_router0 + - brdnode_151_router0 + - brdnode_2_r1 + - brdnode_2_r2 + - brdnode_152_router0 + - brdnode_153_router0 + - brdnode_3_rr + - rnode_3_client + - brdnode_154_router0 + - brdnode_4_router0 + - hnode_180_exabgp + - brdnode_20_r1 + - rnode_20_r2 + - rnode_20_r3 + - brdnode_20_r4 + retries: 45 + interval: 3 + +probes: + - name: AS151 learns AS150 prefix through the BIRD route server + type: exec + service: brdnode_151_router0 + command: birdc show route | grep -q '10.150.0.0/24' + expect_exit: 0 + retries: 40 + interval: 5 + + - name: AS152 FRR router learns AS153 route through mixed-backend iBGP + type: exec + service: brdnode_152_router0 + command: vtysh -c "show ip bgp" | grep -q '10.153.0.0/24' + expect_exit: 0 + retries: 40 + interval: 5 + + - name: AS3 FRR route-reflector client learns AS154 route + type: exec + service: rnode_3_client + command: vtysh -c "show ip bgp" | grep -q '10.154.0.0/24' + expect_exit: 0 + retries: 40 + interval: 5 + + - name: AS4 FRR router learns ExaBGP static route + type: exec + service: brdnode_4_router0 + command: vtysh -c "show ip bgp 198.51.100.0/24" | grep -q '198.51.100.0/24' + expect_exit: 0 + retries: 40 + interval: 5 + +test_programs: + - name: A63 control-plane regression runtime validation + script: test_runtime.py + timeout: 1200 diff --git a/examples/basic/A63_control_plane_regression/test_runtime.py b/examples/basic/A63_control_plane_regression/test_runtime.py new file mode 100644 index 000000000..e5907f547 --- /dev/null +++ b/examples/basic/A63_control_plane_regression/test_runtime.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.testing import ComposeRuntimeTest, ComposeService + + +ROUTING_BACKEND_LABEL = "org.seedsecuritylabs.seedemu.meta.seedemu_routing_backend" +ASN_LABEL = "org.seedsecuritylabs.seedemu.meta.asn" +NODE_LABEL = "org.seedsecuritylabs.seedemu.meta.nodename" + + +def require_backend(test: ComposeRuntimeTest, service: ComposeService, backend: str) -> None: + actual = service.labels.get(ROUTING_BACKEND_LABEL) + label = "AS{} {} uses {} routing backend".format( + service.labels.get(ASN_LABEL), + service.labels.get(NODE_LABEL), + backend, + ) + test.structural_check(label, actual == backend, "expected {}, found {}".format(backend, actual)) + + +def record_mpls_host_capability(test: ComposeRuntimeTest, service: ComposeService) -> None: + result = test.exec(service, "test -d /proc/sys/net/mpls || test -e /proc/modules", timeout=20) + available = result["exit"] == 0 + test.structural_check( + "MPLS dataplane probe is host-gated", + True, + "host kernel capability visible" if available else "host kernel capability not visible; dataplane probe skipped", + ) + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + rs100 = test.require_service(100, "ix100", "IX100 BIRD route server is generated") + as150_router = test.require_service(150, "router0") + as151_router = test.require_service(151, "router0") + rs107 = test.require_service(107, "ix107", "IX107 FRR route server is generated") + as157_router = test.require_service(157, "router0") + as158_router = test.require_service(158, "router0") + as2_r1 = test.require_service(2, "r1") + as2_r2 = test.require_service(2, "r2") + as152_router = test.require_service(152, "router0") + as153_router = test.require_service(153, "router0") + + as3_rr = test.require_service(3, "rr") + as3_client = test.require_service(3, "client") + as154_router = test.require_service(154, "router0") + + as4_router = test.require_service(4, "router0") + speaker = test.require_service(180, "exabgp") + + as20_r1 = test.require_service(20, "r1") + as20_r2 = test.require_service(20, "r2") + as20_r3 = test.require_service(20, "r3") + as20_r4 = test.require_service(20, "r4") + + if rs100: + require_backend(test, rs100, "bird") + test.exec_check("IX100 route server starts BIRD", rs100, "pgrep -x bird >/dev/null") + test.exec_check("IX100 route server peers with AS150", rs100, "grep -q 'neighbor 10.100.0.150 as 150' /etc/bird/bird.conf") + test.exec_check("IX100 route server peers with AS151", rs100, "grep -q 'neighbor 10.100.0.151 as 151' /etc/bird/bird.conf") + test.exec_check("IX100 route server renders rs client", rs100, "grep -q 'rs client' /etc/bird/bird.conf") + + if as150_router: + require_backend(test, as150_router, "bird") + test.exec_check("AS150 route-server client starts BIRD", as150_router, "pgrep -x bird >/dev/null") + test.exec_check("AS150 route-server session is established", as150_router, "birdc show protocols | grep -q 'p_rs100.*Established'", retries=15) + + if as151_router: + require_backend(test, as151_router, "bird") + test.exec_check("AS151 route-server client starts BIRD", as151_router, "pgrep -x bird >/dev/null") + test.exec_check("AS151 route-server session is established", as151_router, "birdc show protocols | grep -q 'p_rs100.*Established'", retries=15) + + if as151_router: + test.exec_check("AS151 learns AS150 route through route server", as151_router, "birdc show route | grep -q '10.150.0.0/24'", retries=15) + + if rs107: + require_backend(test, rs107, "frr") + test.exec_check("IX107 route server starts FRR bgpd", rs107, "pgrep -x bgpd >/dev/null") + test.exec_check("IX107 route server does not start BIRD", rs107, "! pgrep -x bird >/dev/null") + test.exec_check("IX107 route server renders AS157 as RS client", rs107, "grep -q 'neighbor 10.107.0.157 route-server-client' /etc/frr/frr.conf") + test.exec_check("IX107 route server renders AS158 as RS client", rs107, "grep -q 'neighbor 10.107.0.158 route-server-client' /etc/frr/frr.conf") + test.exec_check("IX107 route server sees AS157 BGP session", rs107, "vtysh -c 'show ip bgp summary' | grep -q '10.107.0.157'", retries=15) + test.exec_check("IX107 route server sees AS158 BGP session", rs107, "vtysh -c 'show ip bgp summary' | grep -q '10.107.0.158'", retries=15) + + if as157_router: + require_backend(test, as157_router, "bird") + test.exec_check("AS157 route-server session is established", as157_router, "birdc show protocols | grep -q 'p_rs107.*Established'", retries=15) + + if as158_router: + require_backend(test, as158_router, "bird") + test.exec_check("AS158 route-server session is established", as158_router, "birdc show protocols | grep -q 'p_rs107.*Established'", retries=15) + test.exec_check("AS158 learns AS157 route through FRR route server", as158_router, "birdc show route | grep -q '10.157.0.0/24'", retries=15) + + if as2_r1: + require_backend(test, as2_r1, "bird") + test.exec_check("AS2 r1 starts BIRD", as2_r1, "pgrep -x bird >/dev/null") + test.exec_check("AS2 r1 does not carry FRR config", as2_r1, "test ! -e /etc/frr/frr.conf") + test.exec_check("AS2 r1 renders OSPF intent", as2_r1, "grep -q 'protocol ospf ospf1' /etc/bird/bird.conf") + test.exec_check("AS2 r1 iBGP to FRR r2 is established", as2_r1, "birdc show protocols | grep -q 'ibgp1.*Established'", retries=15) + test.exec_check("AS2 r1 learns AS153 route", as2_r1, "birdc show route | grep -q '10.153.0.0/24'", retries=15) + + if as2_r2: + require_backend(test, as2_r2, "frr") + test.exec_check("AS2 r2 starts FRR bgpd", as2_r2, "pgrep -x bgpd >/dev/null") + test.exec_check("AS2 r2 does not start BIRD", as2_r2, "! pgrep -x bird >/dev/null") + test.exec_check("AS2 r2 renders FRR OSPF intent", as2_r2, "grep -q 'router ospf' /etc/frr/frr.conf") + test.exec_check("AS2 r2 learns AS152 route through iBGP", as2_r2, "vtysh -c 'show ip bgp' | grep -q '10.152.0.0/24'", retries=15) + + if as152_router: + require_backend(test, as152_router, "frr") + test.exec_check("AS152 FRR router learns AS153 route", as152_router, "vtysh -c 'show ip bgp' | grep -q '10.153.0.0/24'", retries=15) + + if as153_router: + require_backend(test, as153_router, "bird") + test.exec_check("AS153 BIRD router learns AS152 route", as153_router, "birdc show route | grep -q '10.152.0.0/24'", retries=15) + + if as3_rr: + require_backend(test, as3_rr, "frr") + test.exec_check("AS3 RR starts FRR bgpd", as3_rr, "pgrep -x bgpd >/dev/null") + test.exec_check("AS3 RR renders cluster-id", as3_rr, "grep -q 'bgp cluster-id 10.3.0.1' /etc/frr/frr.conf") + test.exec_check("AS3 RR renders route-reflector client", as3_rr, "grep -q 'route-reflector-client' /etc/frr/frr.conf") + + if as3_client: + require_backend(test, as3_client, "frr") + test.exec_check("AS3 RR client starts FRR bgpd", as3_client, "pgrep -x bgpd >/dev/null") + test.exec_check("AS3 RR client learns AS154 route", as3_client, "vtysh -c 'show ip bgp' | grep -q '10.154.0.0/24'", retries=15) + + if as154_router: + require_backend(test, as154_router, "bird") + test.exec_check("AS154 BIRD router peers with AS3 RR", as154_router, "birdc show protocols | grep -q 'Established'", retries=15) + + if as4_router: + require_backend(test, as4_router, "frr") + test.exec_check("AS4 ExaBGP peer is an FRR router", as4_router, "grep -q 'neighbor 10.104.0.180 remote-as 180' /etc/frr/frr.conf") + test.exec_check("AS4 router does not start BIRD", as4_router, "! pgrep -x bird >/dev/null") + test.exec_check("AS4 learns ExaBGP static route", as4_router, "vtysh -c 'show ip bgp 198.51.100.0/24' | grep -q '198.51.100.0/24'", retries=15) + + if speaker: + test.exec_check("ExaBGP speaker process is running", speaker, "pgrep -f 'exabgp /etc/exabgp/exabgp.conf' >/dev/null", retries=15) + test.exec_check("ExaBGP manual control FIFO is available", speaker, "test -p /run/exabgp/manual.in") + test.exec_check("ExaBGP config peers with AS4 router", speaker, "grep -q 'neighbor 10.104.0.4' /etc/exabgp/exabgp.conf") + test.exec_check("ExaBGP config announces static IPv4 route", speaker, "grep -q 'route 198.51.100.0/24 next-hop self' /etc/exabgp/exabgp.conf") + + if as20_r1: + test.exec_check("AS20 r1 has MPLS/LDP on net0", as20_r1, "grep -q '^net0$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf") + record_mpls_host_capability(test, as20_r1) + + if as20_r2: + test.exec_check("AS20 r2 has MPLS/LDP on net0 and net1", as20_r2, "grep -q '^net0$' /mpls_ifaces.txt && grep -q '^net1$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf") + + if as20_r3: + test.exec_check("AS20 r3 has MPLS/LDP on net1 and net2", as20_r3, "grep -q '^net1$' /mpls_ifaces.txt && grep -q '^net2$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf") + + if as20_r4: + test.exec_check("AS20 r4 has MPLS/LDP on net2", as20_r4, "grep -q '^net2$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf") + + test.write_summary("a63-control-plane-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D00_ethereum_poa/component_poa.py b/examples/blockchain/D00_ethereum_poa/component_poa.py index dc0ff2b8c..df5952a4b 100755 --- a/examples/blockchain/D00_ethereum_poa/component_poa.py +++ b/examples/blockchain/D00_ethereum_poa/component_poa.py @@ -70,7 +70,7 @@ def run(dumpfile=None, total_eth_nodes=20, total_accounts_per_node=2): faucet.setDisplayName('Faucet') faucet.addHostName('faucet' + DOMAIN) # Add this name to /etc/hosts - # Create the Utility server + # # Create the Utility server util_server:EthUtilityServer = blockchain.createEthUtilityServer( vnode='utility', port=5000, diff --git a/examples/blockchain/D01_ethereum_pos/README.md b/examples/blockchain/D01_ethereum_pos/README.md index c334c0be5..0fcbf5589 100644 --- a/examples/blockchain/D01_ethereum_pos/README.md +++ b/examples/blockchain/D01_ethereum_pos/README.md @@ -1,201 +1,341 @@ -# How to run Ethereum PoS private network - The Merge +# Ethereum Blockchain (PoS) -In this example, we show how the SEED Emulator to emulate -The Merge that is an upgrade to proof-of-stake. -In the real world, consensus transition occurs from proof-of-work -to proof-of-stake. However, in the emulation, we starts the -blockchain with proof-of-authority and then transits to proof-of-stake. -With proof-of-authority, we can allocate a balance to specific accounts -and we do not need to wait for miner to accumulate enough Ethereum to -stake for proof-of-stake consensus. +This example demonstrates how to create an Ethereum proof-of-stake +private blockchain in the SEED Emulator. The blockchain is started as +a PoS chain from the genesis state. It uses Geth for the execution +layer and Lighthouse for the consensus layer. + +The example creates separate Geth nodes, Beacon nodes, and Validator +Client nodes. It also creates a Beacon Setup node, which prepares the +configuration files and genesis data needed by the Beacon nodes and +Validator Clients. ![](pics/POS-1.png) -## Table of contents -### [1. Emulate The Merge](#1-emulate-the-merge-1) +## Table of Content + +- [Create the Internet Base](#create-the-internet-base) +- [Create a PoS Blockchain](#create-a-pos-blockchain) +- [Create Pre-funded Accounts](#create-pre-funded-accounts) +- [Create the Beacon Setup Node](#create-the-beacon-setup-node) +- [Create Geth Nodes](#create-geth-nodes) +- [Create Beacon Nodes](#create-beacon-nodes) +- [Set Bootnodes](#set-bootnodes) +- [Create Validator Client Nodes](#create-validator-client-nodes) +- [Create Faucet and Utility Servers](#create-faucet-and-utility-servers) +- [Binding to Physical Nodes](#binding-to-physical-nodes) +- [Generate the Docker Output](#generate-the-docker-output) +- [How to Run](#how-to-run) +- [Access Points](#access-points) +- [Relationship with Other Examples](#relationship-with-other-examples) -#### [1.1 Internet Emulator Base](#11-internet-emulator-base-1) -#### [1.2 Creating Ethereum POS Node](#12-creating-ethereum-pos-node-1) +## Create the Internet Base -#### [1.3 Creating Beacon Setup Node ](#13-creating-beacon-setup-node-1) +In this example, we emulate the Internet using 10 stub ASes: + +```text +150, 151, 152, 153, 154, 160, 161, 162, 163, 164 +``` -#### [1.4 Set a node as a bootnode](#14-set-a-node-as-a-bootnode-1) +Each stub AS has 3 hosts. The emulator therefore has 30 physical hosts +in total. -#### [1.5 Add Validator](#15-add-validator-1) +```python +hosts_per_stub_as = 3 +emu = Makers.makeEmulatorBaseWith10StubASAndHosts( + hosts_per_stub_as = hosts_per_stub_as) +``` -### [2. Introduction to PoS](#2-introduction-to-pos-1) +## Create a PoS Blockchain -# 1. Emulate `The Merge` +To create a blockchain, we first create the `EthereumService`, and then +create a blockchain inside this service. The service supports multiple +chains. -## 1.1 Internet Emulator Base +```python +eth = EthereumService() +blockchain = eth.createBlockchain( + chainName="pos", + consensus=ConsensusMechanism.POS) +``` -In this example, we emulate the internet with 10 stub ASes. +For a PoS blockchain, the SEED Emulator uses a split Ethereum +architecture: -ASes : [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] +- Geth nodes run the execution layer. +- Beacon nodes run the consensus layer. +- Validator Client nodes hold validator keys and perform block proposal + and attestation duties. +- The Beacon Setup node generates and distributes the testnet + configuration and genesis data. -Each stub AS has 3 hosts, and the emulator has 30 hosts in total -For example, the ips of 3 hosts in each AS 150 and AS 151 will be assigned as below +## Create Pre-funded Accounts -- 10.150.0.71 -- 10.150.0.72 -- 10.150.0.73 -- 10.151.0.71 -- 10.151.0.72 -- 10.151.0.73 +The example creates 10 pre-funded user accounts. These accounts are +derived from a mnemonic using the Ethereum wallet derivation path used +by wallets such as MetaMask. ```python -# Create the Internat Emulator Base with 10 Stub AS (150-154, 160-164) using the Makers utility tool. -# hosts_per_stub_as=3 : create 3 hosts per one stub AS. -# It will create hosts(physical node) named `host_{}`.format(counter), counter starts from 0. -hosts_per_stub_as = 3 -emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_total_per_as) +accounts_total = 10 +pre_funded_amount = 1000000 +mnemonic = "gentle always fun glass foster produce north tail security list example gain" + +blockchain.addLocalAccountsFromMnemonic( + mnemonic=mnemonic, + total=accounts_total, + balance=pre_funded_amount) ``` -## 1.2 Creating Ethereum POS Node +All accounts created during the build time are added, together with +their balances, to the genesis block. Therefore, their balances are +recognized as soon as the blockchain starts. + +Users can import the mnemonic into MetaMask and use the generated +accounts on this private chain. + + +## Create the Beacon Setup Node -We create the POS blockchain sub-layer in the Ethereum layer and -creates the POS ethereum nodes using the Blockchain::createNode() method. +A PoS blockchain needs genesis data for both the execution layer and +the consensus layer. The Beacon Setup node prepares the Beacon chain +configuration, validator key information, deposit data, and genesis +state used by the rest of the PoS nodes. ```python -# Create the Ethereum layer -eth = EthereumService() +beaconsetupServer: PoSBeaconSetupServer = \ + blockchain.createBeaconSetupNode("BeaconSetupNode") +emu.getVirtualNode("BeaconSetupNode").setDisplayName( + "Ethereum-POS-BeaconSetup") +``` + +The Beacon Setup node does not run as a normal Geth node or Beacon +node. Its main purpose is to generate and serve the bootstrapping data +needed by the other PoS nodes. + + +## Create Geth Nodes + +Geth nodes run the Ethereum execution layer. They maintain the +execution state, process transactions, expose JSON-RPC APIs, and +communicate with Beacon nodes through the authenticated Engine API. + +```python +for i in range(geth_node_number): + gethServer: PoSGethServer = blockchain.createGethNode( + f"gethnode{i}") + gethServer.enableGethHttp() + gethServer.appendClassName(f"Ethereum-POS-Geth-{i+1}") + gethServer.addHostName(f"gethnode{i}" + DOMAIN) +``` + +The `enableGethHttp()` API enables the HTTP JSON-RPC interface. This is +useful for applications, tools, MetaMask, Faucet, Utility Server, and +the Eth Explorer. + + +## Create Beacon Nodes + +Beacon nodes run the Ethereum consensus layer. They maintain Beacon +chain state, slots, epochs, validators, attestations, and finality +information. Each Beacon node is connected to one Geth node. -# Create the Blockchain layer which is a sub-layer of Ethereum layer. -# chainName="pos": set the blockchain name as "pos" -# consensus="ConsensusMechnaism.POS" : set the consensus of the blockchain as "ConsensusMechanism.POS". -# supported consensus option: ConsensusMechanism.POA, ConsensusMechanism.POW, ConsensusMechanism.POS -blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) +```python +for i in range(beacon_node_number): + beaconServer: PoSBeaconServer = blockchain.createBeaconNode( + f"beaconnode{i}") + beaconServer.appendClassName(f"Ethereum-POS-Beacon-{i+1}") + beaconServer.connectToGethNode( + f"gethnode{(i+1)%len(geth_nodes)}") +``` -# set `terminal_total_difficulty`, which is the value to designate when the Merge is happen. -# The default value is 20. -# In this example, a block is sealed for every 15 seconds. If we set `terminal_total_difficulty` to 20, -# the Ethereum blockchain will keep POA for approximately 150 sec (20/2\*15) -# and then conduct the Merge to change the consensus mechanism to POS. -blockchain.setTerminalTotalDifficulty(20) +The Beacon node talks to its connected Geth node through the Engine API. +This is how the consensus layer asks the execution layer to build, +validate, and import execution payloads. -asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] -i = 1 -for asn in asns: - for id in range(hosts_per_stub_as): - # Create a blockchain virtual node named "eth{}".format(i) - e:EthereumServer = blockchain.createNode("eth{}".format(i)) +## Set Bootnodes - # Create Docker Container Label named 'Ethereum-POS-i' - e.appendClassName('Ethereum-POS-{}'.format(i)) +The example sets one Geth node and one Beacon node as bootnodes. +Bootnodes help other nodes discover peers when the private chain starts. - # Enable Geth to communicate with geth node via http - e.enableGethHttp() +```python +geth_nodes[0].setBootNode(True) +beacon_nodes[0].setBootNode(True) ``` -## 1.3 Creating Beacon Setup Node +The Geth bootnode is used by execution-layer nodes. The Beacon bootnode +is used by consensus-layer nodes. + -In order to run Ethereum PoS, you need to run not only the mainnet (execution layer) but also the beacon chain (consensus layer). In this example, we use Geth software to run execution layer and Lighthouse to run consensus layer. -To run a beacon node, config information is needed. -Like we set genesis.yaml when running Geth, genesis configurations is needed when running Lighthouse. A Beacon Setup Node take care of configuration files which is needed to run Beacon Node. The following code shows how we set beacon setup node. -The Beacon Setup Node is essential to run POS. +## Create Validator Client Nodes + +Validator Client nodes hold validator keys and connect to Beacon nodes. +They do not maintain the whole Beacon chain state themselves. Instead, +they rely on Beacon nodes to provide duties and unsigned blocks or +attestations, then sign the required messages with validator keys. ```python -# Set host in asn 150 with id 0 (ip : 10.150.0.71) as BeaconSetupNode. -if asn == 150 and id == 0: - e.setBeaconSetupNode() +for i in range(vc_node_number): + VcServer: PoSVcServer = blockchain.createVcNode(f"vcnode{i}") + VcServer.appendClassName(f"Ethereum-POS-Validator-{i+1}") + VcServer.connectToBeaconNode( + f"beaconnode{(i+1)%len(beacon_nodes)}") + VcServer.enablePOSValidatorAtGenesis() ``` -The role of a Beacon Setup Node +This example enables all Validator Client nodes at genesis. This means +their validators are part of the initial Beacon state, so they can +participate in block proposal and attestation after the PoS network +starts. + +Adding validators after the chain has started is demonstrated in the +[D23_validator](../D23_validator/) example. + -- generate config files for beaconchain -- create validator keys -- deposit for validators that is enabled at the point of Genesis. -- distributes all those data to the other nodes. +## Create Faucet and Utility Servers -A Beacon Setup Node does not run any ethereum node. It's only role is to generate -and distribute data for beacon nodes. To deposit for validators, it makes an api request to the Geth node and send transactions. By default, it will connect to one of node that runs the bootnode. +The example also creates a Faucet server and a Utility server. These +servers are useful for later examples and for interacting with the +private chain. -# 1.4 Set a node as a bootnode +```python +faucet: FaucetServer = blockchain.createFaucetServer( + vnode="faucet", + port=80, + linked_eth_node="gethnode1", + balance=10000, + max_fund_amount=10) +faucet.setDisplayName("Faucet") +``` -We can set a node as a bootnode that bootstraps all blockchain nodes. -If a node is set as a bootnode, it will run a http server that sends -its blockchain node url so that the other nodes can connect to it. -When it is a POS blockchain, the node serves as a BootNode in both layers: execution layer and consensus layer. -If bootnode does not set to any node, we should specify -peer nodes urls manually. +The Faucet server can fund user accounts during runtime. ```python -# Set host in asn 150 with id 1 (ip : 10.150.0.72) as BootNode. -# This node will serve as a BootNode in both execution layer (geth) and consensus layer (lighthouse). -if asn == 150 and id == 1: - e.setBootNode(True) +util_server: EthUtilityServer = blockchain.createEthUtilityServer( + vnode="utility", + port=5000, + linked_eth_node="gethnode2", + linked_faucet_node="faucet") +util_server.setDisplayName("UtilityServer") ``` -# 1.5 Add Validator +The Utility server can deploy smart contracts and provide contract +addresses to other applications. -There are 2 ways to add validator in this emulator -- Enable validator at genesis -- Enable validator at running +## Binding to Physical Nodes -We can specifies which validators will be activated from the genesis state as well. -This way is to enable validator at genesis. -But once the beaconchain is initiated, there is no way to add validators in genesis configurations. -To be a validator, we need to stake 32 Ethereum and wait until the validator is activated. -The activation requires a specific amount of time to get the validator's stake information -from the execution layer, verify the data, and wait until the validator to be activated. -We emulate this by enable validator at running. +All virtual nodes need to be bound to physical nodes. This example +binds all Ethereum-related virtual nodes to hosts in the base Internet +topology. ```python -# Set hosts in asn 152 and 162 with id 0 and 1 as validator node. -# Validator is added by deposit 32 Ethereum and is activated in realtime after the Merge. -# isManual=True : deposit 32 Ethereum by manual. -# Other than deposit part, create validator key and running a validator node is done by codes. -if asn in [152, 162]: - if id == 0: - e.enablePOSValidatorAtRunning() - if id == 1: - e.enablePOSValidatorAtRunning(is_manual=True) +for _, servers in blockchain.getAllServerNames().items(): + for server in servers: + emu.addBinding(Binding( + server, + filter=Filter(nodeName="host_*"), + action=Action.FIRST)) +``` + -# Set hosts in asn 152, 153, 154, and 160 as validator node. -# These validators are activated by default from genesis status. -# Before the Merge, when the consensus in this blockchain is still POA, -# these hosts will be the signer nodes. -if asn in [151,153,154,160]: - e.enablePOSValidatorAtGenesis() +## Generate the Docker Output + +The example enables both the Internet Map and the Eth Explorer when +compiling the Docker output. + +```python +docker = Docker( + internetMapEnabled=True, + etherViewEnabled=ether_view_enabled, + platform=platform) +emu.compile(docker, output, override=True) ``` -If we use the method: `enablePOSValidatorAtRunning()` with parameter `is_manual=False`, -A new validator will be added automatically once the emulator runs. -However, if `is_manual` is set to `True`, we need to run deposit.sh script by manual. -All other tasks will be executed but deposit action. It will create a validator key and -run validator node with lighthouse. But it will not be activated until we stake 32 Ethereum -by executing the `deposit.sh` script under `/tmp` folder. The following example shows how to -run the `deposit.sh`. +The Internet Map shows the emulated Internet topology. The Eth Explorer +shows the Ethereum PoS chain state, including slots, epochs, blocks, +validators, and execution-layer transactions. +`ether_view_enabled` is controlled by command-line options. It is enabled by +default for manual runs, and can be disabled for CI: + +```bash +python3 ethereum_pos.py --no-ether-view ``` -$ docker ps | grep -i 10.152.0.72 -b127d8503f24 output_hnode_152_host_1 "/start.sh" 15 minutes ago Up 15 minutes as152h-Ethereum-POS-8-10.152.0.72 - -$ docksh b127 -=============================================================== +## How to Run + +Run the standardized test workflow from the repository root: + +```bash +tools/run-example-test.sh D01 compile +tools/run-example-test.sh D01 all +``` -root@b127d8503f24 / # ls -bin boot dev etc home ifinfo.txt interface_setup lib lib32 lib64 libx32 media mnt opt proc root run sbin seedemu_sniffer seedemu_worker srv start.sh sys tmp usr var +The test manifest splits validation across lifecycle stages: -root@b127d8503f24 / # cd tmp -root@b127d8503f24 /tmp # ls -beacon-bootstrapper beacon-setup-node deposit.sh eth-bootstrapper eth-genesis.json eth-nodes eth-node-urls eth-password jwt.hex keystore local-testnet seed.pass testnet.tar.gz tmp6i66iy80 +- `runtime.readiness` checks that the expected Geth, Beacon, Validator, + Beacon Setup, Faucet, and Utility containers are running. +- `probes` checks key runtime facts: Geth/Lighthouse processes are + active, the Beacon Setup node produced `genesis.ssz`, Geth exposes a + local funded account, and the PoS chain advances execution blocks. +- `test_programs` runs the stateful transaction test: it sends a signed + ETH transfer and verifies the receipt plus both account balance + changes. -root@b127d8503f24 /tmp # ./deposit.sh -"0xa162184cbc0515d7d4a65eb4341429ac765a95c06bac3496040c1840b7c6065a" +For manual use, run the following commands from this example directory: + +```bash +python3 ethereum_pos.py [amd|arm] +cd output +docker compose up ``` -# 2. Introduction to PoS +Arguments: + +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. +- `--platform amd|arm`: Standard test-framework platform argument. +- `--output DIR`: Standard test-framework output directory argument. +- `--ether-view`: Enable Eth Explorer output. This is the default for manual + runs. +- `--no-ether-view`: Disable Eth Explorer output. The test manifest uses this + option for CI. + +Startup may take some time. The Geth nodes, Beacon nodes, Validator +Clients, Beacon Setup node, Faucet, Utility Server, Internet Map, and +Eth Explorer all need to initialize. + + +## Access Points + +After the emulator starts, the following services are commonly used: + +| Service | Default URL | Function | +| --- | --- | --- | +| Internet Map | `http://127.0.0.1:8080/map.html` | Shows the emulated Internet topology | +| Eth Explorer | `http://127.0.0.1:5000` | Shows the Ethereum PoS private chain | +| Geth HTTP RPC | `http://:8545` | Allows tools and wallets to interact with the execution layer | +| Beacon API | `http://:8000` | Allows tools to inspect consensus-layer state | + +The exact Geth and Beacon node IP addresses can be found from +`docker compose ps`, the Internet Map, or the generated Docker +Compose file. + -A consensus mechanism is a system that blockchains use to validate the authenticity of transactions and maintain the security of the underlying blockchain. As one of the features of the blockchain, anyone can participate in maintaining block, which is called mining in proof-of-work and is called validating in proof-of-stake. However, the open membership can cause a security issue such as Sybil attacks. To prevent this attack and make the blockchain system more reliable, a participant should show some effort. +## Relationship with Other Examples -In proof-of-work consensus protocol, a participant needs to show computational efforts, which consumes a lot of energy. To mine a block, a miner should calculate the hash that satisfies a particular condition and do it faster than any other miners. And this consensus requires a computer with high performance and the waste of power was severe. The competitors in the world runs countless machines and consumes a lot of electricity to mine one block. Only one miner per one block can get rewards and the efforts from the other miners can only produce a waste of the power. This tremendous dissipation can cause the problem in eco system. That is why Ethereum has changed its consensus protocol to proof-of-stake from proof-of-work recently. +This example is the base PoS example. Other examples build on top of +it: -In proof-of-stake consensus protocol, a participant should stake a promised amount of money to be involved in maintaining blockchains. By staking some amount of money, a staker holds qualification to generate a block. The power dissipation problem is solved as only one staker (called validator instead of miner in proof-of-stake) will validate (called mine in proof-of-stake) a block and link it to the blockchain at each time. There is no unnecessary power consumption anymore. +- [D10_eth_explorer](../D10_eth_explorer/) focuses on the Eth Explorer. +- [D20_faucet](../D20_faucet/) demonstrates account funding through the + Faucet server. +- [D21_deploy_contract](../D21_deploy_contract/) demonstrates smart + contract deployment through the Utility server. +- [D23_validator](../D23_validator/) demonstrates adding a validator + after the chain has started. diff --git a/examples/blockchain/D01_ethereum_pos/ethereum_pos.py b/examples/blockchain/D01_ethereum_pos/ethereum_pos.py index c11e949ac..4b2aae100 100755 --- a/examples/blockchain/D01_ethereum_pos/ethereum_pos.py +++ b/examples/blockchain/D01_ethereum_pos/ethereum_pos.py @@ -1,105 +1,220 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu import * -import sys, os - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -# Create Emulator Base with 10 Stub AS (150-154, 160-164) using Makers utility method. -# hosts_per_stub_as=3 : create 3 hosts per one stub AS. -# It will create hosts(physical node) named `host_{}`.format(counter), counter starts from 0. -hosts_per_stub_as = 3 -emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as = hosts_per_stub_as) - -# Create the Ethereum layer -eth = EthereumService() - -# Create the Blockchain layer which is a sub-layer of Ethereum layer. -# chainName="pos": set the blockchain name as "pos" -# consensus="ConsensusMechnaism.POS" : set the consensus of the blockchain as "ConsensusMechanism.POS". -# supported consensus option: ConsensusMechanism.POA, ConsensusMechanism.POW, ConsensusMechanism.POS -blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) - - -asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] - -################################################### -# Ethereum Node - -i = 1 -for asn in asns: - for id in range(hosts_per_stub_as): - # Create a blockchain virtual node named "eth{}".format(i) - e:EthereumServer = blockchain.createNode("eth{}".format(i)) - - # Create Docker Container Label named 'Ethereum-POS-i' - e.appendClassName('Ethereum-POS-{}'.format(i)) - - # Enable Geth to communicate with geth node via http - e.enableGethHttp() - - # Set host in asn 150 with id 0 (ip : 10.150.0.71) as BeaconSetupNode. - if asn == 150 and id == 0: - e.setBeaconSetupNode() - - # Set host in asn 150 with id 1 (ip : 10.150.0.72) as BootNode. - # This node will serve as a BootNode in both execution layer (geth) and consensus layer (lighthouse). - if asn == 150 and id == 1: - e.setBootNode(True) - - # Set hosts in asn 152 and 162 with id 0 and 1 as validator node. - # Validator is added by deposit 32 Ethereum and is activated in realtime after the Merge. - # isManual=True : deposit 32 Ethereum by manual. - # Other than deposit part, create validator key and running a validator node is done by codes. - if asn in [151]: - if id == 0: - e.enablePOSValidatorAtRunning() - if id == 1: - e.enablePOSValidatorAtRunning(is_manual=True) - - # Set hosts in asn 152, 153, 154, and 160 as validator node. - # These validators are activated by default from genesis status. - # Before the Merge, when the consensus in this blockchain is still POA, - # these hosts will be the signer nodes. - if asn in [152,153,154,160,161,162,163,164]: - e.enablePOSValidatorAtGenesis() - - # Customizing the display names (for visualiztion purpose) - if e.isBeaconSetupNode(): - emu.getVirtualNode('eth{}'.format(i)).setDisplayName('Ethereum-BeaconSetup') - else: - emu.getVirtualNode('eth{}'.format(i)).setDisplayName('Ethereum-POS-{}'.format(i)) - - # Binding the virtual node to the physical node. - emu.addBinding(Binding('eth{}'.format(i), filter=Filter(asn=asn, nodeName='host_{}'.format(id)))) - - i = i+1 - - -# Add layer to the emulator -emu.addLayer(eth) - -emu.render() - -# Enable internetMap -# Enable etherView -docker = Docker(internetMapEnabled=True, etherViewEnabled=True, platform=platform) - -emu.compile(docker, './output', override = True) + + +DOMAIN = ".net" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D01 Ethereum PoS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--beacon-nodes", + type=int, + default=3, + help="number of geth/beacon node pairs", + ) + parser.add_argument( + "--validators-per-beacon", + type=int, + default=3, + help="validator clients per beacon node", + ) + parser.add_argument( + "--ether-view", + dest="ether_view_enabled", + action="store_true", + default=True, + help="enable Eth Explorer output in generated Docker files (default)", + ) + parser.add_argument( + "--no-ether-view", + dest="ether_view_enabled", + action="store_false", + help="disable Eth Explorer output; useful for CI builds", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + if args.beacon_nodes < 1: + parser.error("--beacon-nodes must be >= 1") + if args.validators_per_beacon < 1: + parser.error("--validators-per-beacon must be >= 1") + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator(total_beacon_nodes: int = 3, vc_per_beacon: int = 3) -> Emulator: + ############################################################################### + # Configure the number of nodes + geth_node_number = total_beacon_nodes + beacon_node_number = total_beacon_nodes + vc_node_number = vc_per_beacon * beacon_node_number + # Create Emulator Base with 10 Stub AS (150-154, 160-164) using Makers utility method. + # hosts_per_stub_as=3 : create 3 hosts per one stub AS. + # It will create hosts(physical node) named `host_{}`.format(counter), counter starts from 0. + hosts_per_stub_as = 3 + emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as = hosts_per_stub_as) + + # Create the Ethereum layer + eth = EthereumService() + # Create the Blockchain layer which is a sub-layer of Ethereum layer. + # chainName="pos": set the blockchain name as "pos" + # consensus="ConsensusMechnaism.POS" : set the consensus of the blockchain as "ConsensusMechanism.POS". + # supported consensus option: ConsensusMechanism.POA, ConsensusMechanism.POW, ConsensusMechanism.POS + blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) + # Generate a list of wallet-compatible local accounts and prefund them. + accounts_total = 10 + pre_funded_amount = 1000000 + mnemonic = "gentle always fun glass foster produce north tail security list example gain" + blockchain.addLocalAccountsFromMnemonic( + mnemonic=mnemonic, + total=accounts_total, + balance=pre_funded_amount, + ) + asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] + geth_nodes: List[PoSGethServer] = [] + beacon_nodes: List[PoSBeaconServer] = [] + vc_nodes: List[PoSVcServer] =[] + # Create the BeaconSetupNode + beaconsetupServer: PoSBeaconSetupServer = blockchain.createBeaconSetupNode(f"BeaconSetupNode") + emu.getVirtualNode(f'BeaconSetupNode').setDisplayName('Ethereum-POS-BeaconSetup') + for i in range(geth_node_number): + gethServer: PoSGethServer = blockchain.createGethNode(f"gethnode{i}") + gethServer.enableGethHttp() + gethServer.appendClassName(f'Ethereum-POS-Geth-{i+1}') + gethServer.addHostName(f"gethnode{i}" + DOMAIN) + geth_nodes.append(gethServer) + emu.getVirtualNode(f'gethnode{i}').setDisplayName(f'Ethereum-POS-Geth-{i+1}') + + # Create Beacon nodes and connect to Geth nodes + for i in range(beacon_node_number): + beaconServer: PoSBeaconServer = blockchain.createBeaconNode(f"beaconnode{i}") + beaconServer.appendClassName(f'Ethereum-POS-Beacon-{i+1}') + beaconServer.connectToGethNode(f"gethnode{(i+1)%len(geth_nodes)}") + beacon_nodes.append(beaconServer) + emu.getVirtualNode(f'beaconnode{i}').setDisplayName(f'Ethereum-POS-Beacon-{i+1}') + + # Set boot nodes + geth_nodes[0].setBootNode(True) + beacon_nodes[0].setBootNode(True) + + # Create Validator nodes and connect to Beacon nodes + for i in range(vc_node_number): + VcServer: PoSVcServer=blockchain.createVcNode(f"vcnode{i}") + VcServer.appendClassName(f'Ethereum-POS-Validator-{i+1}') + VcServer.connectToBeaconNode(f"beaconnode{(i+1)%len(beacon_nodes)}") + VcServer.enablePOSValidatorAtGenesis() + vc_nodes.append(VcServer) + emu.getVirtualNode(f'vcnode{i}').setDisplayName(f'Ethereum-POS-Validator-{i+1}') + + faucet_geth_node = "gethnode{}".format(1 if geth_node_number > 1 else 0) + utility_geth_node = "gethnode{}".format(2 if geth_node_number > 2 else geth_node_number - 1) + + # Create the Faucet server + faucet:FaucetServer = blockchain.createFaucetServer( + vnode='faucet', + port=80, + linked_eth_node=faucet_geth_node, + balance=10000, + max_fund_amount=10) + faucet.setDisplayName('Faucet') + faucet.addHostName('faucet' + DOMAIN) + # Create the Utility server + util_server:EthUtilityServer = blockchain.createEthUtilityServer( + vnode='utility', + port=5000, + linked_eth_node=utility_geth_node, + linked_faucet_node='faucet') + util_server.setDisplayName('UtilityServer') + util_server.addHostName('utility' + DOMAIN) + + # Add Ethereum service to the emulator + emu.addLayer(eth) + + # Bind all virtual servers (including faucet/utility) to physical hosts + for _, servers in blockchain.getAllServerNames().items(): + for server in servers: + emu.addBinding(Binding(server, filter=Filter(nodeName="host_*"), + action=Action.FIRST)) + + # Add /etc/hosts layer + emu.addLayer(EtcHosts()) + + # Configure IP range for each AS + base_layer = emu.getLayer('Base') + for asn in asns: + as_obj = base_layer.getAutonomousSystem(asn) + net = as_obj.getNetwork('net0') + net.setHostIpRange(hostStart=71, hostEnd=199, hostStep=1) + + return emu + + +def run( + dumpfile=None, + total_beacon_nodes: int = 3, + vc_per_beacon: int = 3, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + ether_view_enabled: bool = True, +): + emu = build_emulator(total_beacon_nodes=total_beacon_nodes, vc_per_beacon=vc_per_beacon) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + docker = Docker( + internetMapEnabled=True, + etherViewEnabled=ether_view_enabled, + platform=platform, + ) + emu.compile(docker, str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + total_beacon_nodes=args.beacon_nodes, + vc_per_beacon=args.validators_per_beacon, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ether_view_enabled=args.ether_view_enabled, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D01_ethereum_pos/example.yaml b/examples/blockchain/D01_ethereum_pos/example.yaml new file mode 100644 index 000000000..029e50bcd --- /dev/null +++ b/examples/blockchain/D01_ethereum_pos/example.yaml @@ -0,0 +1,131 @@ +id: blockchain-d01-ethereum-pos +name: Ethereum PoS Private Chain +description: A private Ethereum proof-of-stake chain with Geth execution nodes, Lighthouse beacon nodes, genesis validators, Faucet, and Utility services. +runner: blockchain +script: ethereum_pos.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - ethereum-transaction + +compile: + enabled: true + output: output + args: + - --no-ether-view + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D01 Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D01 Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D01 genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D01 beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D01 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D01 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + +probes: + - name: Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: Beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: Geth IPC exposes a funded local account + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: "geth attach --exec 'eth.accounts.length > 0' | grep -q true" + retries: 30 + interval: 3 + + - name: PoS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + +test_programs: + - name: D01 Ethereum transaction validation + script: test_runtime.py + timeout: 900 diff --git a/examples/blockchain/D01_ethereum_pos/test_runtime.py b/examples/blockchain/D01_ethereum_pos/test_runtime.py new file mode 100644 index 000000000..0f06f7599 --- /dev/null +++ b/examples/blockchain/D01_ethereum_pos/test_runtime.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import EthereumRuntimeTest + + +def main() -> int: + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D01 has a Geth node for the transaction test", + minimum_count=1, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + + if geth_nodes: + test.send_ether_and_verify( + "A signed ETH transfer is included and changes balances", + geth_nodes[0], + value_wei=10**18, + ) + + test.write_summary("d01-ethereum-pos-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D10_eth_explorer/README.md b/examples/blockchain/D10_eth_explorer/README.md new file mode 100644 index 000000000..54dfba263 --- /dev/null +++ b/examples/blockchain/D10_eth_explorer/README.md @@ -0,0 +1,178 @@ +# D10 Eth Explorer Example + +This example demonstrates how to build an Ethereum PoS private network in SEED Emulator and automatically attach a blockchain explorer visualization component to it. The visualization is adapted from the open-source projects [eth2-beaconchain-explorer](https://github.com/gobitfly/eth2-beaconchain-explorer.git) and [beaconchain](https://github.com/gobitfly/beaconchain.git). The related Docker build files are located in [`docker_images/seedemu-ethexplorer`](../../../docker_images/seedemu-ethexplorer). + +Unlike the Internet Map, which focuses on containers, ASes, network links, and topology, the Eth Explorer connects to the Geth execution-layer node and the Beacon consensus-layer node inside the emulated network. It indexes on-chain data and presents blocks, epochs, slots, validators, and execution-layer transactions through a web UI. With both visualizations enabled, the example lets users inspect both the emulated Internet topology and the Ethereum PoS chain state. + +## Example Goals + +`ethereum_pos.py` performs the following tasks: + +- Creates a base Internet topology with 10 stub ASes: `150-154` and `160-164`. +- Creates an Ethereum PoS node set based on the command-line argument. +- Creates Geth nodes, Beacon nodes, and Validator Client nodes for the PoS network. +- Creates a `BeaconSetupNode` to generate Beacon chain configuration, validator keys, genesis data, and other bootstrapping materials. +- Configures bootnodes for both the execution layer and the consensus layer. +- Enables HTTP RPC on Geth nodes so the explorer backend can read execution-layer data. +- Enables both `internetMapEnabled=True` and `etherViewEnabled=True` when compiling the Docker output. + +The first command-line argument controls the node scale. If the argument is `N`, the script creates: + +| Type | Count | Purpose | +| --- | --- | --- | +| Geth nodes | `N` | Ethereum execution-layer nodes that provide blocks, transactions, account state, and JSON-RPC APIs | +| Beacon nodes | `N` | Ethereum consensus-layer nodes that maintain Beacon Chain state, slots, epochs, attestations, and related consensus data | +| Validator Client nodes | `3 * N` | Validator clients that connect to Beacon nodes and participate in PoS validation | +| BeaconSetupNode | `1` | Generates and distributes configuration and keys required to start the PoS private chain | + +The script automatically calculates how many hosts each stub AS needs and binds the virtual nodes to those hosts. To support larger deployments, it also expands the host IP range of `net0` in each stub AS. + +## How to Run + +Run the following commands from the project root: + +```bash +cd examples/blockchain/D10_eth_explorer +python ethereum_pos.py [amd|arm] +``` + +Example: + +```bash +python ethereum_pos.py 3 amd +``` + +Arguments: + +- ``: The number of Beacon/Geth nodes to create. This also affects the number of Validator Client nodes. +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. + +After the script finishes, it generates an `output` directory in the current example directory. Start the emulation from that directory: + +```bash +cd output +docker compose up +``` +When the simulator generates the docker-compose.yml, it will select a Beacon node (`CL_HOST`) by filtering based on the container name containing `"-Beacon-"`, and select a Geth node (`EL_HOST`) by filtering based on `"-Geth-"`. +Startup may take some time. The explorer backend waits for the databases, ClickHouse, Redis, Bigtable emulator services, and Geth/Beacon nodes to become available. It then initializes schemas, reads the chain genesis information, and starts indexing data. + +## Access Points + +The example compiles the Docker output with: + +```python +docker = Docker(internetMapEnabled=True, etherViewEnabled=True, platform=platform) +``` + +After startup, two main visualization entry points are available: + +| Service | Default URL | Function | +| --- | --- | --- | +| Internet Map | `http://127.0.0.1:8080/map.html` | Shows the SEED Emulator network topology, ASes, hosts, containers, and links | +| Eth Explorer | `http://127.0.0.1:5000` | Shows the Ethereum PoS private chain through a blockchain explorer UI | + +If a local port is already in use, adjust the port through the `Docker` constructor in `ethereum_pos.py`, for example with `internetMapPort` or `etherViewPort`. + +## Visualization Architecture + +The Eth Explorer services are automatically injected into the generated `docker-compose.yml` by the Docker compiler. The core services are: + +| Service | Purpose | +| --- | --- | +| `seedemu-ethexplorer-web` | Built from `eth2-beaconchain-explorer`; provides the web UI, search, validator tagging, and frontend data updates | +| `seedemu-ethexplorer-backend` | Built from `beaconchain/backend`; handles execution-layer and consensus-layer indexing, statistics, and schema initialization | +| `postgres` | Stores explorer business data and indexed results | +| `clickhouse` | Stores analytical chain data for efficient statistical queries | +| `redis` | Provides cache and session storage | +| `littlebigtable` / `rawbigtable` | Emulate Bigtable storage to preserve compatibility with the original projects' data access model | + +The compiler looks for nodes whose display names contain `Geth-` and `Beacon-` in the emulated topology. The first matched nodes are used as the explorer data sources and passed to the explorer through environment variables: + +- `EL_HOST`: The execution-layer Geth node address. +- `CL_HOST`: The consensus-layer Beacon node address. + +The explorer containers are attached to `beacon-network` and to the relevant emulated networks, so they can access both the database services and the Geth/Beacon nodes inside the emulation. + +## Docker Image Layout + +The customized image files are located in [`docker_images/seedemu-ethexplorer`](../../../docker_images/seedemu-ethexplorer): + +```text +docker_images/seedemu-ethexplorer +├── docker-compose.yml +├── beaconchain/backend +│ ├── Dockerfile +│ ├── start.sh +│ └── local_deployment/ +└── eth2-beaconchain-explorer + ├── Dockerfile + ├── start.sh + └── local-deployment/ +``` + +Key files: + +- `eth2-beaconchain-explorer/Dockerfile` builds `explorer`, `validator-tagger`, and `frontend-data-updater`. +- `eth2-beaconchain-explorer/start.sh` generates the runtime configuration and starts the web service and frontend data update tasks. +- `beaconchain/backend/Dockerfile` builds the `bc` backend command. +- `beaconchain/backend/start.sh` initializes Bigtable, Postgres, and ClickHouse schemas, then starts `eth1indexer`, `rewards-exporter`, `statistics`, and other backend tasks. +- The two `provision-explorer-config-custom.sh` scripts dynamically generate `config.yml` from the emulated chain, including the genesis timestamp, genesis validators root, chain ID, network ID, Geth RPC address, and Beacon API address. + +These images are referenced by the SEED Emulator Docker compiler as: + +```python +SEEDEMU_ETH_EXPLORER_BACKEND_IMAGE = "handsonsecurity/seedemu-ethexplorer-backend:1.0" +SEEDEMU_ETH_EXPLORER_WEB_IMAGE = "handsonsecurity/seedemu-ethexplorer-web:1.0" +``` + +## Explorer Features + +The Eth Explorer UI is based on the open-source Beacon Chain Explorer and is configured for the local test chain. Common features include: + +- Viewing the overall chain status, such as the current epoch, slot, finalized/checkpoint state, and participation rate. +- Inspecting Beacon Chain slots and blocks, including proposer, block root, state root, attestations, and other consensus-layer data. +- Viewing the validator list and individual validator details, including activation status, balance, proposal activity, and attestation performance. +- Inspecting execution-layer blocks and transactions to understand data produced and synchronized by Geth nodes. +- Searching by slot, epoch, block, validator, and other chain objects. +- Viewing aggregated statistics, such as validator, block, reward, and chain activity data. + +Immediately after the emulation starts, the explorer may not show complete data. This is expected while Geth, Beacon nodes, database schema initialization, and indexing tasks are still starting. Wait for several slots or check the `seedemu_ethexplorer_backend` and `seedemu_ethexplorer_web` logs to confirm indexing progress. + +## Relationship with Internet Map + +This example enables two visualization layers: + +- Internet Map focuses on the network emulation layer. It is useful for observing AS topology, host placement, container status, and network connectivity. +- Eth Explorer focuses on the blockchain application layer. It is useful for observing execution-layer and consensus-layer data of the PoS private chain. + +Using both together helps debug experiments from both the network and blockchain perspectives. For example, if a Beacon node appears abnormal in the explorer, users can switch to the Internet Map to inspect the AS, host state, and network connectivity of that node. + +## Troubleshooting + +### Cannot access `http://127.0.0.1:5000` + +First check whether `output/docker-compose.yml` contains `seedemu-ethexplorer-web`, and confirm that the containers are running: + +```bash +docker compose ps +``` + +If port `5000` is already occupied, update `Docker(etherViewPort=...)` and regenerate the output. + +### The explorer starts but shows no chain data + +This is common during the early stage of a local private-chain startup. The explorer needs to wait for: + +- Geth HTTP RPC to become available. +- Beacon API to become available. +- Genesis information to be readable. +- Postgres, ClickHouse, Redis, and Bigtable emulator services to become ready. +- Backend indexing tasks to finish initialization and begin writing data. + +Check logs to locate startup or indexing issues: + +```bash +docker compose logs -f seedemu-ethexplorer-backend +docker compose logs -f seedemu-ethexplorer-web +``` diff --git a/examples/blockchain/D10_eth_explorer/ethereum_pos.py b/examples/blockchain/D10_eth_explorer/ethereum_pos.py new file mode 100644 index 000000000..8d373f7ea --- /dev/null +++ b/examples/blockchain/D10_eth_explorer/ethereum_pos.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu import Platform +from examples.blockchain.D01_ethereum_pos import ethereum_pos as d01_ethereum_pos + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D10 Ethereum PoS EthExplorer example.") + parser.add_argument( + "legacy_args", + nargs="*", + help="legacy form: [amd|arm]", + ) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--beacon-nodes", + type=int, + default=3, + help="number of geth/beacon node pairs", + ) + parser.add_argument( + "--validators-per-beacon", + type=int, + default=3, + help="validator clients per beacon node", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + + if len(args.legacy_args) > 2: + parser.error("legacy arguments must be: [amd|arm]") + + legacy_platform = None + if args.legacy_args: + first = args.legacy_args[0].lower() + if first in ("amd", "arm"): + legacy_platform = first + else: + try: + args.beacon_nodes = int(first) + except ValueError: + parser.error("legacy beacon node count must be an integer") + + if len(args.legacy_args) == 2: + legacy_platform = args.legacy_args[1].lower() + if legacy_platform not in ("amd", "arm"): + parser.error("legacy platform must be amd or arm") + + args.platform = args.platform or legacy_platform or "amd" + if args.beacon_nodes < 1: + parser.error("--beacon-nodes must be >= 1") + if args.validators_per_beacon < 1: + parser.error("--validators-per-beacon must be >= 1") + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def run( + dumpfile=None, + total_beacon_nodes: int = 3, + vc_per_beacon: int = 3, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, +): + d01_ethereum_pos.run( + dumpfile=dumpfile, + total_beacon_nodes=total_beacon_nodes, + vc_per_beacon=vc_per_beacon, + output=output, + platform=platform, + override=override, + render=render, + ether_view_enabled=True, + ) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + total_beacon_nodes=args.beacon_nodes, + vc_per_beacon=args.validators_per_beacon, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D10_eth_explorer/example.yaml b/examples/blockchain/D10_eth_explorer/example.yaml new file mode 100644 index 000000000..d981c249d --- /dev/null +++ b/examples/blockchain/D10_eth_explorer/example.yaml @@ -0,0 +1,167 @@ +id: blockchain-d10-eth-explorer +name: Ethereum PoS Private Chain with EthExplorer +description: A D01-style Ethereum proof-of-stake private chain with the EthExplorer visualization enabled and tested. +runner: blockchain +script: ethereum_pos.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - ethereum-transaction + - eth-explorer + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D10 Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D10 Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D10 genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D10 beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D10 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D10 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + + - name: D10 EthExplorer services are running + type: compose-ps + services: + - clickhouse + - postgres + - alloy + - redis + - littlebigtable + - rawbigtable + - seedemu-ethexplorer-backend + - seedemu-ethexplorer-web + retries: 90 + interval: 5 + + + - name: D10 EthExplorer homepage is ready + type: http + url: http://127.0.0.1:5000/ + retries: 90 + interval: 5 + timeout: 10 + +probes: + - name: D10 Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: D10 Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: D10 Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: D10 beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: D10 Geth IPC exposes a funded local account + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: "geth attach --exec 'eth.accounts.length > 0' | grep -q true" + retries: 30 + interval: 3 + + - name: D10 PoS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + + - name: D10 EthExplorer API docs are reachable + type: http + url: http://127.0.0.1:5000/api/v1/docs + retries: 30 + interval: 5 + timeout: 10 + + - name: D10 EthExplorer latest-state API is reachable + type: http + url: http://127.0.0.1:5000/api/v1/latestState + expect_body_regex: "(?i)(slot|epoch|head|finalized)" + retries: 90 + interval: 5 + timeout: 10 + +test_programs: + - name: D10 Ethereum transaction and EthExplorer validation + script: test_runtime.py + timeout: 1500 diff --git a/examples/blockchain/D10_eth_explorer/test_runtime.py b/examples/blockchain/D10_eth_explorer/test_runtime.py new file mode 100644 index 000000000..9e33a9f92 --- /dev/null +++ b/examples/blockchain/D10_eth_explorer/test_runtime.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Callable, Dict, Iterable, Optional + +from seedemu.testing import EthereumRuntimeTest + + +EXPLORER_URL = os.environ.get("D10_ETH_EXPLORER_URL", "http://127.0.0.1:5000").rstrip("/") +EXPLORER_SERVICE = "seedemu-ethexplorer-web" + + +def record(test: EthereumRuntimeTest, name: str, command: str, ok: bool, stdout: str = "", stderr: str = "") -> None: + test.results.append( + { + "name": name, + "service": EXPLORER_SERVICE, + "command": command, + "exit": 0 if ok else 1, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if ok else "failed", + } + ) + + +def fetch(path: str, *, method: str = "GET", data: Optional[bytes] = None, timeout: int = 10) -> tuple[int, str]: + url = EXPLORER_URL + path + request = urllib.request.Request(url, data=data, method=method) + if data is not None: + request.add_header("Content-Type", "application/x-www-form-urlencoded") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + body = response.read().decode("utf-8", errors="replace") + return response.status, body + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + return exc.code, body + + +def wait_for_http( + test: EthereumRuntimeTest, + name: str, + path: str, + *, + predicate: Optional[Callable[[str], bool]] = None, + method: str = "GET", + data: Optional[bytes] = None, + retries: int = 90, + interval: int = 5, + timeout: int = 10, +) -> Optional[str]: + last = "" + command = f"HTTP {method} {EXPLORER_URL}{path}" + for attempt in range(1, retries + 1): + try: + status, body = fetch(path, method=method, data=data, timeout=timeout) + except Exception as exc: + last = str(exc) + else: + last = f"status={status} body={body[:500]}" + if 200 <= status < 300 and (predicate is None or predicate(body)): + record(test, name, command, True, f"attempts={attempt} {last}") + return body + if attempt < retries: + time.sleep(interval) + record(test, name, command, False, last) + return None + + +def wait_for_any_http( + test: EthereumRuntimeTest, + name: str, + paths: Iterable[str], + *, + predicate: Optional[Callable[[str], bool]] = None, + retries: int = 60, + interval: int = 5, + timeout: int = 10, +) -> Optional[tuple[str, str]]: + candidates = list(paths) + last = "" + command = "HTTP GET " + ", ".join(EXPLORER_URL + path for path in candidates) + for attempt in range(1, retries + 1): + for path in candidates: + try: + status, body = fetch(path, timeout=timeout) + except Exception as exc: + last = f"{path}: {exc}" + continue + last = f"{path}: status={status} body={body[:500]}" + if 200 <= status < 300 and (predicate is None or predicate(body)): + record(test, name, command, True, f"attempts={attempt} path={path} body={body[:500]}") + return path, body + if attempt < retries: + time.sleep(interval) + record(test, name, command, False, last) + return None + + +def parse_json_body(body: Optional[str]) -> Optional[Any]: + if body is None: + return None + try: + return json.loads(body) + except json.JSONDecodeError: + return None + + +def int_from_value(value: object) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str): + text = value.strip() + if text.startswith(("0x", "0X")): + try: + return int(text, 16) + except ValueError: + return None + if text.isdigit(): + return int(text) + return None + + +def find_int_by_key(data: Any, key_names: tuple[str, ...], contains: str) -> Optional[int]: + if isinstance(data, dict): + for key, value in data.items(): + key_text = str(key).lower() + if key_text in key_names: + parsed = int_from_value(value) + if parsed is not None: + return parsed + for key, value in data.items(): + key_text = str(key).lower() + if contains in key_text: + parsed = int_from_value(value) + if parsed is not None: + return parsed + found = find_int_by_key(value, key_names, contains) + if found is not None: + return found + elif isinstance(data, list): + for item in data: + found = find_int_by_key(item, key_names, contains) + if found is not None: + return found + return None + + +def body_has_any(*needles: str) -> Callable[[str], bool]: + lowered = [needle.lower() for needle in needles] + + def predicate(body: str) -> bool: + text = body.lower() + return any(needle in text for needle in lowered) + + return predicate + + +def body_has_text(text: str) -> Callable[[str], bool]: + text_lower = text.lower() + + def predicate(body: str) -> bool: + return text_lower in body.lower() + + return predicate + + +def datatable_query() -> str: + return urllib.parse.urlencode({"draw": 1, "start": 0, "length": 10}) + + +def main() -> int: + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D10 has Geth nodes for the transaction and explorer tests", + expected_count=3, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + test.require_ethereum_services( + "D10 has Lighthouse beacon nodes for EthExplorer", + expected_count=3, + class_contains="Ethereum-POS-Beacon", + consensus="POS", + ) + test.require_ethereum_services( + "D10 has genesis validator clients for EthExplorer", + expected_count=9, + class_contains="Ethereum-POS-Validator", + role="validator_at_genesis", + consensus="POS", + ) + + tx_summary: Dict[str, Any] = {} + if geth_nodes: + tx_result = test.send_ether_and_verify( + "A signed ETH transfer is included and changes balances", + geth_nodes[0], + value_wei=10**18, + ) + if tx_result["status"] == "passed": + try: + tx_summary = json.loads(str(tx_result["stdout"])) + record( + test, + "D10 signed transaction fixture is available for EthExplorer checks", + "send Ethereum transaction and parse transaction summary", + True, + json.dumps(tx_summary, sort_keys=True), + ) + except json.JSONDecodeError as exc: + record(test, "D10 transaction summary is parseable", "parse transaction summary", False, str(exc)) + + latest_body = wait_for_http( + test, + "EthExplorer latest-state API exposes chain status", + "/api/v1/latestState", + predicate=body_has_any("slot", "epoch", "head", "finalized"), + retries=120, + interval=5, + ) + latest_data = parse_json_body(latest_body) + latest_slot = find_int_by_key( + latest_data, + ("slot", "currentslot", "headslot", "latestslot", "finalizedslot"), + "slot", + ) + latest_epoch = find_int_by_key( + latest_data, + ("epoch", "currentepoch", "latestepoch", "finalizedepoch"), + "epoch", + ) + + if latest_slot is None: + latest_slot = 1 + record(test, "EthExplorer latest-state slot is parseable", "parse /api/v1/latestState slot", False, latest_body or "") + else: + record(test, "EthExplorer latest-state slot is parseable", "parse /api/v1/latestState slot", True, str(latest_slot)) + if latest_epoch is None: + latest_epoch = max(latest_slot // 32, 0) + record(test, "EthExplorer latest-state epoch fallback is available", "derive epoch from slot", True, str(latest_epoch)) + else: + record(test, "EthExplorer latest-state epoch is parseable", "parse /api/v1/latestState epoch", True, str(latest_epoch)) + + slot_candidates = [] + for candidate in (latest_slot, latest_slot - 1, 1, 0): + if candidate >= 0 and candidate not in slot_candidates: + slot_candidates.append(candidate) + slot_result = wait_for_any_http( + test, + "EthExplorer consensus slot API returns indexed slot data", + [f"/api/v1/slot/{slot}" for slot in slot_candidates], + predicate=body_has_any("slot", "block", "proposer", "epoch"), + retries=120, + interval=5, + ) + indexed_slot = slot_candidates[0] + if slot_result is not None: + indexed_slot = int(slot_result[0].rsplit("/", 1)[-1]) + + wait_for_http( + test, + "EthExplorer consensus slot UI renders indexed slot", + f"/slot/{indexed_slot}", + predicate=body_has_any("slot", "block", str(indexed_slot)), + retries=60, + interval=5, + ) + wait_for_http( + test, + "EthExplorer epoch API returns epoch data", + f"/api/v1/epoch/{latest_epoch}", + predicate=body_has_any("epoch", "validators", "blocks", str(latest_epoch)), + retries=90, + interval=5, + ) + wait_for_http( + test, + "EthExplorer epoch UI renders epoch page", + f"/epoch/{latest_epoch}", + predicate=body_has_any("epoch", str(latest_epoch)), + retries=60, + interval=5, + ) + + wait_for_http( + test, + "EthExplorer validator UI renders validator 0", + "/validator/0", + predicate=body_has_any("validator", "pubkey", "balance", "status"), + retries=90, + interval=5, + ) + wait_for_http( + test, + "EthExplorer validators list UI renders", + "/validators", + predicate=body_has_any("validator", "validators"), + retries=60, + interval=5, + ) + wait_for_http( + test, + "EthExplorer validators data endpoint returns table data", + "/validators/data?" + datatable_query(), + predicate=body_has_any("data", "validator", "pubkey"), + retries=90, + interval=5, + ) + + wait_for_http( + test, + "EthExplorer slots list data endpoint returns table data", + "/slots/data?" + datatable_query(), + predicate=body_has_any("data", "slot", "epoch"), + retries=90, + interval=5, + ) + wait_for_http( + test, + "EthExplorer index data endpoint returns chain activity", + "/index/data", + predicate=body_has_any("slot", "epoch", "block", "validator"), + retries=90, + interval=5, + ) + + tx_hash = str(tx_summary.get("tx_hash", "")) + block_number = tx_summary.get("block_number") + if block_number is None: + record(test, "D10 transaction block number is available for explorer lookup", "read transaction summary", False, json.dumps(tx_summary)) + else: + block_text = str(block_number) + wait_for_http( + test, + "EthExplorer execution block API returns transaction block", + f"/api/v1/execution/block/{block_text}", + predicate=body_has_any("block", block_text), + retries=120, + interval=5, + ) + wait_for_http( + test, + "EthExplorer execution block UI renders transaction block", + f"/block/{block_text}", + predicate=body_has_any("block", block_text), + retries=90, + interval=5, + ) + if tx_hash: + wait_for_http( + test, + "EthExplorer execution block transactions endpoint indexes the signed transfer", + f"/block/{block_text}/transactions?" + datatable_query(), + predicate=body_has_any(tx_hash, tx_hash.removeprefix("0x")), + retries=120, + interval=5, + ) + wait_for_http( + test, + "EthExplorer execution blocks list UI renders", + "/blocks", + predicate=body_has_any("block", "blocks"), + retries=60, + interval=5, + ) + wait_for_http( + test, + "EthExplorer execution blocks data endpoint returns table data", + "/blocks/data?" + datatable_query(), + predicate=body_has_any("data", "block"), + retries=90, + interval=5, + ) + + if tx_hash: + wait_for_http( + test, + "EthExplorer transaction UI renders the signed transfer", + f"/tx/{tx_hash}", + predicate=body_has_text(tx_hash), + retries=120, + interval=5, + ) + wait_for_http( + test, + "EthExplorer transactions data endpoint indexes the signed transfer", + "/transactions/data?" + datatable_query(), + predicate=body_has_any(tx_hash, tx_hash.removeprefix("0x")), + retries=120, + interval=5, + ) + search_data = urllib.parse.urlencode({"search": tx_hash}).encode("utf-8") + wait_for_http( + test, + "EthExplorer search form can find the signed transfer", + "/search", + method="POST", + data=search_data, + predicate=body_has_any(tx_hash, tx_hash.removeprefix("0x"), "transaction"), + retries=30, + interval=5, + ) + else: + record(test, "D10 transaction hash is available for explorer lookup", "read transaction summary", False, json.dumps(tx_summary)) + + wait_for_http( + test, + "EthExplorer transactions list UI renders", + "/transactions", + predicate=body_has_any("transaction", "transactions"), + retries=60, + interval=5, + ) + wait_for_http( + test, + "EthExplorer transactions data endpoint returns table data", + "/transactions/data?" + datatable_query(), + predicate=body_has_any("data", "transaction", "hash"), + retries=90, + interval=5, + ) + + test.write_summary("d10-eth-explorer-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D20_faucet/README.md b/examples/blockchain/D20_faucet/README.md index 60801fa28..9af7a5b55 100644 --- a/examples/blockchain/D20_faucet/README.md +++ b/examples/blockchain/D20_faucet/README.md @@ -1,55 +1,110 @@ # Faucet Server -Before sending a transaction to the blockchain, the user's account needs -to have some fund. In most test nets, a faucet server is provided to -fund user's accounts after receiving requests. We have implemented +Before sending a transaction to the blockchain, the user's account +needs to have some fund. In most test nets, a faucet server is provided +to fund users' accounts after receiving requests. We have implemented such a faucet server in the SEED blockchain emulator. -This example demonstrates how to use it. + +This example demonstrates how to use the Faucet server with either a +PoA blockchain based on [D00_ethereum_poa](../D00_ethereum_poa/) or a +PoS blockchain based on [D01_ethereum_pos](../D01_ethereum_pos/). ## Table of Content -- [Create a faucet server](#create-faucet-server) -- [Fund accounts during the build time](#fund-account-build-time) -- [Fund accounts during the run time (using curl)](#fund-account-run-time-curl) -- [Fund accounts during the run time (using Python)](#fund-account-run-time-python) -- [Advanced fund account for Developer](#fund-account-advanced) +- [How to Run](#how-to-run) +- [Create Faucet Server](#create-faucet-server) +- [How Faucet Works](#how-faucet-works) +- [Fund Accounts Using Faucet](#fund-accounts-using-faucet) +- [Fund Accounts During the Build Time](#fund-accounts-during-the-build-time) +- [Fund Accounts During the Run Time Using curl](#fund-accounts-during-the-run-time-using-curl) +- [Fund Accounts During the Run Time Using FaucetUserService](#fund-accounts-during-the-run-time-using-faucetuserservice) +- [Fund Accounts During the Run Time Using Python](#fund-accounts-during-the-run-time-using-python) +- [Observe Funding Results](#observe-funding-results) +- [Developer Manual](#developer-manual) + + +## How to Run + +Run the following commands from this example directory: + +```bash +python faucet.py [poa|pos] [amd|arm] +cd output +docker compose up +``` + +Arguments: + +- `poa`: Use the PoA blockchain from `D00_ethereum_poa`. This is the + default. +- `pos`: Use the PoS blockchain from `D01_ethereum_pos`. +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. + +Examples: + +```bash +python faucet.py poa amd +``` + +```bash +python faucet.py pos arm +``` + +The script loads the selected blockchain component, gets the existing +Faucet server from that blockchain, adds build-time funding requests, +and creates a `FaucetUserService` container to demonstrate runtime +funding. - ## Create Faucet Server -We first need to add a faucet server to a blockchain. This server -runs a web server, which can transfer the fund in its own account -to whoever requests for fund. This example uses -a pre-built block component (`D00_ethereum_poa`), which already has a faucet server. -In the [D00_ethereum_poa](../D00_ethereum_poa/) example, these lines are -used to create a faucet server +We first need to add a Faucet server to a blockchain. This server runs +a web server, which can transfer funds from its own account to whoever +requests funds. + +In the PoA blockchain from [D00_ethereum_poa](../D00_ethereum_poa/), +the Faucet server is created as follows: ```python -faucet:FaucetServer = blockchain.createFaucetServer( - vnode='faucet', - port=80, - linked_eth_node='eth5', - balance=10000, - max_fund_amount=10) +faucet: FaucetServer = blockchain.createFaucetServer( + vnode='faucet', + port=80, + linked_eth_node='eth5', + balance=10000, + max_fund_amount=10) faucet.setDisplayName('Faucet') ``` -We can specify the following parameters: -- `vnode`: the virtual node name of the faucet server. -- `port`: a port number, used by the faucet server to set up a web server. -- `linked_eth_node`: the faucet server needs to link to an eth node, so it can - sends out transactions to the blockchain. We just need to provide the name - of an existing eth node, but we need to make sure that the eth node - has enabled the http connection (otherwise, it cannot accept external requests). -- `balance`: set the initial balance (ETH) of the account used by the faucet server. - This account will be created during the build time and be added to the genesis block. -- `max_fund_amount`: the maximal amount of fund (in Ethers) that can be transferred - in each request. +In the PoS blockchain from [D01_ethereum_pos](../D01_ethereum_pos/), +the Faucet server is linked to a Geth execution-layer node: -Because this server is already created in the base component, -we just need to get an instance of this object: +```python +faucet: FaucetServer = blockchain.createFaucetServer( + vnode='faucet', + port=80, + linked_eth_node='gethnode1', + balance=10000, + max_fund_amount=10) +faucet.setDisplayName('Faucet') +``` + +We can specify the following parameters: + +- `vnode`: the virtual node name of the Faucet server. +- `port`: the port number used by the Faucet web server. +- `linked_eth_node`: the Ethereum node used by the Faucet server to + send transactions to the blockchain. This node must have HTTP RPC + enabled. +- `balance`: the initial balance, in Ethers, of the Faucet account. + This account is created during the build time and added to the + genesis block. +- `max_fund_amount`: the maximal amount of fund, in Ethers, that can + be transferred in each request. + +Because the selected base blockchain already has a Faucet server, this +example gets the existing Faucet server object: ```python eth = emu.getLayer('EthereumService') @@ -59,85 +114,196 @@ faucet = blockchain.getFaucetServerByName(faucet_name) ``` +## How Faucet Works + +The Faucet server is not an Ethereum node. It is a web service that +owns an Ethereum account. When it receives a `/fundme` request, it +constructs a transaction from its own account to the requested recipient +address, signs the transaction with the Faucet account's private key, +and sends the raw transaction through the linked Ethereum node's HTTP +RPC interface. + +Therefore, the Faucet does not directly modify account balances. It +funds an account by sending a normal Ethereum transaction. + + ## Fund Accounts Using Faucet - -### (1) Fund accounts during the build time +This example demonstrates three ways to fund accounts: + +- Fund known addresses during the build time. +- Fund an address during the run time using `curl`. +- Fund a dynamically created account during the run time using + `FaucetUserService`. + + +## Fund Accounts During the Build Time During the emulator build time, if we already know the account address, -we can ask the faucet to fund it, so when the -emulator starts, the faucet server will carry out the fund transfer. +we can ask the Faucet to fund it. The actual transaction is carried out +after the emulator starts. ```python faucet.fund('0x72943017a1fa5f255fc0f06625aec22319fcd5b3', 2) +faucet.fund('0x5449ba5c5f185e9694146d60cfe72681e2158499', 5) ``` - -### (2) Fund accounts during the run time (using curl) +These API calls generate startup commands for the Faucet container. +When the emulation starts and the Faucet server becomes ready, those +commands send HTTP requests to the Faucet server. -Very often, we do not know the account addresses during the build time, because -the accounts are created during the run time. In this case, during the run -time, the user can send a HTTP request to the faucet server to ask -the faucet server to fund a specified account. Data in the request -can be conveyed in two ways: form and json. Here are the examples -that use `curl` to send HTTP requests to the faucet server (we can -do this from any host). +## Fund Accounts During the Run Time Using curl + +Very often, we do not know the account addresses during the build time, +because the accounts are created during the run time. In this case, the +user can send an HTTP request to the Faucet server to ask it to fund a +specified account. + +First find the Faucet container and its IP address: + +```bash +docker ps | grep -i faucet +docker inspect ``` -curl -X POST -d "address=0x72943017a1fa5f255fc0f06625aec22319fcd5b3&amount=2" http://10.154.0.71:80/fundme -``` +Check whether the Faucet server is running: + +```bash +curl http://:80/ ``` -curl -X POST -H "Content-Type: application/json" -d '{ - "address": "0x72943017a1fa5f255fc0f06625aec22319fcd5b3", - "amount": 2 -}' http://10.154.0.71:80/fundme + +The expected response is: + +```text +OK ``` - -### (3) Fund accounts during the run time (using Python) +Data in the funding request can be conveyed using either form data or +JSON data. -We can write Python programs to interact with the Faucet server. -A helper class -called [FaucetHelper.py](../../../library/blockchain/FaucetHelper.py) -is created to make writing such programs easier. +Using form data: +```bash +curl -X POST \ + -d "address=0x72943017a1fa5f255fc0f06625aec22319fcd5b3&amount=2" \ + http://:80/fundme +``` +Using JSON data: + +```bash +curl -X POST \ + -H "Content-Type: application/json" \ + -d '{"address":"0x72943017a1fa5f255fc0f06625aec22319fcd5b3","amount":2}' \ + http://:80/fundme +``` - -## Advanced fund account for Developer +The requested amount cannot exceed `max_fund_amount`, which is `10` in +this example. -If you want to create a service class that creates Ethereum accounts -dynamically (i.e., during the emulator run time), -you may want to fund the accounts in your service class. -This `FaucetUserService` example shows you how to do that. -In this manual, we show how to use our `FaucetUserService`. -Detailed instructions on how this service is implemented can be found -in our [developer manual](../../../docs/developer_manual/blockchain/faucet-user-service.md) +## Fund Accounts During the Run Time Using FaucetUserService -To use the faucet, we need to specify the vnode name of the faucet server and the -port number. The `FaucetUserService` will use this information to set up -the script to interact with the faucet server. +This example also creates a `FaucetUserService` container. It +demonstrates how a service can create an account during the emulator +run time and then request funds from the Faucet server. ```python faucetUserService = FaucetUserService() -faucetUserService.install('faucetUser') -faucetUserService.setFaucetServerInfo(vnode = 'faucet', port=80) -emu.addBinding(Binding('faucetUser', filter=Filter(asn=164, nodeName='host_0'))) +faucetUserService.install('faucetUser').setDisplayName('FaucetUser') +faucet_info = blockchain.getFaucetServerInfo() +faucetUserService.setFaucetServerInfo(faucet_info[0]['name'], + faucet_info[0]['port']) +emu.addBinding(Binding('faucetUser')) +emu.addLayer(faucetUserService) ``` -In the example above, we hardcoded the faucet server's name and port number. -To make code more portable, we can get the faucet server information from -the blockchain service. +The `FaucetUserService` resolves the Faucet server's IP address from +the Faucet vnode name and port number. It then generates the following +script inside the `faucetUser` container: + +```text +/faucet_user/fundme.py +``` + +When the `faucetUser` container starts, this script waits for the +Faucet server to become available. If no account address is given to +the script, it creates a new Ethereum account and sends a `/fundme` +request for that new account. + +After the emulator starts, use the following command to observe this +runtime funding process: + +```bash +docker logs +``` + +The logs should include the newly created account address and a success +message from the Faucet server. + + +## Fund Accounts During the Run Time Using Python + +We can also write Python programs to interact with the Faucet server. +A helper class called +[FaucetHelper.py](../../../library/blockchain/FaucetHelper.py) is +created to make writing such programs easier. + +The helper sends a JSON request to the Faucet server's `/fundme` API: ```python -faucet_info = blockchain.getFaucetServerInfo() # returns a list of dictionary -faucetUserService.setFaucetServerInfo(faucet_info[0]['name'], faucet_info[0]['port']) +from FaucetHelper import FaucetHelper + +faucet = FaucetHelper("http://:80") +faucet.wait_for_server_ready() +faucet.send_fundme_request( + "0x72943017a1fa5f255fc0f06625aec22319fcd5b3", + 2) ``` -After the emulator starts, we can log into the `faucetUser` container. -We will find a script called `fund.py` in the root folder. -This script is generated during the build time based on the parameters -provided. It sends a fund request for a newly created account. +## Observe Funding Results + +The best way to understand the Faucet is to observe both the web +request and the blockchain transaction. + +First check the `faucetUser` logs: + +```bash +docker logs +``` + +Then check the Faucet server logs: + +```bash +docker logs +``` + +The Faucet logs should show the recipient address, requested amount, +transaction hash, and transaction receipt. + +To verify the recipient balance using Geth HTTP RPC, find a Geth node +IP address and run: + +```bash +curl -s -X POST \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["","latest"],"id":1}' \ + http://:8545 +``` + +The returned balance is in Wei and encoded as a hexadecimal number. + +This example compiles the Docker output with `etherViewEnabled=True`. +When the selected blockchain and explorer services are running, the +generated output can also provide a web interface for observing blocks, +transactions, and account activity. + + +## Developer Manual + +The `FaucetUserService` section above shows how to request funds during +runtime from a service container. For implementation details about how +to build a custom service that uses the Faucet server, see the +[developer manual](../../../docs/developer_manual/blockchain/faucet-user-service.md). diff --git a/examples/blockchain/D20_faucet/example.yaml b/examples/blockchain/D20_faucet/example.yaml new file mode 100644 index 000000000..8a06dd703 --- /dev/null +++ b/examples/blockchain/D20_faucet/example.yaml @@ -0,0 +1,142 @@ +id: blockchain-d20-faucet-pos +name: Ethereum PoS Faucet +description: A D01-style Ethereum proof-of-stake chain with Faucet build-time and runtime funding tests. +runner: blockchain +script: faucet.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - faucet + - faucet-user + +compile: + enabled: true + output: output + args: + - --consensus + - pos + - --no-ether-view + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D20 POS Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D20 POS Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D20 POS genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D20 POS beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D20 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D20 Faucet user service is running + type: ethereum-compose-ps + class_contains: FaucetUserService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D20 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + +probes: + - name: D20 POS Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: D20 POS Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: D20 POS Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: D20 POS beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: D20 Faucet HTTP health endpoint returns OK + type: ethereum-exec + class_contains: FaucetService + command: >- + python3 -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:80/", timeout=5).read().decode())' | grep -q '^OK$' + retries: 60 + interval: 5 + timeout: 15 + + - name: D20 POS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + +test_programs: + - name: D20 POS Faucet funding validation + script: test_runtime.py + timeout: 1800 diff --git a/examples/blockchain/D20_faucet/faucet.py b/examples/blockchain/D20_faucet/faucet.py index 6e55f0399..6877baea9 100755 --- a/examples/blockchain/D20_faucet/faucet.py +++ b/examples/blockchain/D20_faucet/faucet.py @@ -1,58 +1,160 @@ #!/usr/bin/env python3 # encoding: utf-8 -import sys, os +from __future__ import annotations + +import argparse +import os +from contextlib import contextmanager +from pathlib import Path +import sys +import tempfile + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu import * from examples.blockchain.D00_ethereum_poa import ethereum_poa +from examples.blockchain.D01_ethereum_pos import ethereum_pos + + +@contextmanager +def pushd(directory: Path): + previous = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(previous) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D20 Faucet example.") + parser.add_argument( + "legacy_args", + nargs="*", + help="legacy form: [poa|pos] [amd|arm]", + ) + parser.add_argument("--consensus", choices=["poa", "pos"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--ether-view", + dest="ether_view_enabled", + action="store_true", + default=True, + help="enable Eth Explorer output in generated Docker files (default)", + ) + parser.add_argument( + "--no-ether-view", + dest="ether_view_enabled", + action="store_false", + help="disable Eth Explorer output; useful for CI builds", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + + legacy_consensus = None + legacy_platform = None + for item in args.legacy_args: + value = item.lower() + if value in ("poa", "pos") and legacy_consensus is None: + legacy_consensus = value + elif value in ("amd", "arm") and legacy_platform is None: + legacy_platform = value + else: + parser.error("legacy arguments must be: [poa|pos] [amd|arm]") + + args.consensus = args.consensus or legacy_consensus or "poa" + args.platform = args.platform or legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def load_base_blockchain(consensus: str) -> Emulator: + emu = Emulator() + with tempfile.TemporaryDirectory(prefix="seedemu-d20-") as tempdir: + temp_path = Path(tempdir) + dump_path = temp_path / f"blockchain_{consensus}.bin" + with pushd(temp_path): + if consensus == "pos": + ethereum_pos.run(dumpfile=str(dump_path)) + else: + ethereum_poa.run(dumpfile=str(dump_path), total_accounts_per_node=1) + emu.load(str(dump_path)) + return emu + + +def build_emulator(consensus: str = "poa") -> Emulator: + emu = load_base_blockchain(consensus) + + eth = emu.getLayer("EthereumService") + blockchain = eth.getBlockchainByName(eth.getBlockchainNames()[0]) + faucet_name = blockchain.getFaucetServerNames()[0] + faucet = blockchain.getFaucetServerByName(faucet_name) + + # Funding accounts during build time. The actual funding is carried out + # after the emulation starts. + faucet.fund("0x72943017a1fa5f255fc0f06625aec22319fcd5b3", 2) + faucet.fund("0x5449ba5c5f185e9694146d60cfe72681e2158499", 5) + + # Funding accounts during run time, i.e., after the emulation starts. + faucet_user_service = FaucetUserService() + faucet_user_service.install("faucetUser").setDisplayName("FaucetUser") + faucet_info = blockchain.getFaucetServerInfo() + faucet_user_service.setFaucetServerInfo(faucet_info[0]["name"], faucet_info[0]["port"]) + emu.addBinding(Binding("faucetUser")) + emu.addLayer(faucet_user_service) + + return emu + + +def run( + *, + consensus: str = "poa", + dumpfile=None, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + ether_view_enabled: bool = True, +): + emu = build_emulator(consensus=consensus) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + docker = Docker(etherViewEnabled=ether_view_enabled, platform=platform) + emu.compile(docker, str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + consensus=args.consensus, + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ether_view_enabled=args.ether_view_enabled, + ) + return 0 -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -emu = Emulator() - -# Run and load the pre-built ethereum component; it is used as the base blockchain -local_dump_path = './blockchain_poa.bin' -ethereum_poa.run(dumpfile=local_dump_path, total_accounts_per_node=1) -emu.load(local_dump_path) - -# Get the faucet server instance -eth = emu.getLayer('EthereumService') -blockchain = eth.getBlockchainByName(eth.getBlockchainNames()[0]) -faucet_name = blockchain.getFaucetServerNames()[0] -faucet = blockchain.getFaucetServerByName(faucet_name) - -# Funding accounts during the build time. The actual funding -# is carried out after the emulator starts -faucet.fund('0x72943017a1fa5f255fc0f06625aec22319fcd5b3', 2) -faucet.fund('0x5449ba5c5f185e9694146d60cfe72681e2158499', 5) - -# Funding accounts during the run time, i.e., after the emulation starts -faucetUserService = FaucetUserService() -faucetUserService.install('faucetUser').setDisplayName('FaucetUser') -faucet_info = blockchain.getFaucetServerInfo() -faucetUserService.setFaucetServerInfo(faucet_info[0]['name'], - faucet_info[0]['port']) -emu.addBinding(Binding('faucetUser')) -emu.addLayer(faucetUserService) - -emu.render() - -docker = Docker(etherViewEnabled=True, platform=platform) -emu.compile(docker, './output', override=True) -# user_node.print(indent=4) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D20_faucet/test_runtime.py b/examples/blockchain/D20_faucet/test_runtime.py new file mode 100644 index 000000000..2f810b841 --- /dev/null +++ b/examples/blockchain/D20_faucet/test_runtime.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import re +import shlex +import time +from typing import Dict, Optional + +from seedemu.testing import EthereumRuntimeTest + + +WEI_PER_ETH = 10**18 +BUILD_TIME_FUNDS = ( + ("0x72943017a1fa5f255fc0f06625aec22319fcd5b3", 2), + ("0x5449ba5c5f185e9694146d60cfe72681e2158499", 5), +) +JSON_REQUEST_RECIPIENT = "0x1000000000000000000000000000000000000020" +FORM_REQUEST_RECIPIENT = "0x1000000000000000000000000000000000000021" +FAUCET_USER_RECIPIENT = "0x1000000000000000000000000000000000000022" +INVALID_RECIPIENT = "0xnot-an-ethereum-address" + + +def service_name(service: object) -> str: + return getattr(service, "name", str(service)) + + +def record( + test: EthereumRuntimeTest, + name: str, + service: object, + command: str, + ok: bool, + stdout: str = "", + stderr: str = "", +) -> Dict[str, object]: + result = { + "name": name, + "service": service_name(service), + "command": command, + "exit": 0 if ok else 1, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if ok else "failed", + } + test.results.append(result) + return result + + +def int_from_geth_output(output: object) -> int: + matches = re.findall(r"\b\d+\b", str(output)) + if not matches: + raise RuntimeError("could not parse integer from geth output: {}".format(output)) + return int(matches[-1]) + + +def get_balance_wei(test: EthereumRuntimeTest, geth_service: object, address: str) -> int: + result = test.geth_eval(geth_service, 'eth.getBalance("{}").toString(10)'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth balance query failed") + return int_from_geth_output(result["stdout"]) + + +def wait_for_balance_at_least( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + minimum_wei: int, + *, + retries: int = 60, + interval: int = 5, +) -> Optional[int]: + last_balance: Optional[int] = None + last_error = "" + for attempt in range(1, retries + 1): + try: + last_balance = get_balance_wei(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_balance >= minimum_wei: + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + True, + "balance={} attempts={}".format(last_balance, attempt), + ) + return last_balance + if attempt < retries: + time.sleep(interval) + + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + False, + "last_balance={}".format(last_balance), + last_error, + ) + return last_balance + + +def wait_for_balance_increase( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + before_wei: int, + delta_wei: int, +) -> None: + wait_for_balance_at_least( + test, + name, + geth_service, + address, + before_wei + delta_wei, + retries=60, + interval=5, + ) + + +def faucet_request_command(url: str, address: str, amount: int, mode: str, expected_status: int, expect_text: str) -> str: + script = ''' +import json +import sys +import urllib.error +import urllib.parse +import urllib.request + +url = __URL__ +address = __ADDRESS__ +amount = __AMOUNT__ +mode = __MODE__ +expected_status = __EXPECTED_STATUS__ +expect_text = __EXPECT_TEXT__.lower() + +headers = {} +if mode == "json": + body = json.dumps({"address": address, "amount": amount}).encode("utf-8") + headers["Content-Type"] = "application/json" +elif mode == "form": + body = urllib.parse.urlencode({"address": address, "amount": amount}).encode("utf-8") + headers["Content-Type"] = "application/x-www-form-urlencoded" +else: + raise SystemExit("unknown request mode: {}".format(mode)) + +request = urllib.request.Request(url, data=body, headers=headers, method="POST") +try: + with urllib.request.urlopen(request, timeout=360) as response: + status = response.status + text = response.read().decode("utf-8", errors="replace") +except urllib.error.HTTPError as exc: + status = exc.code + text = exc.read().decode("utf-8", errors="replace") + +print("status={}".format(status)) +print(text) +if status != expected_status: + raise SystemExit(1) +if expect_text and expect_text not in text.lower(): + raise SystemExit(1) +'''.strip() + replacements = { + "__URL__": json.dumps(url), + "__ADDRESS__": json.dumps(address), + "__AMOUNT__": str(int(amount)), + "__MODE__": json.dumps(mode), + "__EXPECTED_STATUS__": str(int(expected_status)), + "__EXPECT_TEXT__": json.dumps(expect_text), + } + for old, new in replacements.items(): + script = script.replace(old, new) + return "python3 -c {}".format(shlex.quote(script)) + + +def send_faucet_request( + test: EthereumRuntimeTest, + name: str, + requester_service: object, + faucet_url: str, + address: str, + amount: int, + *, + mode: str, + expected_status: int = 200, + expect_text: str = "success", +) -> Dict[str, object]: + command = faucet_request_command(faucet_url, address, amount, mode, expected_status, expect_text) + result = test.exec(requester_service, command, timeout=420) + ok = result["exit"] == 0 + return record(test, name, requester_service, command, ok, str(result["stdout"]), str(result["stderr"])) + + +def run_faucet_user_script( + test: EthereumRuntimeTest, + faucet_user_service: object, + address: str, +) -> Dict[str, object]: + command = "python3 faucet_user/fundme.py {}".format(shlex.quote(address)) + result = test.exec(faucet_user_service, command, timeout=420) + ok = result["exit"] == 0 + return record( + test, + "D20 FaucetUserService script can request runtime funds", + faucet_user_service, + command, + ok, + str(result["stdout"]), + str(result["stderr"]), + ) + + +def main() -> int: + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D20 has POS Geth nodes for balance checks", + expected_count=3, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + test.require_ethereum_services( + "D20 has POS Lighthouse beacon nodes", + expected_count=3, + class_contains="Ethereum-POS-Beacon", + consensus="POS", + ) + test.require_ethereum_services( + "D20 has POS genesis validator clients", + expected_count=9, + class_contains="Ethereum-POS-Validator", + role="validator_at_genesis", + consensus="POS", + ) + faucet_services = test.require_ethereum_services( + "D20 has one Faucet service", + expected_count=1, + class_contains="FaucetService", + ) + faucet_user_services = test.require_ethereum_services( + "D20 has one FaucetUser service", + expected_count=1, + class_contains="FaucetUserService", + ) + test.require_ethereum_services( + "D20 has one Utility service", + expected_count=1, + class_contains="EthUtilityServer", + ) + + if not geth_nodes or not faucet_services or not faucet_user_services: + test.write_summary("d20-faucet-runtime-test.json") + return test.exit_code() + + geth = geth_nodes[0] + faucet = faucet_services[0] + faucet_user = faucet_user_services[0] + faucet_fund_url = "http://{}:80/fundme".format(faucet.address) + + for address, amount_eth in BUILD_TIME_FUNDS: + wait_for_balance_at_least( + test, + "D20 build-time Faucet funding reaches {} ETH for {}".format(amount_eth, address), + geth, + address, + amount_eth * WEI_PER_ETH, + retries=90, + interval=5, + ) + + before_json = get_balance_wei(test, geth, JSON_REQUEST_RECIPIENT) + json_result = send_faucet_request( + test, + "D20 Faucet JSON /fundme request succeeds", + faucet_user, + faucet_fund_url, + JSON_REQUEST_RECIPIENT, + 1, + mode="json", + ) + if json_result["status"] == "passed": + wait_for_balance_increase( + test, + "D20 Faucet JSON /fundme transfers 1 ETH", + geth, + JSON_REQUEST_RECIPIENT, + before_json, + WEI_PER_ETH, + ) + + before_form = get_balance_wei(test, geth, FORM_REQUEST_RECIPIENT) + form_result = send_faucet_request( + test, + "D20 Faucet form /fundme request succeeds", + faucet_user, + faucet_fund_url, + FORM_REQUEST_RECIPIENT, + 1, + mode="form", + ) + if form_result["status"] == "passed": + wait_for_balance_increase( + test, + "D20 Faucet form /fundme transfers 1 ETH", + geth, + FORM_REQUEST_RECIPIENT, + before_form, + WEI_PER_ETH, + ) + + before_user = get_balance_wei(test, geth, FAUCET_USER_RECIPIENT) + user_result = run_faucet_user_script(test, faucet_user, FAUCET_USER_RECIPIENT) + if user_result["status"] == "passed": + wait_for_balance_increase( + test, + "D20 FaucetUserService runtime script transfers 10 ETH", + geth, + FAUCET_USER_RECIPIENT, + before_user, + 10 * WEI_PER_ETH, + ) + + send_faucet_request( + test, + "D20 Faucet rejects invalid Ethereum address", + faucet_user, + faucet_fund_url, + INVALID_RECIPIENT, + 1, + mode="json", + expected_status=500, + expect_text="invalid ethereum address", + ) + send_faucet_request( + test, + "D20 Faucet rejects requests above max_fund_amount", + faucet_user, + faucet_fund_url, + JSON_REQUEST_RECIPIENT, + 11, + mode="json", + expected_status=500, + expect_text="max_fund_amount", + ) + + test.write_summary("d20-faucet-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D21_deploy_contract/README.md b/examples/blockchain/D21_deploy_contract/README.md index 0506255bc..9bfb54787 100644 --- a/examples/blockchain/D21_deploy_contract/README.md +++ b/examples/blockchain/D21_deploy_contract/README.md @@ -1,46 +1,103 @@ # EthUtilityServer -The Utility server has two roles. The first role is to help deploy smart contract, -and the second role is to provide contract addresses to others. In Ethereum, once a -contract has been deployed, it will be assigned an address. Users need to -know this address to interact with the contract. The Utility server -hosts a web server for others to register contract addresses. +The Utility server has two roles. The first role is to help deploy +smart contracts, and the second role is to provide contract addresses +to other services. In Ethereum, once a contract has been deployed, it +is assigned an address. Users and applications need this address to +interact with the contract. + +This example demonstrates how to use the Utility server with either a +PoA blockchain based on [D00_ethereum_poa](../D00_ethereum_poa/) or a +PoS blockchain based on [D01_ethereum_pos](../D01_ethereum_pos/). It +also shows how a contract deployment is prepared during emulator build +time and carried out after the emulator starts. + + +## Table of Content + +- [How to Run](#how-to-run) +- [Create Utility Server](#create-utility-server) +- [Smart Contract Files](#smart-contract-files) +- [Deploy a Contract](#deploy-a-contract) +- [How Deployment Works](#how-deployment-works) +- [Interact with the Utility Server Using curl](#interact-with-the-utility-server-using-curl) +- [Observe Deployment Results](#observe-deployment-results) +- [Interact with the Utility Server Using Python](#interact-with-the-utility-server-using-python) + + +## How to Run + +Run the following commands from this example directory: + +```bash +python deploy_contract.py [poa|pos] [amd|arm] +cd output +docker compose up +``` + +Arguments: + +- `poa`: Use the PoA blockchain from `D00_ethereum_poa`. This is the + default. +- `pos`: Use the PoS blockchain from `D01_ethereum_pos`. +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. + +Examples: + +```bash +python deploy_contract.py poa amd +``` + +```bash +python deploy_contract.py pos arm +``` + +The script loads the selected blockchain component, gets the existing +Utility server from that blockchain, registers the contract files with +the Utility server, and generates the Docker output. -- [Create a Utility server](#create-utility-server) -- [Deploy a contract](#deploy-contract) -- [Interact with the Uility server](#interact-with-server) - ## Create Utility Server -We first need to add a Utility server to a blockchain. -This example uses a pre-built block component (`D00_ethereum_poa`), -which already has a Utility server. -In the [D00_ethereum_poa](../D00_ethereum_poa/) example, these lines are -used to create a utility server +We first need to add a Utility server to a blockchain. The Utility +server runs a web server for contract address registration and lookup. +It can also deploy contracts after the emulator starts. +In the PoA blockchain from [D00_ethereum_poa](../D00_ethereum_poa/), +the Utility server is created as follows: ```python -util_server:EthUtilityServer = blockchain.createEthUtilityServer( - vnode='utility', - port=5000, - linked_eth_node='eth6', - linked_faucet_node='faucet') +util_server: EthUtilityServer = blockchain.createEthUtilityServer( + vnode='utility', + port=5000, + linked_eth_node='eth6', + linked_faucet_node='faucet') +``` + +In the PoS blockchain from [D01_ethereum_pos](../D01_ethereum_pos/), +the Utility server is linked to a Geth execution-layer node: + +```python +util_server: EthUtilityServer = blockchain.createEthUtilityServer( + vnode='utility', + port=5000, + linked_eth_node='gethnode2', + linked_faucet_node='faucet') ``` We can specify the following parameters: + - `vnode`: the virtual node name of the Utility server. -- `port`: a port number, used by the server to set up a web server. -- `linked_eth_node`: the faucet server needs to link to an eth node, so it can - sends out transactions to the blockchain. We just need to provide the name - of an existing eth node, but we need to make sure that the eth node - has enabled the http connection (otherwise, it cannot accept external requests). -- `linked_faucet_node`: the Utility server will create an Ethereum account, which - is used to deploy contracts. The Utility server will use the faucet server - to find the account. - -Because this server is already created in the base component, -we just need to get an instance of this object: +- `port`: the port number used by the Utility server web API. +- `linked_eth_node`: the Ethereum node used by the Utility server to + send contract deployment transactions. This node must have HTTP RPC + enabled. +- `linked_faucet_node`: the Faucet server used to fund the account that + deploys contracts. + +Because the selected base blockchain already has a Utility server, this +example gets the existing Utility server object: ```python eth = emu.getLayer('EthereumService') @@ -50,75 +107,220 @@ utility = blockchain.getUtilityServerByName(name) ``` - -## Deploy a contract +## Smart Contract Files + +The contract files used by this example are located in the `Contracts` +directory: + +```text +Contracts/contract.sol +Contracts/contract.abi +Contracts/contract.bin +``` + +The files have different purposes: + +- `contract.sol`: the Solidity source code. This is the human-readable + contract program. +- `contract.abi`: the contract interface description. Applications use + it to know which functions exist and how to encode calls. +- `contract.bin`: the compiled EVM bytecode. This is the code deployed + to the blockchain. + +The example contract is a small `Crowdfunding` contract: + +```solidity +contract Crowdfunding { + uint256 amount; + + receive() external payable { + amount += msg.value; + } + + function claimFunds(address payable _to, uint _amount) public payable { + _to.transfer(_amount); + } +} +``` + +The `receive()` function is triggered when ETH is sent directly to the +contract address. The `claimFunds()` function transfers funds from the +contract balance to a specified address. This contract is only a simple +teaching example; it does not implement access control. + -To deploy a contract using the Utility server, -we need to set `abi` and `bin` file. A path can be both relative and absolute. +## Deploy a Contract + +To deploy a contract using the Utility server, we need to provide the +ABI file and the bytecode file. The paths can be either relative or +absolute. ```python -utility.deployContractByFilePath(contract_name='test', - abi_path="./Contracts/contract.abi", - bin_path="./Contracts/contract.bin") +utility.deployContractByFilePath( + contract_name='test', + abi_path='./Contracts/contract.abi', + bin_path='./Contracts/contract.bin') +``` + +This API does not immediately deploy the contract. During emulator build +time, it registers the contract files with the Utility server. The +actual deployment happens after the emulator starts. + +The contract is registered under the name `test`. After deployment, the +Utility server stores a mapping from this name to the deployed contract +address. + + +## How Deployment Works + +A contract deployment is an Ethereum transaction. The Utility server +must therefore have an account with enough ETH to pay gas. + +When the Utility container starts, it runs the following setup script: + +```bash +python3 ./fund_account.py +python3 ./deploy_contract.py ``` -It should be noted that the contract deployment will happen after the emulator -starts, so this API will set up the contract deployment on the Utility server. +The setup process works as follows: +1. The Utility server connects to the linked Ethereum node through HTTP + RPC. +2. It creates a new Ethereum account and saves the address and private + key in `/utility_server/account.json`. +3. It sends a `/fundme` request to the linked Faucet server. +4. It waits until the new account has a positive balance. +5. It reads the registered ABI and BIN files. +6. It constructs a contract deployment transaction. +7. It signs the transaction using the generated account's private key. +8. It sends the raw transaction through the linked Ethereum node. +9. After the transaction is confirmed, it saves the deployed contract + address in `/utility_server/deployed_contracts/contract_address.txt`. - -## Interact with the Utility server using `curl` +The Utility server is therefore not the smart contract itself. It is a +helper service that funds a deployer account, sends the deployment +transaction, and records the resulting contract address. -After the emulator starts, we can interact with the Utility server -using the `curl` command. To do that, -we first need to get the server's IP address (the -port number used in the example is `5000`). -```sh -$ docker ps | grep -i utility -e22e918b4c90 output_hnode_150_host_0 ... as162h-UtilityServer-10.150.0.71 +## Interact with the Utility Server Using curl + +After the emulator starts, we can interact with the Utility server using +`curl`. First find the Utility container and its IP address: + +```bash +docker ps | grep -i utility +docker inspect ``` -In this example, the url of the server is `http://10.150.0.71:5000/`. +Check whether the Utility server is running: +```bash +curl http://:5000/ +``` -### Register a contract +The expected response is: + +```text +OK +``` -To register a contract with the Utility server, we use a POST request -to send the contract information (JSON format) -to the server's `/register_contract` API. -See the following example: +### Register a Contract -```sh +To register a contract address manually, send a POST request to the +`/register_contract` API: + +```bash curl -X POST \ -H "Content-Type: application/json" \ - -d '{"contract_name": "test999", "contract_address": "0xc0ffee254729296a45a3885639AC7E10F9d54979"}' \ - http://10.150.0.71:5000/register_contract + -d '{"contract_name":"test999","contract_address":"0xc0ffee254729296a45a3885639AC7E10F9d54979"}' \ + http://:5000/register_contract ``` -### Get a list of registered contracts +This only registers a name-address mapping in the Utility server. It +does not deploy a new contract to the blockchain. + +### Get a List of Registered Contracts -We can use the following API to get a list of all the registered contracts. +Use either of the following APIs to get all registered contract +addresses: -```sh -curl http://10.150.0.71:5000/all -curl http://10.150.0.71:5000/contracts_info +```bash +curl http://:5000/all +curl http://:5000/contracts_info ``` +### Get a Contract Address by Name -### Get a contract address by its name +To get the address of a particular contract, provide the `name` +argument: + +```bash +curl http://:5000/contracts_info?name=test +``` -If we just want to get the address of a particular contract, we can provide -the `name` argument. -```sh -curl http://10.150.0.71:5000/contracts_info?name=test +## Observe Deployment Results + +The best way to understand contract deployment is to observe both the +Utility server logs and the blockchain state. + +First check the Utility server logs: + +```bash +docker logs ``` +The logs should show that the Utility server connected to the Ethereum +node, requested funds from the Faucet server, funded the deployer +account, and deployed the contract. + +Then inspect the generated files inside the Utility container: + +```bash +docker exec -it bash +cat /utility_server/account.json +cat /utility_server/contracts/contract_file_paths.txt +cat /utility_server/deployed_contracts/contract_address.txt +``` + +The `account.json` file contains the deployer account created by the +Utility server. The `contract_address.txt` file contains the deployed +contract address, for example: + +```json +{ + "test": "0x..." +} +``` + +To verify that the address is really a contract on the blockchain, use +Geth HTTP RPC to query the code stored at the address: + +```bash +curl -s -X POST \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_getCode","params":["","latest"],"id":1}' \ + http://:8545 +``` + +If the result is `"0x"`, there is no contract code at that address. If +the result is a long hexadecimal string, the contract bytecode has been +stored on chain. + +This example compiles the Docker output with `etherViewEnabled=True`. +When the selected blockchain and explorer services are running, the +generated output can also provide a web interface for observing blocks, +transactions, and contract deployment activity. + + +## Interact with the Utility Server Using Python -## Interact with the Utility server using Python +We can write Python programs to interact with the Utility server. A +helper class called +[UtilityServerHelper.py](../../../library/blockchain/UtilityServerHelper.py) +is created to make writing such programs easier. -We can write Python programs to interact with the Utility server. -A helper class -called [UtilityServerHelper.py](../../../library/blockchain/UtilityServerHelper.py) -is created to make writing such programs easier. +For example, a program can query the address of the deployed `test` +contract from the Utility server and then use that address together +with the contract ABI to interact with the smart contract. diff --git a/examples/blockchain/D21_deploy_contract/deploy_contract.py b/examples/blockchain/D21_deploy_contract/deploy_contract.py index 64a32ce03..a129b85a4 100755 --- a/examples/blockchain/D21_deploy_contract/deploy_contract.py +++ b/examples/blockchain/D21_deploy_contract/deploy_contract.py @@ -1,64 +1,158 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +import os +from contextlib import contextmanager +from pathlib import Path +import sys +import tempfile + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu import * from examples.blockchain.D00_ethereum_poa import ethereum_poa -from seedemu.services.EthereumService import * -import os, sys - -def run(dumpfile = None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) +from examples.blockchain.D01_ethereum_pos import ethereum_pos + + +@contextmanager +def pushd(directory: Path): + previous = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(previous) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D21 deploy-contract example.") + parser.add_argument( + "legacy_args", + nargs="*", + help="legacy form: [poa|pos] [amd|arm]", + ) + parser.add_argument("--consensus", choices=["poa", "pos"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--ether-view", + dest="ether_view_enabled", + action="store_true", + default=True, + help="enable Eth Explorer output in generated Docker files (default)", + ) + parser.add_argument( + "--no-ether-view", + dest="ether_view_enabled", + action="store_false", + help="disable Eth Explorer output; useful for CI builds", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + + legacy_consensus = None + legacy_platform = None + for item in args.legacy_args: + value = item.lower() + if value in ("poa", "pos") and legacy_consensus is None: + legacy_consensus = value + elif value in ("amd", "arm") and legacy_platform is None: + legacy_platform = value else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) + parser.error("legacy arguments must be: [poa|pos] [amd|arm]") + + args.consensus = args.consensus or legacy_consensus or "poa" + args.platform = args.platform or legacy_platform or "amd" + return args + - ############################################################################### +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def load_base_blockchain(consensus: str) -> Emulator: emu = Emulator() + with tempfile.TemporaryDirectory(prefix="seedemu-d21-") as tempdir: + temp_path = Path(tempdir) + dump_path = temp_path / f"blockchain_{consensus}.bin" + with pushd(temp_path): + if consensus == "pos": + ethereum_pos.run(dumpfile=str(dump_path)) + else: + ethereum_poa.run( + dumpfile=str(dump_path), + hosts_per_as=1, + total_eth_nodes=10, + total_accounts_per_node=1, + ) + emu.load(str(dump_path)) + return emu - # Run and load the pre-built ethereum component; it is used as the base blockchain - # This component already has already created a utility server - local_dump_path = './blockchain_poa.bin' - ethereum_poa.run(dumpfile=local_dump_path, hosts_per_as=1, - total_eth_nodes=10, total_accounts_per_node=1) - emu.load(local_dump_path) - - # Get the utility server - eth = emu.getLayer('EthereumService') + +def build_emulator(consensus: str = "poa") -> Emulator: + emu = load_base_blockchain(consensus) + + eth = emu.getLayer("EthereumService") blockchain = eth.getBlockchainByName(eth.getBlockchainNames()[0]) - name = blockchain.getUtilityServerNames()[0] - utility = blockchain.getUtilityServerByName(name) + utility_name = blockchain.getUtilityServerNames()[0] + utility = blockchain.getUtilityServerByName(utility_name) + + utility.deployContractByFilePath( + contract_name="test", + abi_path=str((SCRIPT_DIR / "Contracts" / "contract.abi").resolve()), + bin_path=str((SCRIPT_DIR / "Contracts" / "contract.bin").resolve()), + ) - # Deploy contract on the utility server - # The actual deployment is carried out at the run time. - # This API just sets it up - utility.deployContractByFilePath(contract_name='test', - abi_path="./Contracts/contract.abi", - bin_path="./Contracts/contract.bin") + return emu - # Generate the emulator files + +def run( + *, + consensus: str = "poa", + dumpfile=None, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + ether_view_enabled: bool = True, +): + emu = build_emulator(consensus=consensus) if dumpfile is not None: emu.dump(dumpfile) - else: + return + + if render: emu.render() - docker = Docker(etherViewEnabled=True, platform=platform) - emu.compile(docker, './output', override = True) + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + docker = Docker(etherViewEnabled=ether_view_enabled, platform=platform) + emu.compile(docker, str(output_dir), override=override) -if __name__ == "__main__": - run() +def main() -> int: + args = parse_args() + run( + consensus=args.consensus, + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ether_view_enabled=args.ether_view_enabled, + ) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D21_deploy_contract/example.yaml b/examples/blockchain/D21_deploy_contract/example.yaml new file mode 100644 index 000000000..bb0c3dc28 --- /dev/null +++ b/examples/blockchain/D21_deploy_contract/example.yaml @@ -0,0 +1,144 @@ +id: blockchain-d21-deploy-contract-pos +name: Ethereum PoS Contract Deployment +description: A D01-style Ethereum proof-of-stake chain that deploys and exercises a Crowdfunding smart contract through the Utility server. +runner: blockchain +script: deploy_contract.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - faucet + - utility-server + - smart-contract-deployment + +compile: + enabled: true + output: output + args: + - --consensus + - pos + - --no-ether-view + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D21 POS Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D21 POS Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D21 POS genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D21 POS beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D21 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D21 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + +probes: + - name: D21 POS Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: D21 POS Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: D21 POS Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: D21 POS beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: D21 Geth IPC exposes a funded local account + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: "geth attach --exec 'eth.accounts.length > 0' | grep -q true" + retries: 30 + interval: 3 + + - name: D21 Utility HTTP health endpoint returns OK + type: ethereum-exec + class_contains: EthUtilityServer + command: >- + python3 -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:5000/", timeout=5).read().decode())' | grep -q '^OK$' + retries: 90 + interval: 5 + timeout: 20 + + - name: D21 POS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + +test_programs: + - name: D21 contract deployment and interaction validation + script: test_runtime.py + timeout: 1800 diff --git a/examples/blockchain/D21_deploy_contract/test_runtime.py b/examples/blockchain/D21_deploy_contract/test_runtime.py new file mode 100644 index 000000000..c452d73dd --- /dev/null +++ b/examples/blockchain/D21_deploy_contract/test_runtime.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import re +import shlex +import time +from typing import Any, Callable, Dict, Optional + +from seedemu.testing import EthereumRuntimeTest + + +WEI_PER_ETH = 10**18 +CONTRACT_NAME = "test" +CONTRACT_FUND_WEI = WEI_PER_ETH +CLAIM_WEI = WEI_PER_ETH // 2 +CLAIM_RECIPIENT = "0x1000000000000000000000000000000000000021" +MANUAL_CONTRACT_NAME = "manual_d21" +MANUAL_CONTRACT_ADDRESS = "0xc0ffee254729296a45a3885639AC7E10F9d54979" +CLAIM_FUNDS_SELECTOR = "ed2b40ea" + + +def service_name(service: object) -> str: + return getattr(service, "name", str(service)) + + +def record( + test: EthereumRuntimeTest, + name: str, + service: object, + command: str, + ok: bool, + stdout: str = "", + stderr: str = "", +) -> Dict[str, object]: + result = { + "name": name, + "service": service_name(service), + "command": command, + "exit": 0 if ok else 1, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if ok else "failed", + } + test.results.append(result) + return result + + +def int_from_geth_output(output: object) -> int: + matches = re.findall(r"\b\d+\b", str(output)) + if not matches: + raise RuntimeError("could not parse integer from geth output: {}".format(output)) + return int(matches[-1]) + + +def parse_geth_int(result: Dict[str, object]) -> int: + for stream_name in ("stdout", "stderr"): + output = result[stream_name] + try: + return int_from_geth_output(output) + except RuntimeError: + continue + raise RuntimeError( + "could not parse integer from geth output: stdout={} stderr={}".format( + result["stdout"], result["stderr"] + ) + ) + + +def parse_geth_code(result: Dict[str, object]) -> str: + for stream_name in ("stdout", "stderr"): + lines = [line.strip().strip('"') for line in str(result[stream_name]).splitlines()] + for line in reversed(lines): + if not line: + continue + if re.fullmatch(r"0x[0-9a-fA-F]*", line): + return line + if re.fullmatch(r"[0-9a-fA-F]+", line) and len(line) % 2 == 0: + return "0x" + line + raise RuntimeError( + "could not parse contract code from stdout={} stderr={}".format( + result["stdout"], result["stderr"] + ) + ) + + +def extract_geth_json(test: EthereumRuntimeTest, result: Dict[str, object]) -> Dict[str, Any]: + return test._extract_json("{}\n{}".format(result["stdout"], result["stderr"])) + + +def get_balance_wei(test: EthereumRuntimeTest, geth_service: object, address: str) -> int: + result = test.geth_eval(geth_service, 'eth.getBalance("{}").toString(10)'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth balance query failed") + return parse_geth_int(result) + + +def wait_for_balance_at_least( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + minimum_wei: int, + *, + retries: int = 60, + interval: int = 5, +) -> Optional[int]: + last_balance: Optional[int] = None + last_error = "" + for attempt in range(1, retries + 1): + try: + last_balance = get_balance_wei(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_balance >= minimum_wei: + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + True, + "balance={} attempts={}".format(last_balance, attempt), + ) + return last_balance + if attempt < retries: + time.sleep(interval) + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + False, + "last_balance={}".format(last_balance), + last_error, + ) + return None + + +def get_contract_code(test: EthereumRuntimeTest, geth_service: object, address: str) -> str: + result = test.geth_eval(geth_service, 'eth.getCode("{}")'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth code query failed") + return parse_geth_code(result) + + +def wait_for_contract_code( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + *, + retries: int = 60, + interval: int = 5, +) -> Optional[str]: + last_code = "" + last_error = "" + for attempt in range(1, retries + 1): + try: + last_code = get_contract_code(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_code != "0x": + record( + test, + name, + geth_service, + "eth.getCode({}) != 0x".format(address), + True, + "code_bytes={} attempts={}".format((len(last_code) - 2) // 2, attempt), + ) + return last_code + if attempt < retries: + time.sleep(interval) + record(test, name, geth_service, "eth.getCode({}) != 0x".format(address), False, last_code, last_error) + return None + + +def wait_for_json_file( + test: EthereumRuntimeTest, + name: str, + service: object, + path: str, + predicate: Optional[Callable[[Any], bool]] = None, + *, + retries: int = 120, + interval: int = 5, +) -> Optional[Any]: + command = "cat {}".format(shlex.quote(path)) + last_stdout = "" + last_stderr = "" + for attempt in range(1, retries + 1): + result = test.exec(service, command, timeout=30) + last_stdout = str(result["stdout"]) + last_stderr = str(result["stderr"]) + if result["exit"] == 0: + try: + data = json.loads(last_stdout) + except json.JSONDecodeError as exc: + last_stderr = str(exc) + else: + if predicate is None or predicate(data): + record(test, name, service, command, True, "attempts={} data={}".format(attempt, last_stdout)) + return data + if attempt < retries: + time.sleep(interval) + record(test, name, service, command, False, last_stdout, last_stderr) + return None + + +def utility_http_command( + path: str, + *, + method: str = "GET", + payload: Optional[Dict[str, object]] = None, + expected_status: int = 200, + expect_text: Optional[str] = None, +) -> str: + code = '''import json +import urllib.error +import urllib.parse +import urllib.request + +url = "http://127.0.0.1:5000{path}" +method = "{method}" +expected_status = {expected_status} +expect_text = {expect_text!r} +payload = {payload} +headers = {{}} +body = None +if payload is not None: + body = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" +request = urllib.request.Request(url, data=body, headers=headers, method=method) +try: + with urllib.request.urlopen(request, timeout=10) as response: + status = response.status + text = response.read().decode("utf-8", errors="replace") +except urllib.error.HTTPError as exc: + status = exc.code + text = exc.read().decode("utf-8", errors="replace") +print("status={{}}".format(status)) +print(text) +if status != expected_status: + raise SystemExit(1) +if expect_text is not None and expect_text.lower() not in text.lower(): + raise SystemExit(1) +'''.format( + path=path, + method=method, + expected_status=expected_status, + expect_text=expect_text, + payload=json.dumps(payload) if payload is not None else "None", + ) + return "python3 -c {}".format(shlex.quote(code)) + + +def run_utility_http_check( + test: EthereumRuntimeTest, + name: str, + utility_service: object, + path: str, + *, + method: str = "GET", + payload: Optional[Dict[str, object]] = None, + expected_status: int = 200, + expect_text: Optional[str] = None, +) -> Dict[str, object]: + command = utility_http_command( + path, + method=method, + payload=payload, + expected_status=expected_status, + expect_text=expect_text, + ) + result = test.exec(utility_service, command, timeout=30) + return record( + test, + name, + utility_service, + command, + result["exit"] == 0, + str(result["stdout"]), + str(result["stderr"]), + ) + + +def encode_claim_funds(recipient: str, amount_wei: int) -> str: + recipient_hex = recipient.lower() + if not re.fullmatch(r"0x[0-9a-f]{40}", recipient_hex): + raise ValueError("invalid recipient address: {}".format(recipient)) + recipient_word = recipient_hex[2:].rjust(64, "0") + amount_word = hex(amount_wei)[2:].rjust(64, "0") + return "0x" + CLAIM_FUNDS_SELECTOR + recipient_word + amount_word + + +def signed_contract_call_raw_transaction( + test: EthereumRuntimeTest, + service: object, + contract_address: str, + data_hex: str, + *, + gas: int = 200000, + value_wei: int = 0, + password: str = "admin", +) -> str: + try: + from eth_account import Account + except ImportError as exc: + raise RuntimeError("eth_account is required to sign contract call transactions") from exc + + state = test._local_account_transaction_state(service) + sender = str(state["sender"]) + chain_id = test._chain_id_for_service(service) + private_key = test._private_key_for_local_account(service, sender, password, chain_id=chain_id) + transaction = { + "chainId": chain_id, + "nonce": test._rpc_int(state["nonce"]), + "gas": gas, + "gasPrice": test._rpc_int(state["gasPrice"]), + "to": contract_address, + "value": int(value_wei), + "data": bytes.fromhex(data_hex[2:] if data_hex.startswith("0x") else data_hex), + } + signed = Account.sign_transaction(transaction, private_key) + raw_transaction = getattr(signed, "rawTransaction", None) + if raw_transaction is None: + raw_transaction = getattr(signed, "raw_transaction") + raw_hex = raw_transaction.hex() + if not raw_hex.startswith("0x"): + raw_hex = "0x" + raw_hex + return raw_hex + + +def claim_funds_script(contract_address: str, recipient: str, raw_transaction: str, receipt_retries: int = 90) -> str: + script = ''' +var contract = __CONTRACT__; +var recipient = __RECIPIENT__; +var contractBefore = eth.getBalance(contract); +var recipientBefore = eth.getBalance(recipient); +var txHash = eth.sendRawTransaction(__RAW_TRANSACTION__); +var receipt = null; +for (var i = 0; i < __RECEIPT_RETRIES__; i++) { + receipt = eth.getTransactionReceipt(txHash); + if (receipt !== null && receipt.blockNumber !== null) { break; } + admin.sleep(1); +} +var contractAfter = eth.getBalance(contract); +var recipientAfter = eth.getBalance(recipient); +JSON.stringify({ + contract: contract, + recipient: recipient, + txHash: txHash, + receiptBlockNumber: receipt === null ? null : receipt.blockNumber.toString(10), + receiptStatus: receipt === null || receipt.status === null ? null : receipt.status.toString(10), + contractBefore: contractBefore.toString(10), + contractAfter: contractAfter.toString(10), + recipientBefore: recipientBefore.toString(10), + recipientAfter: recipientAfter.toString(10), + contractDelta: contractBefore.minus(contractAfter).toString(10), + recipientDelta: recipientAfter.minus(recipientBefore).toString(10) +}) +'''.strip() + script = script.replace("__CONTRACT__", json.dumps(contract_address)) + script = script.replace("__RECIPIENT__", json.dumps(recipient)) + script = script.replace("__RAW_TRANSACTION__", json.dumps(raw_transaction)) + script = script.replace("__RECEIPT_RETRIES__", str(receipt_retries)) + return script + + +def send_contract_ether_script(contract_address: str, raw_transaction: str, receipt_retries: int = 90) -> str: + script = ''' +var sender = eth.accounts[0]; +var contract = __CONTRACT__; +var senderBefore = eth.getBalance(sender); +var contractBefore = eth.getBalance(contract); +var txHash = eth.sendRawTransaction(__RAW_TRANSACTION__); +var receipt = null; +for (var i = 0; i < __RECEIPT_RETRIES__; i++) { + receipt = eth.getTransactionReceipt(txHash); + if (receipt !== null && receipt.blockNumber !== null) { break; } + admin.sleep(1); +} +var senderAfter = eth.getBalance(sender); +var contractAfter = eth.getBalance(contract); +JSON.stringify({ + sender: sender, + contract: contract, + txHash: txHash, + receiptBlockNumber: receipt === null ? null : receipt.blockNumber.toString(10), + receiptStatus: receipt === null || receipt.status === null ? null : receipt.status.toString(10), + senderBefore: senderBefore.toString(10), + senderAfter: senderAfter.toString(10), + contractBefore: contractBefore.toString(10), + contractAfter: contractAfter.toString(10), + senderDelta: senderBefore.minus(senderAfter).toString(10), + contractDelta: contractAfter.minus(contractBefore).toString(10) +}) +'''.strip() + script = script.replace("__CONTRACT__", json.dumps(contract_address)) + script = script.replace("__RAW_TRANSACTION__", json.dumps(raw_transaction)) + script = script.replace("__RECEIPT_RETRIES__", str(receipt_retries)) + return script + + +def send_contract_ether_and_verify( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + contract_address: str, + value_wei: int, +) -> Dict[str, object]: + try: + raw_transaction = signed_contract_call_raw_transaction( + test, + geth_service, + contract_address, + "0x", + gas=100000, + value_wei=value_wei, + ) + except Exception as exc: + return record(test, name, geth_service, "prepare contract receive transaction", False, "", str(exc)) + + script = send_contract_ether_script(contract_address, raw_transaction) + result = test.geth_eval(geth_service, script, timeout=180) + stdout = str(result["stdout"]) + stderr = str(result["stderr"]) + if result["exit"] != 0: + return record(test, name, geth_service, "contract receive transaction", False, stdout, stderr) + + try: + data = extract_geth_json(test, result) + if not data.get("txHash"): + raise ValueError("missing transaction hash") + if data.get("receiptBlockNumber") in (None, ""): + raise ValueError("transaction was not included in a block") + if data.get("receiptStatus") not in ("0x1", "1", 1, True, None): + raise ValueError("transaction receipt status is {}".format(data.get("receiptStatus"))) + contract_delta = int(data["contractDelta"]) + sender_delta = int(data["senderDelta"]) + if contract_delta != value_wei: + raise ValueError("contract delta {} != {}".format(contract_delta, value_wei)) + if sender_delta < value_wei: + raise ValueError("sender delta {} is smaller than transfer value {}".format(sender_delta, value_wei)) + except Exception as exc: + return record(test, name, geth_service, "contract receive transaction", False, stdout, str(exc)) + + summary = { + "sender": data["sender"], + "contract": data["contract"], + "tx_hash": data["txHash"], + "block_number": data["receiptBlockNumber"], + "sender_delta_wei": data["senderDelta"], + "contract_delta_wei": data["contractDelta"], + "value_wei": str(value_wei), + } + return record(test, name, geth_service, "contract receive transaction", True, json.dumps(summary, sort_keys=True)) + + +def claim_funds_and_verify( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + contract_address: str, + recipient: str, + amount_wei: int, +) -> Dict[str, object]: + try: + data_hex = encode_claim_funds(recipient, amount_wei) + raw_transaction = signed_contract_call_raw_transaction(test, geth_service, contract_address, data_hex) + except Exception as exc: + return record(test, name, geth_service, "prepare claimFunds transaction", False, "", str(exc)) + + script = claim_funds_script(contract_address, recipient, raw_transaction) + result = test.geth_eval(geth_service, script, timeout=180) + stdout = str(result["stdout"]) + stderr = str(result["stderr"]) + if result["exit"] != 0: + return record(test, name, geth_service, "claimFunds transaction", False, stdout, stderr) + + try: + data = extract_geth_json(test, result) + if not data.get("txHash"): + raise ValueError("missing transaction hash") + if data.get("receiptBlockNumber") in (None, ""): + raise ValueError("transaction was not included in a block") + if data.get("receiptStatus") not in ("0x1", "1", 1, True, None): + raise ValueError("transaction receipt status is {}".format(data.get("receiptStatus"))) + recipient_delta = int(data["recipientDelta"]) + contract_delta = int(data["contractDelta"]) + if recipient_delta != amount_wei: + raise ValueError("recipient delta {} != {}".format(recipient_delta, amount_wei)) + if contract_delta != amount_wei: + raise ValueError("contract delta {} != {}".format(contract_delta, amount_wei)) + except Exception as exc: + return record(test, name, geth_service, "claimFunds transaction", False, stdout, str(exc)) + + summary = { + "contract": data["contract"], + "recipient": data["recipient"], + "tx_hash": data["txHash"], + "block_number": data["receiptBlockNumber"], + "contract_delta_wei": data["contractDelta"], + "recipient_delta_wei": data["recipientDelta"], + "value_wei": str(amount_wei), + } + return record(test, name, geth_service, "claimFunds transaction", True, json.dumps(summary, sort_keys=True)) + + +def main() -> int: + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D21 has POS Geth nodes for deployment checks", + expected_count=3, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + test.require_ethereum_services( + "D21 has POS Lighthouse beacon nodes", + expected_count=3, + class_contains="Ethereum-POS-Beacon", + consensus="POS", + ) + test.require_ethereum_services( + "D21 has POS genesis validator clients", + expected_count=9, + class_contains="Ethereum-POS-Validator", + role="validator_at_genesis", + consensus="POS", + ) + utility_services = test.require_ethereum_services( + "D21 has one Utility service", + expected_count=1, + class_contains="EthUtilityServer", + ) + test.require_ethereum_services( + "D21 has one Faucet service", + expected_count=1, + class_contains="FaucetService", + ) + + if not geth_nodes or not utility_services: + test.write_summary("d21-deploy-contract-runtime-test.json") + return test.exit_code() + + geth = geth_nodes[0] + utility = utility_services[0] + + contract_paths = wait_for_json_file( + test, + "D21 Utility registered contract files", + utility, + "/utility_server/contracts/contract_file_paths.txt", + lambda data: isinstance(data, dict) and CONTRACT_NAME in data, + retries=5, + interval=1, + ) + if isinstance(contract_paths, dict): + command = "test -s /utility_server/contracts/test.abi && test -s /utility_server/contracts/test.bin" + result = test.exec(utility, command, timeout=30) + record( + test, + "D21 Utility imported contract ABI and bytecode", + utility, + command, + result["exit"] == 0, + str(result["stdout"]), + str(result["stderr"]), + ) + + account_data = wait_for_json_file( + test, + "D21 Utility created deployer account", + utility, + "/utility_server/account.json", + lambda data: isinstance(data, dict) and str(data.get("address", "")).startswith("0x"), + retries=120, + interval=5, + ) + if isinstance(account_data, dict): + deployer_address = str(account_data.get("address")) + wait_for_balance_at_least( + test, + "D21 deployer account is funded by Faucet", + geth, + deployer_address, + 1, + retries=60, + interval=5, + ) + + deployed_contracts = wait_for_json_file( + test, + "D21 Utility recorded deployed contract address", + utility, + "/utility_server/deployed_contracts/contract_address.txt", + lambda data: isinstance(data, dict) and re.fullmatch(r"0x[0-9a-fA-F]{40}", str(data.get(CONTRACT_NAME, ""))) is not None, + retries=120, + interval=5, + ) + contract_address = None + if isinstance(deployed_contracts, dict): + contract_address = str(deployed_contracts.get(CONTRACT_NAME)) + + if contract_address: + run_utility_http_check( + test, + "D21 Utility /contracts_info returns deployed contract", + utility, + "/contracts_info", + expect_text=contract_address, + ) + run_utility_http_check( + test, + "D21 Utility /contracts_info?name=test returns deployed contract", + utility, + "/contracts_info?name={}".format(CONTRACT_NAME), + expect_text=contract_address, + ) + run_utility_http_check( + test, + "D21 Utility /all returns deployed contract", + utility, + "/all", + expect_text=contract_address, + ) + run_utility_http_check( + test, + "D21 Utility can manually register contract metadata", + utility, + "/register_contract", + method="POST", + payload={"contract_name": MANUAL_CONTRACT_NAME, "contract_address": MANUAL_CONTRACT_ADDRESS}, + expect_text=MANUAL_CONTRACT_ADDRESS, + ) + run_utility_http_check( + test, + "D21 Utility returns manually registered contract metadata", + utility, + "/contracts_info?name={}".format(MANUAL_CONTRACT_NAME), + expect_text=MANUAL_CONTRACT_ADDRESS, + ) + + code = wait_for_contract_code( + test, + "D21 deployed contract has bytecode on chain", + geth, + contract_address, + retries=60, + interval=5, + ) + if code is not None: + receive_result = send_contract_ether_and_verify( + test, + "D21 Crowdfunding receive() accepts 1 ETH", + geth, + contract_address, + CONTRACT_FUND_WEI, + ) + if receive_result["status"] == "passed": + claim_funds_and_verify( + test, + "D21 Crowdfunding claimFunds() transfers contract balance", + geth, + contract_address, + CLAIM_RECIPIENT, + CLAIM_WEI, + ) + + test.write_summary("d21-deploy-contract-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D22_oracle/README.md b/examples/blockchain/D22_oracle/README.md index f963bf0f8..26c3b14e7 100644 --- a/examples/blockchain/D22_oracle/README.md +++ b/examples/blockchain/D22_oracle/README.md @@ -1,106 +1,354 @@ # Oracle -This example demonstrates how a blockchain can -access data from the world outside. Blockchain -programs (i.e., smart contract) cannot directly -access external data. It has to be done using -a special mechanism called oracle, -which is a means for smart contracts to access data from -the world outside the blockchain. It typically involves -a smart contract (running inside the blockchain) and -an external node running outside of the blockchain. -The external node take data from the outside world and -put it into the blockchain. - -This example is created to demonstrate how oracle works. -We have created two new nodes, one for running the oracle node (getting -data from external sources) and the other for running the -user program (getting data from the oracle). +This example demonstrates how a blockchain application can use an +oracle to access data from outside the blockchain. Smart contracts run +inside the blockchain and cannot directly fetch external data, such as +prices, weather, exchange rates, or web API results. An oracle bridges +this gap by running an external program that fetches or generates data +and writes it back to a smart contract through a transaction. +This example uses either a PoA blockchain based on +[D00_ethereum_poa](../D00_ethereum_poa/) or a PoS blockchain based on +[D01_ethereum_pos](../D01_ethereum_pos/). It adds two application nodes: +an `oracle_node` and an `oracle_user`. -## Oracle Node -The main purpose of this node is to fetch external data and save -the data to the blockchain. When the emulator starts, -this node will automatically carry out -the following tasks: +## Table of Content -- Create an account, and fund it using Faucet -- Deploy the oracle contract -- Register the oracle address with the utility server -- Start the server to listen to the update-price message, and do the - following once such a message is received: +- [How to Run](#how-to-run) +- [Why Oracle Is Needed](#why-oracle-is-needed) +- [Example Architecture](#example-architecture) +- [The Oracle Contract](#the-oracle-contract) +- [Oracle Node](#oracle-node) +- [User Node](#user-node) +- [How the Oracle Workflow Works](#how-the-oracle-workflow-works) +- [Observe the Oracle Workflow](#observe-the-oracle-workflow) +- [Important Notes](#important-notes) - - Fetch the data from outside. In this example, for the sake of - simplicity, we just randomly generate a price value. In practice, - the data is obtained from an external data source - - Write the price data to the oracle contract by invoking the `setPrice` - function. This will trigger a transaction +## How to Run -## User Node +Run the following commands from this example directory: + +```bash +python simple_oracle.py [poa|pos] [amd|arm] +cd output +docker compose up +``` + +Arguments: + +- `poa`: Use the PoA blockchain from `D00_ethereum_poa`. This is the + default. +- `pos`: Use the PoS blockchain from `D01_ethereum_pos`. +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. + +Examples: + +```bash +python simple_oracle.py poa amd +``` + +```bash +python simple_oracle.py pos arm +``` + +The script loads the selected blockchain component, adds `oracle_node` +and `oracle_user` hosts to AS164, installs helper scripts on both hosts, +and generates the Docker output. + + +## Why Oracle Is Needed -This node demonstrates how oracle is used. Typically, a user contract -would interact with the oracle contract to get the external data. -For the simplicity, we directly interact with the oracle contract -using a Python program. The program is already installed on -the user node, and we need to go to this container, -manually run this program: `user_get_price.py`. The program will -do the followings: - -- Get the oracle contract address from the Utility server -- Invoke the oracle contract' `updatePrice` function. This will trigger - a transaction -- Sleep for a few seconds, then invoke the oracle contract's - `getPrice` function, and print out the price. - This is just a local call. -- The program will repeat the whole updatePrice-getPrice process. -``` -root@8631f635cd20 /oracle # python3 user_get_price.py -== EthereumHelper: Successfully connected to http://10.150.0.74:8545 -... - INFO - Utility server is connected. -... - INFO - Successfully invoke updatePrice(). -Price 30 -Price 30 -Price 43 -Price 43 -Price 43 -... - INFO - Successfully invoke updatePrice(). -Price 43 -Price 43 -Price 59 -Price 59 +Smart contracts cannot directly access external data. For example, a +contract cannot safely call a web API such as a price feed, because +every blockchain node must execute the same transaction and reach the +same result. If different nodes receive different API responses, the +blockchain cannot reach consensus. + +An oracle solves this problem by moving external data access outside the +smart contract: + +```text +external data source + -> oracle node outside the blockchain + -> transaction to oracle smart contract + -> data stored on chain +``` + +In this example, the oracle node does not fetch a real external price. +For simplicity, it generates a random price and writes it into the +Oracle contract. The same workflow can be adapted to fetch real external +data. + + +## Example Architecture + +This example uses the following components: + +| Component | Role | +| --- | --- | +| Geth node | Provides Ethereum JSON-RPC, accepts transactions, executes EVM code, and stores contract state | +| Beacon/Validator nodes | Used in PoS mode to confirm blocks and transactions | +| Faucet server | Funds newly created oracle and user accounts so they can pay gas | +| Utility server | Stores the Oracle contract address under a known name | +| `oracle_node` | Deploys the Oracle contract, listens for update requests, and writes prices on chain | +| `oracle_user` | Requests price updates and reads the current price from the Oracle contract | + +The D22 script creates two new hosts in AS164: + +```python +oracle_user = as_.createHost('oracle_user').joinNetwork('net0') +oracle_node = as_.createHost('oracle_node').joinNetwork('net0') ``` +For PoS mode, the application scripts use the Geth hostnames from the +D01 blockchain: + +```python +ETH_NODE1 = 'gethnode1.net' +ETH_NODE2 = 'gethnode2.net' +``` + +For PoA mode, they use hostnames from the D00 blockchain: + +```python +ETH_NODE1 = 'eth3.net' +ETH_NODE2 = 'eth5.net' +``` + + ## The Oracle Contract -The oracle contract in this example is very simple. It mainly has -the following three functions: +The Oracle contract is located in: +```text +contract/Oracle.sol ``` -function getPrice() public view returns (uint) { - return price; -} -event UpdatePriceMessage(address indexed _from); +The contract is simple: + +```solidity +contract Oracle { + address public owner; + uint256 price; + + constructor(){ + owner = msg.sender; + price = 0; + } + + receive() external payable { } -function updatePrice() public payable { - emit UpdatePriceMessage(msg.sender); + function getPrice() public view returns (uint) { + return price; + } + + event UpdatePriceMessage(address indexed _from); + + function updatePrice() public payable { + emit UpdatePriceMessage(msg.sender); + } + + function setPrice(uint p) public { + price = p; + } } +``` + +The important functions are: + +- `updatePrice()`: emits an `UpdatePriceMessage` event. This is how a + user requests a new price. +- `setPrice(uint p)`: writes a new price to the contract state. In this + example, the oracle node calls this function. +- `getPrice()`: returns the current price. This is a local read-only + call and does not require a transaction. + +The contract also defines an `UpdatePriceMessage` event. The oracle node +monitors this event to know when a user wants the price to be updated. + + +## Oracle Node + +The `oracle_node` container represents the off-chain oracle service. It +is not a special Ethereum node. It is a normal host that runs Python +programs and talks to the blockchain through Geth JSON-RPC. + +When the container starts, it runs: + +```bash +bash /oracle/oracle_node_start.sh +``` + +The script runs two programs: + +```bash +python3 deploy_oracle_contract.py +python3 oracle_node_set_price.py +``` + +The first program performs the setup work: + +1. Connects to the blockchain through Geth JSON-RPC. +2. Creates a new Ethereum account for the oracle node. +3. Requests funds from the Faucet server. +4. Deploys the Oracle contract. +5. Saves the account and Oracle contract address to + `/oracle/oracle_account.json`. +6. Registers the Oracle contract address with the Utility server using + the name `oracle-contract`. + +The second program runs in a loop. It monitors the Oracle contract for +`UpdatePriceMessage` events. When it detects an update request, it +generates a random price and invokes `setPrice(price)` on the Oracle +contract. This is a real Ethereum transaction and requires gas. + +## User Node + +The `oracle_user` container represents a user application. It is also a +normal host that runs Python programs and talks to the blockchain +through Geth JSON-RPC. + +When the container starts, it automatically runs: + +```bash +python3 /oracle/user_create_account.py +``` + +This script creates a user account, requests funds from the Faucet +server, and stores the account data in: + +```text +/oracle/user_account.json +``` + +The user program is not started automatically. After the emulator has +started, log into the `oracle_user` container and run: + +```bash +python3 /oracle/user_get_price.py +``` + +This program performs the following steps: + +1. Gets the Oracle contract address from the Utility server by querying + the name `oracle-contract`. +2. Invokes `updatePrice()` on the Oracle contract. This sends a + transaction and emits an `UpdatePriceMessage` event. +3. Repeatedly calls `getPrice()` to read and print the current price. + + +## How the Oracle Workflow Works + +The full oracle workflow is: + +```text +oracle_user invokes updatePrice() + -> Oracle contract emits UpdatePriceMessage + -> oracle_node detects the event + -> oracle_node generates a random price + -> oracle_node invokes setPrice(price) + -> Oracle contract stores the new price + -> oracle_user calls getPrice() + -> user sees the updated price +``` + +There are two different types of contract interaction in this workflow: + +- `updatePrice()` and `setPrice()` are transactions. They are signed by + accounts, sent to Geth, included in blocks, and require gas. +- `getPrice()` is a local read-only call. It does not change blockchain + state and does not require gas. + +This distinction is important when building applications with smart +contracts. + + +## Observe the Oracle Workflow + +After starting the emulator, find the oracle containers: + +```bash +docker ps | grep -i oracle +``` + +Check the `oracle_node` logs: + +```bash +docker logs +``` + +The logs should show that the oracle account was funded, the Oracle +contract was deployed, and the contract address was registered with the +Utility server. The oracle node then waits for update events. + +Check the `oracle_user` account file: + +```bash +docker exec -it bash +cat /oracle/user_account.json +``` + +Run the user program: + +```bash +python3 /oracle/user_get_price.py +``` + +The output should show that `updatePrice()` was invoked and that the +program is reading prices from the Oracle contract: + +```text +Successfully invoke updatePrice(). +Price 0 +Price 37 +Price 37 +Price 58 +``` + +The exact prices are random. The price changes when the oracle node +handles an update event and sends a `setPrice()` transaction. + +You can also query the Utility server to find the Oracle contract +address: + +```bash +curl http://:5000/contracts_info?name=oracle-contract +``` + +To verify that the Oracle contract exists on chain, query its bytecode +through Geth JSON-RPC: + +```bash +curl -s -X POST \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_getCode","params":["","latest"],"id":1}' \ + http://:8545 +``` + +If the result is not `"0x"`, the contract bytecode is stored on chain. + + +## Important Notes + +This example demonstrates the oracle workflow, not a production oracle +security design. + +In this teaching contract, anyone can call `setPrice(uint p)`: + +```solidity function setPrice(uint p) public { price = p; } ``` -When the user needs to get the most recent price data, -it invokes the `updatePrice` function. This will emit -a `UpdatePriceMessage`, which informs the oracle node -to go fetch the data. The oracle node then -writes the price data to the oracle contract by invoking -`setPrice`. After the data is stored on the blockchain, -the user can use a local call `getPrice` to read the -price value. - - +A real oracle contract should restrict who can update the price, for +example by checking that `msg.sender` is an authorized oracle address. +The example also generates a random price instead of fetching data from +a real external source. In a real application, the oracle node would +fetch data from an API, a database, a sensor, or another trusted data +source, validate it, and then submit it to the blockchain. diff --git a/examples/blockchain/D22_oracle/code/EthereumHelper.py b/examples/blockchain/D22_oracle/code/EthereumHelper.py index 07e409d83..1c4685440 100644 --- a/examples/blockchain/D22_oracle/code/EthereumHelper.py +++ b/examples/blockchain/D22_oracle/code/EthereumHelper.py @@ -1,182 +1,169 @@ -from web3 import Web3, HTTPProvider -from web3.middleware import geth_poa_middleware -import logging -import time +from web3 import Web3 +try: + from web3.middleware import geth_poa_middleware +except ImportError: + try: + from web3.middleware import ExtraDataToPOAMiddleware as geth_poa_middleware + except ImportError: + geth_poa_middleware = None +import time from sys import stderr class EthereumHelper: - _web: Web3 + _web3: Web3 _chain_id: int - _max_fee: float + _max_fee: float _max_priority_fee: float - def __init__(self, chain_id:int=1337): - """! - @brief constructor. - """ - + def __init__(self, chain_id: int = 1337): self._chain_id = chain_id - self._max_fee = 3.0 - self._max_priority_fee = 2.0 + self._max_fee = 3.0 + self._max_priority_fee = 2.0 - return - def __log(self, message: str): - """! - @brief log to stderr. - - @param message message. - """ - print('== EthereumHelper: ' + message, file=stderr) - + print("== EthereumHelper: " + message, file=stderr) + + @staticmethod + def _is_connected(web3: Web3) -> bool: + method = getattr(web3, "is_connected", None) or getattr(web3, "isConnected") + return bool(method()) + + @staticmethod + def _to_wei(amount, unit: str) -> int: + method = getattr(Web3, "to_wei", None) or getattr(Web3, "toWei") + return int(method(amount, unit)) + + @staticmethod + def _raw_transaction(signed_tx): + return getattr(signed_tx, "raw_transaction", None) or getattr(signed_tx, "rawTransaction") + + @staticmethod + def _private_key_hex(account) -> str: + key = getattr(account, "key", None) or getattr(account, "privateKey") + key_hex = key.hex() + if not key_hex.startswith("0x"): + key_hex = "0x" + key_hex + return key_hex + + def _block_number(self) -> int: + value = getattr(self._web3.eth, "block_number", None) + if value is not None: + return int(value) + return int(self._web3.eth.blockNumber) + + def _get_transaction_count(self, address: str) -> int: + method = getattr(self._web3.eth, "get_transaction_count", None) or getattr( + self._web3.eth, "getTransactionCount" + ) + return int(method(address)) + + def _send_raw_transaction(self, raw_transaction): + method = getattr(self._web3.eth, "send_raw_transaction", None) or getattr( + self._web3.eth, "sendRawTransaction" + ) + return method(raw_transaction) + + @staticmethod + def _build_transaction(function, transaction_info): + method = getattr(function, "build_transaction", None) or getattr(function, "buildTransaction") + return method(transaction_info) def create_account(self): - """! - @brief Create an account - @return account address and private ke - """ - account = self._web3.eth.account.create() address = account.address - key = account.privateKey.hex() + key = self._private_key_hex(account) return address, key - - - def connect_to_blockchain(self, url:str, isPOA=False, wait=True): - """! - @brief Connect to a blockchain node - @param url The URL of the node (e.g., http://10.150.0.71:8545) - @param isPOA Is the POA used for the consensus protocol (Proof-Of-Authority)? - @returns Return self, for the purpose of API chaining. - """ - + def connect_to_blockchain(self, url: str, isPOA=False, wait=True): self._url = url - + while True: - self._web3 = Web3(Web3.HTTPProvider(url)) - if isPOA: - self._web3.middleware_onion.inject(geth_poa_middleware, layer=0) - - if self._web3.isConnected(): - self.__log("Successfully connected to {}".format(url)) - break - else: - if wait: - self.__log("Failed to connect to {}, retrying ...".format(url)) - time.sleep(10) + self._web3 = Web3(Web3.HTTPProvider(url)) + if isPOA and geth_poa_middleware is not None: + self._web3.middleware_onion.inject(geth_poa_middleware, layer=0) + + if self._is_connected(self._web3): + self.__log("Successfully connected to {}".format(url)) + break + if wait: + self.__log("Failed to connect to {}, retrying ...".format(url)) + time.sleep(10) + else: + break return self._web3 - def wait_for_blocknumber(self, block_number=5): - """! - @brief Wait for the blockchain to reach the specified block number - """ - - block_now = self._web3.eth.blockNumber + block_now = self._block_number() while block_now < block_number: - self.__log("Waiting for the block number to reach {} (current: {})".\ - format(block_number, block_now)) + self.__log( + "Waiting for the block number to reach {} (current: {})".format(block_number, block_now) + ) time.sleep(10) - block_now = self._web3.eth.blockNumber - + block_now = self._block_number() - def deploy_contract(self, contract_file, sender_address, sender_key, - amount=0, gas=3000000, wait=True): - """! - @brief Deploy a smart contract - - @returns Return the transaction hash and the contract address. - """ + def deploy_contract(self, contract_file, sender_address, sender_key, amount=0, gas=3000000, wait=True): with open(contract_file) as contract: - data = contract.read() - data = data.strip() + data = contract.read().strip() + if data and not data.startswith("0x"): + data = "0x" + data - tx_hash = self.send_raw_transaction(None, sender_address, sender_key, - amount=amount, data=data, gas=gas) - rx_receipt = None + tx_hash = self.send_raw_transaction(None, sender_address, sender_key, amount=amount, data=data, gas=gas) + tx_receipt = None if wait: - tx_receipt = self._web3.eth.wait_for_transaction_receipt(tx_hash) + tx_receipt = self._web3.eth.wait_for_transaction_receipt(tx_hash, timeout=300) return tx_hash, tx_receipt - def transfer_fund(self, receiver_address, sender_address, sender_key, - amount=0, gas=3000000, wait=True): - """! - @brief Transfer fund to a receiver - @param amount The unit is Ether - @returns Return the transaction hash and receipt - """ - - tx_hash = self.send_raw_transaction(receiver_address, - sender_address, sender_key, - amount=amount, gas=gas) - rx_receipt = None + def transfer_fund(self, receiver_address, sender_address, sender_key, amount=0, gas=3000000, wait=True): + tx_hash = self.send_raw_transaction(receiver_address, sender_address, sender_key, amount=amount, gas=gas) + tx_receipt = None if wait: - tx_receipt = self._web3.eth.wait_for_transaction_receipt(tx_hash) + tx_receipt = self._web3.eth.wait_for_transaction_receipt(tx_hash, timeout=300) return tx_hash, tx_receipt - - - def send_raw_transaction(self, recipient, sender_address, sender_key, - data:str='', amount=0, gas=300000): - """! - @param amount The unit is Ether - """ - + def send_raw_transaction(self, recipient, sender_address, sender_key, data: str = "", amount=0, gas=300000): transaction = { - 'nonce': self._web3.eth.getTransactionCount(sender_address), - 'from': sender_address, - 'to': recipient, - 'value': Web3.toWei(amount, 'ether'), - 'chainId': self._chain_id, - 'gas': gas, - 'maxFeePerGas': Web3.toWei(self._max_fee, 'gwei'), - 'maxPriorityFeePerGas': Web3.toWei(self._max_priority_fee, 'gwei'), - 'data': data + "nonce": self._get_transaction_count(sender_address), + "from": sender_address, + "value": self._to_wei(amount, "ether"), + "chainId": self._chain_id, + "gas": gas, + "maxFeePerGas": self._to_wei(self._max_fee, "gwei"), + "maxPriorityFeePerGas": self._to_wei(self._max_priority_fee, "gwei"), + "data": data, } + if recipient is not None: + transaction["to"] = recipient signed_tx = self._web3.eth.account.sign_transaction(transaction, sender_key) - tx_hash = self._web3.eth.sendRawTransaction(signed_tx.rawTransaction) + tx_hash = self._send_raw_transaction(self._raw_transaction(signed_tx)) return tx_hash - def invoke_contract_function(self, function, sender_address, sender_key, - amount=0, gas=3000000, wait=True): - """! - @brief Invoke a contract function via a transaction. - - @param function The pre-built function - @param amount The unit is Ether - @param wait Wait for the transaction receipt - - @returns Return the transaction hash. - """ - + def invoke_contract_function(self, function, sender_address, sender_key, amount=0, gas=3000000, wait=True): assert self._web3 is not None assert function is not None transaction_info = { - 'nonce': self._web3.eth.getTransactionCount(sender_address), - 'from': sender_address, - 'value': Web3.toWei(amount, 'ether'), - 'chainId': self._chain_id, - 'gas': gas, - 'maxFeePerGas': Web3.toWei(self._max_fee, 'gwei'), - 'maxPriorityFeePerGas': Web3.toWei(self._max_priority_fee, 'gwei') + "nonce": self._get_transaction_count(sender_address), + "from": sender_address, + "value": self._to_wei(amount, "ether"), + "chainId": self._chain_id, + "gas": gas, + "maxFeePerGas": self._to_wei(self._max_fee, "gwei"), + "maxPriorityFeePerGas": self._to_wei(self._max_priority_fee, "gwei"), } - transaction = function.buildTransaction(transaction_info) - signed_tx = self._web3.eth.account.sign_transaction(transaction, sender_key) - tx_hash = self._web3.eth.sendRawTransaction(signed_tx.rawTransaction) + transaction = self._build_transaction(function, transaction_info) + signed_tx = self._web3.eth.account.sign_transaction(transaction, sender_key) + tx_hash = self._send_raw_transaction(self._raw_transaction(signed_tx)) - tx_receipt = None + tx_receipt = None if wait: tx_receipt = self._web3.eth.wait_for_transaction_receipt(tx_hash, timeout=300) return tx_hash, tx_receipt - diff --git a/examples/blockchain/D22_oracle/code/oracle_node_set_price.py b/examples/blockchain/D22_oracle/code/oracle_node_set_price.py index 1c37e77c4..43092365a 100755 --- a/examples/blockchain/D22_oracle/code/oracle_node_set_price.py +++ b/examples/blockchain/D22_oracle/code/oracle_node_set_price.py @@ -1,8 +1,6 @@ #!/bin/env python3 import os, json, logging, time, random, socket -from web3 import Web3 -from FaucetHelper import FaucetHelper from EthereumHelper import EthereumHelper @@ -25,46 +23,72 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def block_number(web3): + value = getattr(web3.eth, "block_number", None) + if value is not None: + return int(value) + return int(web3.eth.blockNumber) + + +def event_logs(event, from_block, to_block): + if hasattr(event, "get_logs"): + try: + return event.get_logs(fromBlock=from_block, toBlock=to_block) + except TypeError: + return event.get_logs(from_block=from_block, to_block=to_block) + return event.getLogs(fromBlock=from_block, toBlock=to_block) + + eth = EthereumHelper(chain_id=chain_id) web3 = eth.connect_to_blockchain(eth_url, isPOA=True) -# Get data +# Get data with open('./oracle_account.json', 'r') as f: - data = json.load(f) + data = json.load(f) account_address = data['account_address'] -account_key = data['private_key'] -oracle_address = data['oracle_address'] +account_key = data['private_key'] +oracle_address = data['oracle_address'] -# Invoke the oracle contract to set price +# Invoke the oracle contract to set price with open('../contract/Oracle.abi', 'r') as f: oracle_abi = f.read() oracle_contract = web3.eth.contract(address=oracle_address, abi=oracle_abi) +previous_blocknumber = block_number(web3) -previous_blocknumber = 0 while True: - # Monitor event - logs = oracle_contract.events.UpdatePriceMessage().getLogs(fromBlock="latest") + latest_blocknumber = block_number(web3) + from_block = previous_blocknumber + 1 + if latest_blocknumber < from_block: + print("Sleeping for 10 seconds ...") + time.sleep(10) + continue + + try: + logs = event_logs(oracle_contract.events.UpdatePriceMessage(), from_block, latest_blocknumber) + except Exception as exc: + logging.warning("Failed to read oracle update events: %s", exc) + time.sleep(10) + continue + if not logs: + previous_blocknumber = latest_blocknumber print("Sleeping for 10 seconds ...") time.sleep(10) continue for log in logs: - blocknumber = log['blockNumber'] - if blocknumber <= previous_blocknumber: - pass - else: - previous_blocknumber = blocknumber - - # Invoke the setPrice() - price = random.randint(0, 99) - setPriceFunc = oracle_contract.functions.setPrice(price) - _, receipt = eth.invoke_contract_function(setPriceFunc, account_address, account_key) - - if receipt['status'] == 0: - logging.error("Failed to set price in the oracle contract.") - else: - logging.info("Successfully set price in the oracle contract.") + previous_blocknumber = max(previous_blocknumber, int(log['blockNumber'])) + + # Invoke the setPrice() + price = random.randint(0, 99) + setPriceFunc = oracle_contract.functions.setPrice(price) + _, receipt = eth.invoke_contract_function(setPriceFunc, account_address, account_key) + if receipt['status'] == 0: + logging.error("Failed to set price in the oracle contract.") + else: + logging.info("Successfully set price in the oracle contract.") + previous_blocknumber = max(previous_blocknumber, latest_blocknumber) diff --git a/examples/blockchain/D22_oracle/example.yaml b/examples/blockchain/D22_oracle/example.yaml new file mode 100644 index 000000000..2d6436ffb --- /dev/null +++ b/examples/blockchain/D22_oracle/example.yaml @@ -0,0 +1,155 @@ +id: blockchain-d22-oracle-pos +name: Ethereum PoS Oracle +description: A D01-style Ethereum proof-of-stake chain with an oracle node, user node, Utility registration, and Oracle contract event workflow tests. +runner: blockchain +script: simple_oracle.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - faucet + - utility-server + - oracle-contract + - oracle-node + - oracle-user + +compile: + enabled: true + output: output + args: + - --consensus + - pos + - --no-ether-view + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D22 POS Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D22 POS Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D22 POS genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D22 POS beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D22 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D22 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + +probes: + - name: D22 POS Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: D22 POS Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: D22 POS Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: D22 POS beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: D22 Geth IPC exposes a funded local account + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: "geth attach --exec 'eth.accounts.length > 0' | grep -q true" + retries: 30 + interval: 3 + + - name: D22 Faucet HTTP health endpoint returns OK + type: ethereum-exec + class_contains: FaucetService + command: >- + python3 -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:80/", timeout=5).read().decode())' | grep -q '^OK$' + retries: 60 + interval: 5 + timeout: 15 + + - name: D22 Utility HTTP health endpoint returns OK + type: ethereum-exec + class_contains: EthUtilityServer + command: >- + python3 -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:5000/", timeout=5).read().decode())' | grep -q '^OK$' + retries: 90 + interval: 5 + timeout: 20 + + - name: D22 POS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + +test_programs: + - name: D22 oracle deployment and workflow validation + script: test_runtime.py + timeout: 1800 diff --git a/examples/blockchain/D22_oracle/simple_oracle.py b/examples/blockchain/D22_oracle/simple_oracle.py index 75368c556..405a8b427 100755 --- a/examples/blockchain/D22_oracle/simple_oracle.py +++ b/examples/blockchain/D22_oracle/simple_oracle.py @@ -1,139 +1,239 @@ #!/usr/bin/env python3 # encoding: utf-8 -import sys -from seedemu import * -from examples.blockchain.D05_ethereum_small import ethereum_small -import os, sys - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -# These hostnames are already set up in D05_ethereum_small -ETH_NODE1 = 'eth-3.net' -ETH_NODE2 = 'eth-5.net' -FAUCET = 'faucet.net' -UTILITY = 'utility.net' -FAUCET_PORT = 80 -UTILITY_PORT = 5000 -ETH_PORT = 8545 -CHAIN_ID = 1337 -ORACLE_NAME = 'oracle-contract' - -def get_file_content(filename): - """! - @brief Get the content of a file - """ - with open(filename, "r") as f: - return f.read() - -def installSoftware(node: Node): - """ - @brief Install the software and library - """ - software_list = ['curl', 'python3', 'python3-pip'] - for software in software_list: - node.addSoftware(software) - - node.addBuildCommand('pip3 install web3==5.31.1') - - node.setFile('/oracle/EthereumHelper.py', - get_file_content('./code/EthereumHelper.py')) - - node.setFile('/oracle/FaucetHelper.py', - get_file_content('./code/FaucetHelper.py')) - - node.setFile('/oracle/UtilityServerHelper.py', - get_file_content('./code/UtilityServerHelper.py')) - - +from __future__ import annotations -emu = Emulator() - -# Run and load the pre-built ethereum component -local_dump_path = './ethereum-small.bin' -ethereum_small.run(dumpfile=local_dump_path) -emu.load(local_dump_path) - -# Add two new nodes for oracle node and user node -base = emu.getLayer('Base') -as_ = base.getAutonomousSystem(150) -oracle_user = as_.createHost('oracle_user').joinNetwork('net0') -oracle_node = as_.createHost('oracle_node').joinNetwork('net0') - -# Install needed software -installSoftware(oracle_user) -installSoftware(oracle_node) - -# Set up the user node -oracle_user.setFile('/contract/Oracle.abi', - get_file_content('./contract/Oracle.abi')) - -oracle_user.setFile('/oracle/user_create_account.py', - get_file_content('./code/user_create_account.py').format( - chain_id=CHAIN_ID, - eth_node=ETH_NODE1, eth_port=ETH_PORT, - faucet_server=FAUCET, faucet_port=FAUCET_PORT - )) - -oracle_user.setFile('/oracle/user_get_price.py', - get_file_content('./code/user_get_price.py').format( - chain_id=CHAIN_ID, - faucet_server=FAUCET, faucet_port=FAUCET_PORT, - utility_server=UTILITY, utility_port=UTILITY_PORT, - eth_node=ETH_NODE1, eth_port=ETH_PORT, - oracle_contract_name=ORACLE_NAME - )) - -oracle_user.appendStartCommand('python3 /oracle/user_create_account.py &') +import argparse +import os +from contextlib import contextmanager +from pathlib import Path +import sys +import tempfile -# Set up the oracle node -oracle_node.setFile('/contract/Oracle.abi', - get_file_content('./contract/Oracle.abi')) -oracle_node.setFile('/contract/Oracle.bin', - get_file_content('./contract/Oracle.bin')) -oracle_node.setFile('/oracle/deploy_oracle_contract.py', - get_file_content('./code/deploy_oracle_contract.py').format( - chain_id=CHAIN_ID, - faucet_server=FAUCET, faucet_port=FAUCET_PORT, - utility_server=UTILITY, utility_port=UTILITY_PORT, - eth_node=ETH_NODE1, eth_port=ETH_PORT, - oracle_contract_name=ORACLE_NAME - )) +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) -oracle_node.setFile('/oracle/oracle_node_set_price.py', - get_file_content('./code/oracle_node_set_price.py').format( - chain_id=CHAIN_ID, - faucet_server=FAUCET, faucet_port=FAUCET_PORT, - utility_server=UTILITY, utility_port=UTILITY_PORT, - eth_node=ETH_NODE1, eth_port=ETH_PORT - )) +from seedemu import * +from examples.blockchain.D00_ethereum_poa import ethereum_poa +from examples.blockchain.D01_ethereum_pos import ethereum_pos +from seedemu.services.EthereumService import * -oracle_node.setFile('/oracle/oracle_node_start.sh', - get_file_content('./code/oracle_node_start.sh')) +@contextmanager +def pushd(directory: Path): + previous = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(previous) -oracle_node.appendStartCommand('bash /oracle/oracle_node_start.sh &') +def get_file_content(filename: str) -> str: + return (SCRIPT_DIR / filename).read_text(encoding="utf-8") -emu.render() -docker = Docker(etherViewEnabled=True, platform=platform) -emu.compile(docker, './output', override=True) +def installSoftware(node: Node): + software_list = ["curl", "python3", "python3-pip", "build-essential", "python3-dev"] + for software in software_list: + node.addSoftware(software) + node.addBuildCommand( + "pip3 install --break-system-packages web3==6.20.4 requests || pip3 install web3==6.20.4 requests" + ) + + node.setFile("/oracle/EthereumHelper.py", get_file_content("code/EthereumHelper.py")) + node.setFile("/oracle/FaucetHelper.py", get_file_content("code/FaucetHelper.py")) + node.setFile("/oracle/UtilityServerHelper.py", get_file_content("code/UtilityServerHelper.py")) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D22 oracle example.") + parser.add_argument("legacy_args", nargs="*", help="legacy form: [poa|pos] [amd|arm]") + parser.add_argument("--consensus", choices=["poa", "pos"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--ether-view", + dest="ether_view_enabled", + action="store_true", + default=True, + help="enable Eth Explorer output in generated Docker files (default)", + ) + parser.add_argument( + "--no-ether-view", + dest="ether_view_enabled", + action="store_false", + help="disable Eth Explorer output; useful for CI builds", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + + legacy_consensus = None + legacy_platform = None + for item in args.legacy_args: + value = item.lower() + if value in ("poa", "pos") and legacy_consensus is None: + legacy_consensus = value + elif value in ("amd", "arm") and legacy_platform is None: + legacy_platform = value + else: + parser.error("legacy arguments must be: [poa|pos] [amd|arm]") + + args.consensus = args.consensus or legacy_consensus or "poa" + args.platform = args.platform or legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def load_base_blockchain(consensus: str) -> Emulator: + emu = Emulator() + with tempfile.TemporaryDirectory(prefix="seedemu-d22-") as tempdir: + temp_path = Path(tempdir) + dump_path = temp_path / f"blockchain_{consensus}.bin" + with pushd(temp_path): + if consensus == "pos": + ethereum_pos.run(dumpfile=str(dump_path)) + else: + ethereum_poa.run( + dumpfile=str(dump_path), + hosts_per_as=1, + total_eth_nodes=10, + total_accounts_per_node=1, + ) + emu.load(str(dump_path)) + return emu + + +def build_emulator(consensus: str = "poa") -> Emulator: + if consensus == "pos": + eth_node1 = "gethnode1.net" + else: + eth_node1 = "eth3.net" + + faucet = "faucet.net" + utility = "utility.net" + faucet_port = 80 + utility_port = 5000 + eth_port = 8545 + chain_id = 1337 + oracle_name = "oracle-contract" + + emu = load_base_blockchain(consensus) + + base_layer = emu.getLayer("Base") + as_ = base_layer.getAutonomousSystem(164) + oracle_user = as_.createHost("oracle_user").joinNetwork("net0") + oracle_node = as_.createHost("oracle_node").joinNetwork("net0") + + installSoftware(oracle_user) + installSoftware(oracle_node) + + oracle_user.setFile("/contract/Oracle.abi", get_file_content("contract/Oracle.abi")) + oracle_user.setFile( + "/oracle/user_create_account.py", + get_file_content("code/user_create_account.py").format( + chain_id=chain_id, + eth_node=eth_node1, + eth_port=eth_port, + faucet_server=faucet, + faucet_port=faucet_port, + ), + ) + oracle_user.setFile( + "/oracle/user_get_price.py", + get_file_content("code/user_get_price.py").format( + chain_id=chain_id, + faucet_server=faucet, + faucet_port=faucet_port, + utility_server=utility, + utility_port=utility_port, + eth_node=eth_node1, + eth_port=eth_port, + oracle_contract_name=oracle_name, + ), + ) + oracle_user.appendStartCommand("python3 /oracle/user_create_account.py &") + + oracle_node.setFile("/contract/Oracle.abi", get_file_content("contract/Oracle.abi")) + oracle_node.setFile("/contract/Oracle.bin", get_file_content("contract/Oracle.bin")) + oracle_node.setFile( + "/oracle/deploy_oracle_contract.py", + get_file_content("code/deploy_oracle_contract.py").format( + chain_id=chain_id, + faucet_server=faucet, + faucet_port=faucet_port, + utility_server=utility, + utility_port=utility_port, + eth_node=eth_node1, + eth_port=eth_port, + oracle_contract_name=oracle_name, + ), + ) + oracle_node.setFile( + "/oracle/oracle_node_set_price.py", + get_file_content("code/oracle_node_set_price.py").format( + chain_id=chain_id, + faucet_server=faucet, + faucet_port=faucet_port, + utility_server=utility, + utility_port=utility_port, + eth_node=eth_node1, + eth_port=eth_port, + ), + ) + oracle_node.setFile("/oracle/oracle_node_start.sh", get_file_content("code/oracle_node_start.sh")) + oracle_node.appendStartCommand("bash /oracle/oracle_node_start.sh &") + + return emu + + +def run( + *, + consensus: str = "poa", + dumpfile=None, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + ether_view_enabled: bool = True, +): + emu = build_emulator(consensus=consensus) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + docker = Docker(etherViewEnabled=ether_view_enabled, platform=platform) + emu.compile(docker, str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + consensus=args.consensus, + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ether_view_enabled=args.ether_view_enabled, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D22_oracle/test_runtime.py b/examples/blockchain/D22_oracle/test_runtime.py new file mode 100644 index 000000000..b4d0aba3b --- /dev/null +++ b/examples/blockchain/D22_oracle/test_runtime.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import re +import shlex +import time +from typing import Any, Callable, Dict, Optional + +from seedemu.testing import EthereumRuntimeTest + + +ORACLE_CONTRACT_NAME = "oracle-contract" +ORACLE_USER_ASN = 164 +ORACLE_USER_NODE = "oracle_user" +ORACLE_NODE_ASN = 164 +ORACLE_NODE_NODE = "oracle_node" + + +def service_name(service: object) -> str: + return getattr(service, "name", str(service)) + + +def record( + test: EthereumRuntimeTest, + name: str, + service: object, + command: str, + ok: bool, + stdout: str = "", + stderr: str = "", +) -> Dict[str, object]: + result = { + "name": name, + "service": service_name(service), + "command": command, + "exit": 0 if ok else 1, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if ok else "failed", + } + test.results.append(result) + return result + + +def int_from_geth_output(output: object) -> int: + matches = re.findall(r"\b\d+\b", str(output)) + if not matches: + raise RuntimeError("could not parse integer from geth output: {}".format(output)) + return int(matches[-1]) + + +def parse_geth_int(result: Dict[str, object]) -> int: + for stream_name in ("stdout", "stderr"): + try: + return int_from_geth_output(result[stream_name]) + except RuntimeError: + continue + raise RuntimeError( + "could not parse integer from geth output: stdout={} stderr={}".format( + result["stdout"], result["stderr"] + ) + ) + + +def parse_geth_code(result: Dict[str, object]) -> str: + for stream_name in ("stdout", "stderr"): + lines = [line.strip().strip('"') for line in str(result[stream_name]).splitlines()] + for line in reversed(lines): + if not line: + continue + if re.fullmatch(r"0x[0-9a-fA-F]*", line): + return line + if re.fullmatch(r"[0-9a-fA-F]+", line) and len(line) % 2 == 0: + return "0x" + line + raise RuntimeError( + "could not parse contract code from stdout={} stderr={}".format( + result["stdout"], result["stderr"] + ) + ) + + +def get_balance_wei(test: EthereumRuntimeTest, geth_service: object, address: str) -> int: + result = test.geth_eval(geth_service, 'eth.getBalance("{}").toString(10)'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth balance query failed") + return parse_geth_int(result) + + +def get_transaction_count(test: EthereumRuntimeTest, geth_service: object, address: str) -> int: + result = test.geth_eval(geth_service, 'eth.getTransactionCount("{}")'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth nonce query failed") + return parse_geth_int(result) + + +def get_contract_code(test: EthereumRuntimeTest, geth_service: object, address: str) -> str: + result = test.geth_eval(geth_service, 'eth.getCode("{}")'.format(address), timeout=60) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth code query failed") + return parse_geth_code(result) + + +def wait_for_balance_at_least( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + minimum_wei: int, + *, + retries: int = 60, + interval: int = 5, +) -> Optional[int]: + last_balance: Optional[int] = None + last_error = "" + for attempt in range(1, retries + 1): + try: + last_balance = get_balance_wei(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_balance >= minimum_wei: + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + True, + "balance={} attempts={}".format(last_balance, attempt), + ) + return last_balance + if attempt < retries: + time.sleep(interval) + record( + test, + name, + geth_service, + "eth.getBalance({}) >= {}".format(address, minimum_wei), + False, + "last_balance={}".format(last_balance), + last_error, + ) + return None + + +def wait_for_nonce_above( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + previous_nonce: int, + *, + retries: int = 90, + interval: int = 5, +) -> Optional[int]: + last_nonce: Optional[int] = None + last_error = "" + for attempt in range(1, retries + 1): + try: + last_nonce = get_transaction_count(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_nonce > previous_nonce: + record( + test, + name, + geth_service, + "eth.getTransactionCount({}) > {}".format(address, previous_nonce), + True, + "nonce={} attempts={}".format(last_nonce, attempt), + ) + return last_nonce + if attempt < retries: + time.sleep(interval) + record( + test, + name, + geth_service, + "eth.getTransactionCount({}) > {}".format(address, previous_nonce), + False, + "last_nonce={}".format(last_nonce), + last_error, + ) + return None + + +def wait_for_contract_code( + test: EthereumRuntimeTest, + name: str, + geth_service: object, + address: str, + *, + retries: int = 60, + interval: int = 5, +) -> Optional[str]: + last_code = "" + last_error = "" + for attempt in range(1, retries + 1): + try: + last_code = get_contract_code(test, geth_service, address) + except Exception as exc: + last_error = str(exc) + else: + if last_code != "0x": + record( + test, + name, + geth_service, + "eth.getCode({}) != 0x".format(address), + True, + "code_bytes={} attempts={}".format((len(last_code) - 2) // 2, attempt), + ) + return last_code + if attempt < retries: + time.sleep(interval) + record(test, name, geth_service, "eth.getCode({}) != 0x".format(address), False, last_code, last_error) + return None + + +def wait_for_json_file( + test: EthereumRuntimeTest, + name: str, + service: object, + path: str, + predicate: Optional[Callable[[Any], bool]] = None, + *, + retries: int = 120, + interval: int = 5, +) -> Optional[Any]: + command = "cat {}".format(shlex.quote(path)) + last_stdout = "" + last_stderr = "" + for attempt in range(1, retries + 1): + result = test.exec(service, command, timeout=30) + last_stdout = str(result["stdout"]) + last_stderr = str(result["stderr"]) + if result["exit"] == 0: + try: + data = json.loads(last_stdout) + except json.JSONDecodeError as exc: + last_stderr = str(exc) + else: + if predicate is None or predicate(data): + record(test, name, service, command, True, "attempts={} data={}".format(attempt, last_stdout)) + return data + if attempt < retries: + time.sleep(interval) + record(test, name, service, command, False, last_stdout, last_stderr) + return None + + +def utility_http_command(path: str, expect_text: Optional[str] = None) -> str: + code = '''import urllib.error +import urllib.request + +url = "http://127.0.0.1:5000{path}" +expect_text = {expect_text!r} +try: + with urllib.request.urlopen(url, timeout=10) as response: + status = response.status + text = response.read().decode("utf-8", errors="replace") +except urllib.error.HTTPError as exc: + status = exc.code + text = exc.read().decode("utf-8", errors="replace") +print("status={{}}".format(status)) +print(text) +if status != 200: + raise SystemExit(1) +if expect_text is not None and expect_text.lower() not in text.lower(): + raise SystemExit(1) +'''.format(path=path, expect_text=expect_text) + return "python3 -c {}".format(shlex.quote(code)) + + +def run_utility_http_check( + test: EthereumRuntimeTest, + name: str, + utility_service: object, + path: str, + *, + expect_text: Optional[str] = None, +) -> Dict[str, object]: + command = utility_http_command(path, expect_text=expect_text) + result = test.exec(utility_service, command, timeout=30) + return record( + test, + name, + utility_service, + command, + result["exit"] == 0, + str(result["stdout"]), + str(result["stderr"]), + ) + + +def run_exec_check( + test: EthereumRuntimeTest, + name: str, + service: object, + command: str, + *, + retries: int = 1, + interval: int = 3, + timeout: int = 45, +) -> Dict[str, object]: + last_result: Dict[str, object] = {} + for attempt in range(1, retries + 1): + last_result = test.exec(service, command, timeout=timeout) + if last_result["exit"] == 0: + break + if attempt < retries: + time.sleep(interval) + return record( + test, + name, + service, + command, + last_result.get("exit") == 0, + "attempts={}\n{}".format(attempt, last_result.get("stdout", "")), + str(last_result.get("stderr", "")), + ) + + +def run_user_get_price(test: EthereumRuntimeTest, oracle_user: object) -> Dict[str, object]: + command = "cd /oracle && timeout 60 python3 -u user_get_price.py" + result = test.exec(oracle_user, command, timeout=75) + stdout = str(result["stdout"]) + stderr = str(result["stderr"]) + combined = (stdout + "\n" + stderr).lower() + ok = result["exit"] in (0, 124) and "successfully invoke updateprice" in combined and "price " in combined + return record( + test, + "D22 oracle_user script invokes updatePrice() and reads prices", + oracle_user, + command, + ok, + stdout, + stderr, + ) + + +def read_price_command(contract_address: str) -> str: + code = '''import json +import socket +from web3 import Web3 + +contract_address = {contract_address!r} +ip = socket.gethostbyname("gethnode1.net") +web3 = Web3(Web3.HTTPProvider("http://{{}}:8545".format(ip))) +is_connected = getattr(web3, "is_connected", None) or getattr(web3, "isConnected") +if not is_connected(): + raise SystemExit("web3 is not connected") +with open("/contract/Oracle.abi", "r") as handle: + abi = handle.read() +contract = web3.eth.contract(address=contract_address, abi=abi) +price = int(contract.functions.getPrice().call()) +print(json.dumps({{"price": price}}, sort_keys=True)) +if price < 0 or price > 99: + raise SystemExit(1) +'''.format(contract_address=contract_address) + return "python3 -c {}".format(shlex.quote(code)) + + +def read_price(test: EthereumRuntimeTest, oracle_user: object, contract_address: str) -> Dict[str, object]: + command = read_price_command(contract_address) + result = test.exec(oracle_user, command, timeout=60) + return record( + test, + "D22 oracle_user reads getPrice() from Oracle contract", + oracle_user, + command, + result["exit"] == 0, + str(result["stdout"]), + str(result["stderr"]), + ) + + +def valid_address(value: object) -> bool: + return re.fullmatch(r"0x[0-9a-fA-F]{40}", str(value)) is not None + + +def main() -> int: + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D22 has POS Geth nodes for oracle checks", + expected_count=3, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + test.require_ethereum_services( + "D22 has POS Lighthouse beacon nodes", + expected_count=3, + class_contains="Ethereum-POS-Beacon", + consensus="POS", + ) + test.require_ethereum_services( + "D22 has POS genesis validator clients", + expected_count=9, + class_contains="Ethereum-POS-Validator", + role="validator_at_genesis", + consensus="POS", + ) + faucet_services = test.require_ethereum_services( + "D22 has one Faucet service", + expected_count=1, + class_contains="FaucetService", + ) + utility_services = test.require_ethereum_services( + "D22 has one Utility service", + expected_count=1, + class_contains="EthUtilityServer", + ) + oracle_user = test.require_service(ORACLE_USER_ASN, ORACLE_USER_NODE, "D22 oracle_user host is generated") + oracle_node = test.require_service(ORACLE_NODE_ASN, ORACLE_NODE_NODE, "D22 oracle_node host is generated") + + if not geth_nodes or not faucet_services or not utility_services or oracle_user is None or oracle_node is None: + test.write_summary("d22-oracle-runtime-test.json") + return test.exit_code() + + geth = geth_nodes[0] + utility = utility_services[0] + + run_exec_check( + test, + "D22 oracle_user scripts and ABI are installed", + oracle_user, + "test -s /contract/Oracle.abi && test -s /oracle/user_create_account.py && test -s /oracle/user_get_price.py", + retries=3, + interval=2, + ) + run_exec_check( + test, + "D22 oracle_node scripts and contract artifacts are installed", + oracle_node, + "test -s /contract/Oracle.abi && test -s /contract/Oracle.bin && test -s /oracle/deploy_oracle_contract.py && test -s /oracle/oracle_node_set_price.py", + retries=3, + interval=2, + ) + + user_account = wait_for_json_file( + test, + "D22 oracle_user created and saved an account", + oracle_user, + "/oracle/user_account.json", + lambda data: isinstance(data, dict) and valid_address(data.get("account_address")) and bool(data.get("private_key")), + retries=120, + interval=5, + ) + oracle_account = wait_for_json_file( + test, + "D22 oracle_node deployed Oracle contract and saved account data", + oracle_node, + "/oracle/oracle_account.json", + lambda data: isinstance(data, dict) + and valid_address(data.get("account_address")) + and valid_address(data.get("oracle_address")) + and bool(data.get("private_key")), + retries=120, + interval=5, + ) + + oracle_account_address = None + oracle_contract_address = None + if isinstance(user_account, dict): + user_address = str(user_account.get("account_address")) + wait_for_balance_at_least( + test, + "D22 oracle_user account is funded by Faucet", + geth, + user_address, + 1, + retries=60, + interval=5, + ) + if isinstance(oracle_account, dict): + oracle_account_address = str(oracle_account.get("account_address")) + oracle_contract_address = str(oracle_account.get("oracle_address")) + wait_for_balance_at_least( + test, + "D22 oracle_node account is funded by Faucet", + geth, + oracle_account_address, + 1, + retries=60, + interval=5, + ) + + if oracle_contract_address: + run_utility_http_check( + test, + "D22 Utility returns oracle-contract address by name", + utility, + "/contracts_info?name={}".format(ORACLE_CONTRACT_NAME), + expect_text=oracle_contract_address, + ) + run_utility_http_check( + test, + "D22 Utility /all contains oracle-contract address", + utility, + "/all", + expect_text=oracle_contract_address, + ) + wait_for_contract_code( + test, + "D22 Oracle contract has bytecode on chain", + geth, + oracle_contract_address, + retries=60, + interval=5, + ) + + run_exec_check( + test, + "D22 oracle_node event listener process is running", + oracle_node, + "pgrep -af oracle_node_set_price.py >/dev/null", + retries=60, + interval=5, + ) + + if oracle_account_address and oracle_contract_address: + oracle_nonce_before = get_transaction_count(test, geth, oracle_account_address) + time.sleep(5) + user_result = run_user_get_price(test, oracle_user) + if user_result["status"] == "passed": + nonce_after = wait_for_nonce_above( + test, + "D22 oracle_node handles UpdatePriceMessage and sends setPrice()", + geth, + oracle_account_address, + oracle_nonce_before, + retries=90, + interval=5, + ) + if nonce_after is not None: + read_price(test, oracle_user, oracle_contract_address) + + test.write_summary("d22-oracle-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D23_validator/README.md b/examples/blockchain/D23_validator/README.md new file mode 100644 index 000000000..fc1292097 --- /dev/null +++ b/examples/blockchain/D23_validator/README.md @@ -0,0 +1,333 @@ +# Validator at Running + +This example demonstrates how to add a validator to an Ethereum PoS +blockchain after the blockchain has already started. It is based on the +PoS blockchain from [D01_ethereum_pos](../D01_ethereum_pos/). + +In D01, all validators are created before the genesis block and become +part of the initial validator set. In this example, the blockchain +starts first. A new validator is then created, imported into a +Lighthouse validator client, funded with a 32 ETH deposit, and finally +activated by the beacon chain. + + +## Table of Content + +- [How to Run](#how-to-run) +- [Example Architecture](#example-architecture) +- [Create the Validator-at-Running Container](#create-the-validator-at-running-container) +- [Run the Validator Setup Script](#run-the-validator-setup-script) +- [What the Setup Script Does](#what-the-setup-script-does) +- [Observe the Validator Status](#observe-the-validator-status) +- [Test Script](#test-script) +- [Restart the Validator Client](#restart-the-validator-client) +- [Important Notes](#important-notes) + + +## How to Run + +Run the following commands from this example directory: + +```bash +python3 validator.py [amd|arm] +cd output +docker compose build +docker compose up +``` + +Arguments: + +- `amd`: Generate Docker configuration for AMD64. This is the default. +- `arm`: Generate Docker configuration for ARM64 hosts. + +This example always uses the PoS blockchain from `D01_ethereum_pos`. It +does not take a `poa` or `pos` argument. + + +## Example Architecture + +This example first creates the same type of Ethereum PoS blockchain as +D01. It contains: + +| Component | Role | +| --- | --- | +| Geth nodes | Execution-layer nodes. They execute transactions and expose Ethereum JSON-RPC | +| Beacon nodes | Consensus-layer nodes. They track slots, epochs, validator status, attestations, and blocks | +| Validator clients | Lighthouse validator clients that hold validator keys and sign duties | +| Beacon setup node | Generates and distributes the local testnet configuration | +| Deposit contract | Execution-layer contract that accepts 32 ETH validator deposits | +| Validator-at-running node | A new validator client container added by this example | + +The validator-at-running container is not part of the genesis validator +set. It joins later by creating a validator key and sending a 32 ETH +deposit transaction to the deposit contract. + + +## Create the Validator-at-Running Container + +The example adds one extra VC node to the D01 PoS blockchain: + +```python +vc_at_running_1: PoSVcServer = blockchain.createVcNode("vcnodeAtRunning") +vc_at_running_1.appendClassName("Ethereum-POS-Validator-AtRunning") +vc_at_running_1.connectToBeaconNode("beaconnode0") +vc_at_running_1.enablePOSValidatorAtRunning() +emu.getVirtualNode("vcnodeAtRunning").setDisplayName( + "Ethereum-POS-Validator-AtRunning-1" +) +``` + +The generated container name is similar to: + +```text +as161h-Ethereum-POS-Validator-AtRunning-1-10.161.0.71 +``` + +You can find the actual container name with: + +```bash +docker ps | grep -i Validator-AtRunning +``` + + +## Run the Validator Setup Script + +The setup script is: + +```text +vc_start_at_running.sh +``` + +Copy it into the validator-at-running container: + +```bash +docker cp vc_start_at_running.sh \ + :/tmp/ +``` + +If you are already inside the generated `output` directory, use: + +```bash +docker cp ../vc_start_at_running.sh \ + :/tmp/ +``` + +Then execute the script: + +```bash +docker exec -it \ + bash /tmp/vc_start_at_running.sh +``` + +During execution, Lighthouse first prints a 24-word BIP-39 mnemonic. +Save this mnemonic. When the script later asks: + +```text +Enter the mnemonic phrase: +``` + +paste the same 24-word mnemonic exactly as printed. + + +## What the Setup Script Does + +The script performs the following steps: + +1. Reads the execution-layer account from `/tmp/keystore` and uses it + as the fee recipient. +2. Creates a Lighthouse wallet named `seed` under + `/tmp/vc/local-testnet/testnet`. +3. Uses `lighthouse validator-manager create` to generate one new + validator key and deposit data. +4. Starts a Lighthouse validator client connected to the beacon node at + `http://10.151.0.72:8000`. +5. Imports the generated validator key into the validator client through + the validator client HTTP API. +6. Sends a 32 ETH deposit transaction through the Geth RPC endpoint at + `http://10.150.0.72:8545`. + +The deposit transaction calls the standard Ethereum deposit contract: + +```text +0x00000000219ab540356cBB839Cbe05303d7705Fa +``` + +After the deposit is included in a block and processed by the beacon +chain, the new validator enters the activation queue. It becomes active +only after the beacon chain activates it. + + +## Observe the Validator Status + +The validator client log is written to: + +```text +/tmp/lighthouse-vc.log +``` + +Use the following command to follow the log: + +```bash +docker exec -it \ + tail -f /tmp/lighthouse-vc.log +``` + +These log entries mean the validator client started and imported the +key, but the validator is not active yet: + +```text +Enabled validator +Imported keystores via standard HTTP API +Connected to beacon node(s) +Awaiting activation +``` + +To query the beacon node directly, use the validator public key printed +in the log: + +```bash +curl -s \ + http://10.151.0.72:8000/eth/v1/beacon/states/head/validators/ \ + | jq +``` + +Common validator states include: + +| State | Meaning | +| --- | --- | +| `pending_initialized` | The deposit has been seen, but the validator is not queued for activation yet | +| `pending_queued` | The validator is waiting in the activation queue | +| `active_ongoing` | The validator is active and can perform duties | + +When the validator becomes active, the log will show messages similar +to the following: + +```text +All validators active +Successfully published attestation +``` + +If this validator is selected to propose a block, the log will show +messages similar to: + +```text +Requesting unsigned block +Publishing signed block +Successfully published block +``` + + +## Test Script + +The test script is: + +```text +test_vc_at_running.py +``` + +It is a runtime monitor. It does not create a validator and it does not +send a deposit. You should run it only after the emulator is running and +after `vc_start_at_running.sh` has been executed. + +Run it from this example directory: + +```bash +python3 test_vc_at_running.py +``` + +By default, it checks the beacon API at `http://10.151.0.72:8000` and +monitors validator rank `10`. This matches the default topology: D01 +creates 9 validators at genesis, so the validator added by this example +is expected to become the 10th validator in the beacon registry. + +The script repeatedly queries: + +```text +/eth/v1/beacon/headers/head +/eth/v1/beacon/states/head/validators +``` + +It waits until the watched validator has been seen in a pending state +and later in an active state. Success looks like this: + +```text +OK: validator_index=9 transitioned pending -> active +``` + +If the validator does not become active before the timeout, the script +exits with code `2`. You can tune the monitor if your topology is +different: + +```bash +python3 test_vc_at_running.py \ + --beacon-api http://10.151.0.72:8000 \ + --rank 10 \ + --timeout-secs 7200 \ + --interval-secs 12 +``` + + +## Restart the Validator Client + +The setup script is intended for the initial setup. Do not repeatedly +run the whole script after the wallet and validator key have already +been created. Re-running the full script can fail with: + +```text +Unable to create wallet: NameAlreadyTaken("seed") +``` + +If the validator client stops after the key has already been imported +and the deposit has already been sent, restart only the Lighthouse +validator client: + +```bash +docker exec -d bash -lc ' +exec lighthouse --debug-level info vc \ + --datadir /tmp/vc/local-testnet/testnet \ + --testnet-dir /tmp/vc/local-testnet/testnet \ + --init-slashing-protection \ + --beacon-nodes http://10.151.0.72:8000 \ + --suggested-fee-recipient \ + --http \ + --http-address 0.0.0.0 \ + --http-port 5062 \ + --http-allow-origin "*" \ + --unencrypted-http-transport \ + --enable-doppelganger-protection \ + > /tmp/lighthouse-vc.log 2>&1 +' +``` + +The fee recipient address is printed near the beginning of the setup +script output: + +```text +[INFO] FEE_RECIPIENT=... +``` + +Then verify that the validator client is running: + +```bash +docker exec -it \ + ps aux | grep lighthouse +``` + + +## Important Notes + +- `Awaiting activation` is not the same as being active. The validator + is active only after the beacon API reports `active_ongoing` or the + log shows successful attestation duties. +- The 32 ETH deposit is a normal execution-layer transaction sent to + the deposit contract. The beacon chain later reads and processes this + deposit. +- The validator public key identifies the validator on the beacon + chain. The execution-layer fee recipient address is different from + the validator public key. +- Keep the generated mnemonic private in real deployments. In this lab, + the mnemonic is used only inside the local emulator. +- This example uses fixed internal addresses, including + `http://10.150.0.72:8545` for Geth RPC and + `http://10.151.0.72:8000` for the beacon node API. These addresses + come from the generated D01-based topology. diff --git a/examples/blockchain/D23_validator/example.yaml b/examples/blockchain/D23_validator/example.yaml new file mode 100644 index 000000000..0c0754b9a --- /dev/null +++ b/examples/blockchain/D23_validator/example.yaml @@ -0,0 +1,161 @@ +id: blockchain-d23-validator-at-running +name: Ethereum PoS Validator At Running +runner: blockchain +script: validator.py +platform: amd +features: + - ethereum-pos + - geth + - lighthouse + - validators-at-genesis + - validator-at-running + - ethereum-deposit + +compile: + enabled: true + output: output + args: + - --no-ether-view + clean: + - output + expected: + - output/docker-compose.yml + timeout: 1800 + +build: + enabled: true + timeout: 3600 + +ci: + test_program_command: test + +runtime: + compose: output/docker-compose.yml + readiness: + - name: D23 POS Geth execution nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Geth + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D23 POS Lighthouse beacon nodes are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Beacon + consensus: POS + expected_count: 3 + retries: 60 + interval: 3 + + - name: D23 POS genesis validator clients are running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + expected_count: 9 + retries: 60 + interval: 3 + + - name: D23 validator-at-running service is running + type: ethereum-compose-ps + class_contains: Ethereum-POS-Validator-AtRunning + role: validator_at_running + consensus: POS + expected_count: 1 + retries: 60 + interval: 3 + + - name: D23 POS beacon setup service is running + type: ethereum-compose-ps + display_contains: Ethereum-POS-BeaconSetup + expected_count: 1 + retries: 60 + interval: 3 + + - name: D23 Faucet service is running + type: ethereum-compose-ps + class_contains: FaucetService + expected_count: 1 + retries: 60 + interval: 3 + + - name: D23 Utility service is running + type: ethereum-compose-ps + class_contains: EthUtilityServer + expected_count: 1 + retries: 60 + interval: 3 + +probes: + - name: D23 POS Geth process is active + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: pgrep geth >/dev/null + retries: 30 + interval: 3 + + - name: D23 POS Lighthouse beacon process is active + type: ethereum-exec + class_contains: Ethereum-POS-Beacon + consensus: POS + command: "pgrep -af 'lighthouse.* bn ' >/dev/null" + retries: 30 + interval: 3 + + - name: D23 POS genesis Lighthouse validator process is active + type: ethereum-exec + class_contains: Ethereum-POS-Validator + role: validator_at_genesis + consensus: POS + command: "pgrep -af 'lighthouse.* vc ' >/dev/null" + retries: 30 + interval: 3 + + - name: D23 POS beacon setup produced genesis state + type: ethereum-exec + display_contains: Ethereum-POS-BeaconSetup + command: test -s /local-testnet/testnet/genesis.ssz + retries: 30 + interval: 3 + + - name: D23 validator-at-running node received testnet files + type: ethereum-exec + class_contains: Ethereum-POS-Validator-AtRunning + role: validator_at_running + consensus: POS + command: test -s /tmp/withdraw-address && test -s /tmp/seed.pass && test -s /tmp/vc/local-testnet/testnet/genesis.ssz + retries: 60 + interval: 3 + + - name: D23 Geth IPC exposes a funded local account + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: "geth attach --exec 'eth.accounts.length > 0' | grep -q true" + retries: 30 + interval: 3 + + - name: D23 deposit contract bytecode exists + type: ethereum-exec + class_contains: Ethereum-POS-Geth + consensus: POS + command: >- + geth attach --exec 'eth.getCode("0x00000000219ab540356cBB839Cbe05303d7705Fa").length > 2' | grep -q true + retries: 30 + interval: 3 + + - name: D23 POS chain produces execution blocks + type: ethereum-block-progress + class_contains: Ethereum-POS-Geth + consensus: POS + min_delta: 1 + block_retries: 60 + block_interval: 5 + timeout: 60 + +test_programs: + - name: D23 validator-at-running validation + script: test_runtime.py + timeout: 9600 diff --git a/examples/blockchain/D23_validator/test_runtime.py b/examples/blockchain/D23_validator/test_runtime.py new file mode 100644 index 000000000..2f90e87b9 --- /dev/null +++ b/examples/blockchain/D23_validator/test_runtime.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, Optional + +import requests + +from seedemu.testing import EthereumRuntimeTest + + +DEPOSIT_CONTRACT = "0x00000000219ab540356cBB839Cbe05303d7705Fa" +VALIDATOR_MNEMONIC = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" +EXPECTED_GENESIS_VALIDATORS = 9 +EXPECTED_VALIDATOR_RANK = EXPECTED_GENESIS_VALIDATORS + 1 +DEPOSIT_RESULT_FILE = "/tmp/d23_deposit_result.json" + + +def service_name(service: object) -> str: + return getattr(service, "name", str(service)) + + +def record( + test: EthereumRuntimeTest, + name: str, + service: object, + command: str, + ok: bool, + stdout: str = "", + stderr: str = "", +) -> Dict[str, object]: + result = { + "name": name, + "service": service_name(service), + "command": command, + "exit": 0 if ok else 1, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if ok else "failed", + } + test.results.append(result) + return result + + +def compose_cp(test: EthereumRuntimeTest, name: str, source: Path, service: object, destination: str) -> bool: + target = "{}:{}".format(service_name(service), destination) + result = subprocess.run( + ["docker", "compose", "-f", str(test.compose_file), "cp", str(source), target], + cwd=str(test.compose_file.parent), + env=os.environ.copy(), + text=True, + capture_output=True, + timeout=120, + check=False, + ) + record( + test, + name, + service, + "docker compose cp {} {}".format(source, target), + result.returncode == 0, + result.stdout, + result.stderr, + ) + return result.returncode == 0 + + +def get_json_file(test: EthereumRuntimeTest, service: object, path: str) -> Optional[Dict[str, Any]]: + result = test.exec(service, "cat {}".format(shlex.quote(path)), timeout=30) + if result["exit"] != 0: + record(test, "D23 deposit result file exists", service, result["command"], False, str(result["stdout"]), str(result["stderr"])) + return None + try: + data = json.loads(str(result["stdout"])) + except json.JSONDecodeError as exc: + record(test, "D23 deposit result file is valid JSON", service, result["command"], False, str(result["stdout"]), str(exc)) + return None + record(test, "D23 deposit result file is valid JSON", service, result["command"], True, json.dumps(data, sort_keys=True)) + return data + + +def read_optional_json_file(test: EthereumRuntimeTest, service: object, path: str) -> Optional[Dict[str, Any]]: + result = test.exec(service, "cat {}".format(shlex.quote(path)), timeout=30) + if result["exit"] != 0: + return None + try: + return json.loads(str(result["stdout"])) + except json.JSONDecodeError: + return None + + +def deposit_result_is_successful(data: Optional[Dict[str, Any]]) -> bool: + if not data: + return False + try: + return ( + data.get("success") is True + and int(data.get("status", 0)) == 1 + and str(data.get("tx_hash", "")).startswith("0x") + and int(data.get("deposit_value_wei", "0")) == 32 * 10**18 + and int(data.get("balance_before_wei", "0")) > int(data.get("balance_after_wei", "0")) + ) + except (TypeError, ValueError): + return False + + +def validate_deposit_result(test: EthereumRuntimeTest, service: object, data: Optional[Dict[str, Any]]) -> bool: + ok = deposit_result_is_successful(data) + record( + test, + "D23 32 ETH deposit transaction succeeded", + service, + "validate {}".format(DEPOSIT_RESULT_FILE), + ok, + json.dumps(data or {}, sort_keys=True), + ) + return ok + + +def manifest_ci_test_program_command() -> str: + manifest = Path(os.environ.get("TEST_RUNNER_MANIFEST", Path(__file__).with_name("example.yaml"))) + try: + import yaml + data = yaml.safe_load(manifest.read_text(encoding="utf-8")) + except Exception: + return "all" + if not isinstance(data, dict): + return "all" + ci_cfg = data.get("ci", {}) + if not isinstance(ci_cfg, dict): + return "all" + return str(ci_cfg.get("test_program_command", "all")) + + +def default_require_activation() -> bool: + command = os.environ.get("TEST_RUNNER_COMMAND", "") + if command != "all": + return False + if os.environ.get("GITHUB_ACTIONS") == "true" and manifest_ci_test_program_command() == "test": + return False + return True + + +def request_json(url: str, timeout: int = 10) -> Dict[str, Any]: + response = requests.get(url, timeout=timeout) + response.raise_for_status() + return response.json() + + +def head_epoch(base_url: str) -> int: + payload = request_json("{}/eth/v1/beacon/headers/head".format(base_url)) + slot = int(payload.get("data", {}).get("header", {}).get("message", {}).get("slot", "0")) + return slot // 32 + + +def validators(base_url: str) -> list[Dict[str, Any]]: + payload = request_json("{}/eth/v1/beacon/states/head/validators".format(base_url)) + data = payload.get("data", []) + if not isinstance(data, list): + return [] + return sorted(data, key=lambda item: int(item.get("index", 0))) + + +def wait_for_validator_count( + test: EthereumRuntimeTest, + name: str, + beacon_service: object, + base_url: str, + minimum: int, + *, + retries: int = 60, + interval: int = 5, +) -> int: + last_count = 0 + last_error = "" + for attempt in range(1, retries + 1): + try: + entries = validators(base_url) + last_count = len(entries) + if last_count >= minimum: + record( + test, + name, + beacon_service, + "GET /eth/v1/beacon/states/head/validators", + True, + "validators={} attempts={}".format(last_count, attempt), + ) + return last_count + except Exception as exc: + last_error = str(exc) + if attempt < retries: + time.sleep(interval) + record( + test, + name, + beacon_service, + "GET /eth/v1/beacon/states/head/validators", + False, + "validators={}".format(last_count), + last_error, + ) + return last_count + + +def wait_for_validator_activation( + test: EthereumRuntimeTest, + beacon_service: object, + base_url: str, + rank: int, + timeout_secs: int, + interval_secs: int, +) -> bool: + start_time = time.time() + pending_seen = False + active_seen = False + watched_index: Optional[int] = None + last_status: Optional[str] = None + last_message = "" + last_error = "" + + while time.time() - start_time <= timeout_secs: + elapsed = int(time.time() - start_time) + try: + epoch = head_epoch(base_url) + entries = validators(base_url) + count = len(entries) + if count < rank: + last_message = "t={}s epoch={} validators={} waiting_for_rank={}".format(elapsed, epoch, count, rank) + else: + entry = entries[rank - 1] + watched_index = int(entry.get("index", rank - 1)) + status = str(entry.get("status", "")).lower() + balance = entry.get("balance") + if status != last_status: + last_message = "t={}s epoch={} validator_index={} status {} -> {} balance={}".format( + elapsed, + epoch, + watched_index, + last_status, + status, + balance, + ) + last_status = status + else: + last_message = "t={}s epoch={} validator_index={} status={} balance={}".format( + elapsed, + epoch, + watched_index, + status, + balance, + ) + if status.startswith("pending"): + pending_seen = True + if status.startswith("active"): + active_seen = True + record( + test, + "D23 validator-at-running reached active state", + beacon_service, + "monitor beacon validator rank {}".format(rank), + True, + "validator_index={} status={} pending_seen={} active_seen={}".format( + watched_index, + status, + pending_seen, + active_seen, + ), + ) + return True + except Exception as exc: + last_error = str(exc) + last_message = "error: {}".format(exc) + time.sleep(interval_secs) + + record( + test, + "D23 validator-at-running reached active state", + beacon_service, + "monitor beacon validator rank {}".format(rank), + False, + "last={} pending_seen={} active_seen={}".format(last_message, pending_seen, active_seen), + last_error, + ) + return False + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate the D23 validator-at-running workflow.") + parser.add_argument("--registry-timeout-secs", type=int, default=1800) + parser.add_argument("--activation-timeout-secs", type=int, default=7200) + parser.add_argument("--activation-interval-secs", type=int, default=12) + parser.add_argument( + "--require-activation", + action="store_true", + default=default_require_activation(), + help="also wait for the new validator to enter the beacon registry and become active", + ) + args = parser.parse_args() + if args.registry_timeout_secs < 1: + parser.error("--registry-timeout-secs must be >= 1") + if args.activation_timeout_secs < 1: + parser.error("--activation-timeout-secs must be >= 1") + if args.activation_interval_secs < 1: + parser.error("--activation-interval-secs must be >= 1") + return args + + +def main() -> int: + args = parse_args() + test = EthereumRuntimeTest(__file__) + + geth_nodes = test.require_ethereum_services( + "D23 has POS Geth execution nodes", + minimum_count=1, + class_contains="Ethereum-POS-Geth", + consensus="POS", + ) + beacon_nodes = test.require_ethereum_services( + "D23 has POS Lighthouse beacon nodes", + minimum_count=1, + class_contains="Ethereum-POS-Beacon", + consensus="POS", + ) + test.require_ethereum_services( + "D23 still has 9 genesis validator clients", + expected_count=EXPECTED_GENESIS_VALIDATORS, + class_contains="Ethereum-POS-Validator", + role="validator_at_genesis", + consensus="POS", + ) + validator_nodes = test.require_ethereum_services( + "D23 has one validator-at-running client node", + expected_count=1, + class_contains="Ethereum-POS-Validator-AtRunning", + role="validator_at_running", + consensus="POS", + ) + + if not geth_nodes or not beacon_nodes or not validator_nodes: + test.write_summary("d23-validator-at-running-runtime-test.json") + return test.exit_code() + + geth = geth_nodes[0] + beacon = beacon_nodes[0] + validator_node = validator_nodes[0] + beacon_api = "http://{}:8000".format(beacon.address) + + test.exec_check( + "D23 deposit contract exists on execution layer", + geth, + 'geth attach --exec \'eth.getCode("{}").length > 2\' | grep -q true'.format(DEPOSIT_CONTRACT), + retries=20, + interval=3, + timeout=60, + ) + test.exec_check( + "D23 validator-at-running node has bootstrap files", + validator_node, + "test -s /tmp/withdraw-address && test -s /tmp/seed.pass && test -s /tmp/vc/local-testnet/testnet/genesis.ssz && ls /tmp/keystore/UTC--* >/dev/null", + retries=60, + interval=5, + timeout=60, + ) + wait_for_validator_count( + test, + "D23 beacon API reports the 9 genesis validators before deposit", + beacon, + beacon_api, + EXPECTED_GENESIS_VALIDATORS, + retries=60, + interval=5, + ) + + deposit_data = read_optional_json_file(test, validator_node, DEPOSIT_RESULT_FILE) + setup_succeeded = deposit_result_is_successful(deposit_data) + if setup_succeeded: + record( + test, + "D23 setup script skipped because deposit result already exists", + validator_node, + "read {}".format(DEPOSIT_RESULT_FILE), + True, + json.dumps(deposit_data, sort_keys=True), + ) + else: + script_path = Path(__file__).resolve().parent / "vc_start_at_running.sh" + if compose_cp(test, "D23 setup script copied into validator-at-running node", script_path, validator_node, "/tmp/vc_start_at_running.sh"): + test.exec_check( + "D23 setup script is executable", + validator_node, + "chmod +x /tmp/vc_start_at_running.sh && test -x /tmp/vc_start_at_running.sh", + retries=1, + timeout=30, + ) + + setup_command = "D23_VALIDATOR_MNEMONIC={} bash /tmp/vc_start_at_running.sh".format( + shlex.quote(VALIDATOR_MNEMONIC) + ) + setup_result = test.exec(validator_node, setup_command, timeout=900) + setup_succeeded = setup_result["exit"] == 0 + record( + test, + "D23 setup script creates/imports validator and sends deposit", + validator_node, + setup_command, + setup_succeeded, + str(setup_result["stdout"]), + str(setup_result["stderr"]), + ) + if setup_succeeded: + deposit_data = get_json_file(test, validator_node, DEPOSIT_RESULT_FILE) + + if setup_succeeded: + validate_deposit_result(test, validator_node, deposit_data) + test.exec_check( + "D23 validator-at-running Lighthouse VC process is active", + validator_node, + "pgrep -af 'lighthouse.* vc ' >/dev/null", + retries=20, + interval=3, + timeout=30, + ) + test.exec_check( + "D23 validator-at-running VC imported validator key", + validator_node, + "test -s /tmp/new_validators/validators.json && grep -Eq 'Imported|Enabled validator|Connected to beacon node|Awaiting activation' /tmp/lighthouse-vc.log", + retries=30, + interval=5, + timeout=30, + ) + if args.require_activation: + registry_retries = max(1, args.registry_timeout_secs // args.activation_interval_secs) + wait_for_validator_count( + test, + "D23 beacon API reports validator-at-running in registry", + beacon, + beacon_api, + EXPECTED_VALIDATOR_RANK, + retries=registry_retries, + interval=args.activation_interval_secs, + ) + wait_for_validator_activation( + test, + beacon, + beacon_api, + EXPECTED_VALIDATOR_RANK, + args.activation_timeout_secs, + args.activation_interval_secs, + ) + else: + record( + test, + "D23 activation monitor is optional", + beacon, + "skip beacon activation monitor", + True, + "deposit succeeded and the validator client is awaiting activation; local all runs the long active-state monitor, while GitHub CI follows example.yaml ci.test_program_command", + ) + + test.write_summary("d23-validator-at-running-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D23_validator/test_vc_at_running.py b/examples/blockchain/D23_validator/test_vc_at_running.py new file mode 100644 index 000000000..d16146000 --- /dev/null +++ b/examples/blockchain/D23_validator/test_vc_at_running.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import argparse +import sys +import time + +import requests + + +def _get_head_epoch(base_url: str, timeout: float) -> int: + resp = requests.get("{}/eth/v1/beacon/headers/head".format(base_url), timeout=timeout) + resp.raise_for_status() + payload = resp.json() + slot_str = ( + payload.get("data", {}) + .get("header", {}) + .get("message", {}) + .get("slot", "0") + ) + slot = int(slot_str) + return slot // 32 + + +def _get_validators(base_url: str, timeout: float) -> list[dict]: + resp = requests.get("{}/eth/v1/beacon/states/head/validators".format(base_url), timeout=timeout) + resp.raise_for_status() + payload = resp.json() + data = payload.get("data", []) + if not isinstance(data, list): + return [] + return data + + +def run_monitor( + base_url: str, + rank: int, + timeout_secs: int, + interval_secs: int, + http_timeout_secs: int, +) -> int: + start_time = time.time() + pending_seen = False + active_seen = False + watched_index = None + last_status = None + last_epoch_printed = None + + while True: + elapsed = int(time.time() - start_time) + if elapsed > timeout_secs: + print("TIMEOUT after {} sec".format(timeout_secs)) + return 2 + + try: + epoch = _get_head_epoch(base_url, timeout=http_timeout_secs) + validators = _get_validators(base_url, timeout=http_timeout_secs) + validators_sorted = sorted(validators, key=lambda v: int(v.get("index", 0))) + count = len(validators_sorted) + + if count < rank: + if epoch != last_epoch_printed: + print("t={}s epoch={} validators={} waiting_for_validator_at_running_active rank={}".format(elapsed, epoch, count, rank)) + last_epoch_printed = epoch + time.sleep(interval_secs) + continue + + entry = validators_sorted[rank - 1] + index = int(entry.get("index")) + status = str(entry.get("status", "")).lower() + + if watched_index is None: + watched_index = index + if index != watched_index: + print("t={}s epoch={} rank={} index_changed {} -> {}".format(elapsed, epoch, rank, watched_index, index)) + watched_index = index + + if last_status != status: + print("t={}s epoch={} validator_index={} status {} -> {}".format(elapsed, epoch, watched_index, last_status, status)) + last_status = status + else: + if epoch != last_epoch_printed: + print("t={}s epoch={} validator_index={} status={}".format(elapsed, epoch, watched_index, status)) + last_epoch_printed = epoch + + if status.startswith("pending"): + pending_seen = True + if status.startswith("active"): + active_seen = True + + if pending_seen and active_seen: + print("OK: validator_index={} transitioned pending -> active".format(watched_index)) + return 0 + + except Exception as e: + print("t={}s error: {}".format(elapsed, e)) + + time.sleep(interval_secs) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--beacon-api", default="http://10.151.0.72:8000") + parser.add_argument("--rank", type=int, default=10) + parser.add_argument("--timeout-secs", type=int, default=7200) + parser.add_argument("--interval-secs", type=int, default=12) + parser.add_argument("--http-timeout-secs", type=int, default=10) + args = parser.parse_args() + + return run_monitor( + base_url=args.beacon_api, + rank=args.rank, + timeout_secs=args.timeout_secs, + interval_secs=args.interval_secs, + http_timeout_secs=args.http_timeout_secs, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/blockchain/D23_validator/validator.py b/examples/blockchain/D23_validator/validator.py new file mode 100755 index 000000000..822b6fc50 --- /dev/null +++ b/examples/blockchain/D23_validator/validator.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +import importlib +import os +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu import * +from examples.blockchain.D01_ethereum_pos import ethereum_pos + + +def make_post_genesis_deposits_practical() -> None: + # D23 submits deposits after genesis; the mainnet follow distance would make + # a local devnet wait for hours before the beacon chain processes them. + ethereum_server = importlib.import_module("seedemu.services.EthereumService.EthereumServer") + + ethereum_server.BEACON_GENESIS = ethereum_server.BEACON_GENESIS.replace( + "ETH1_FOLLOW_DISTANCE: 2048", + "ETH1_FOLLOW_DISTANCE: 1\nEPOCHS_PER_ETH1_VOTING_PERIOD: 1", + ) + if "--eth1-cache-follow-distance" not in ethereum_server.LIGHTHOUSE_BN_CMD: + ethereum_server.LIGHTHOUSE_BN_CMD = ethereum_server.LIGHTHOUSE_BN_CMD.replace( + " --execution-jwt /tmp/jwt.hex {bootnodes_flag}", + " --execution-jwt /tmp/jwt.hex --eth1-cache-follow-distance 1 --eth1-blocks-per-log-query 1 {bootnodes_flag}", + ) + + +@contextmanager +def pushd(directory: Path): + previous = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(previous) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the D23 validator-at-running example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument( + "--beacon-nodes", + type=int, + default=3, + help="number of geth/beacon node pairs inherited from D01", + ) + parser.add_argument( + "--validators-per-beacon", + type=int, + default=3, + help="genesis validator clients per beacon node inherited from D01", + ) + parser.add_argument( + "--ether-view", + dest="ether_view_enabled", + action="store_true", + default=True, + help="enable Eth Explorer output in generated Docker files (default)", + ) + parser.add_argument( + "--no-ether-view", + dest="ether_view_enabled", + action="store_false", + help="disable Eth Explorer output; useful for CI builds", + ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + if args.beacon_nodes < 1: + parser.error("--beacon-nodes must be >= 1") + if args.validators_per_beacon < 1: + parser.error("--validators-per-beacon must be >= 1") + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def load_base_pos(total_beacon_nodes: int, vc_per_beacon: int) -> Emulator: + emu = Emulator() + with tempfile.TemporaryDirectory(prefix="seedemu-d23-") as tempdir: + temp_path = Path(tempdir) + dump_path = temp_path / "blockchain_pos.bin" + with pushd(temp_path): + ethereum_pos.run( + dumpfile=str(dump_path), + total_beacon_nodes=total_beacon_nodes, + vc_per_beacon=vc_per_beacon, + ) + emu.load(str(dump_path)) + return emu + + +def build_emulator(total_beacon_nodes: int = 3, vc_per_beacon: int = 3) -> Emulator: + emu = load_base_pos(total_beacon_nodes=total_beacon_nodes, vc_per_beacon=vc_per_beacon) + + eth = emu.getLayer("EthereumService") + blockchain = eth.getBlockchainByName(eth.getBlockchainNames()[0]) + + vc_at_running: PoSVcServer = blockchain.createVcNode("vcnodeAtRunning") + vc_at_running.appendClassName("Ethereum-POS-Validator-AtRunning") + vc_at_running.setDisplayName("Ethereum-POS-Validator-AtRunning-1") + vc_at_running.connectToBeaconNode("beaconnode0") + vc_at_running.enablePOSValidatorAtRunning() + emu.getVirtualNode("vcnodeAtRunning").setDisplayName("Ethereum-POS-Validator-AtRunning-1") + + emu.addBinding(Binding("vcnodeAtRunning", filter=Filter(nodeName="host_*"), action=Action.FIRST)) + return emu + + +def run( + dumpfile=None, + total_beacon_nodes: int = 3, + vc_per_beacon: int = 3, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + ether_view_enabled: bool = True, +): + make_post_genesis_deposits_practical() + emu = build_emulator(total_beacon_nodes=total_beacon_nodes, vc_per_beacon=vc_per_beacon) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + docker = Docker(internetMapEnabled=True, etherViewEnabled=ether_view_enabled, platform=platform) + emu.compile(docker, str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + total_beacon_nodes=args.beacon_nodes, + vc_per_beacon=args.validators_per_beacon, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ether_view_enabled=args.ether_view_enabled, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/blockchain/D23_validator/vc_start_at_running.sh b/examples/blockchain/D23_validator/vc_start_at_running.sh new file mode 100755 index 000000000..c6fd927a9 --- /dev/null +++ b/examples/blockchain/D23_validator/vc_start_at_running.sh @@ -0,0 +1,359 @@ +#!/bin/bash + +set -euo pipefail + +######################################## +# Config +######################################## + +TESTNET_DIR="/tmp/vc/local-testnet/testnet" +DATADIR="/tmp/vc/local-testnet/testnet" + +BEACON_NODE="${D23_BEACON_NODE:-http://10.151.0.72:8000}" +GETH_RPC="${D23_GETH_RPC:-http://10.150.0.72:8545}" + +WITHDRAW_ADDRESS_FILE="/tmp/withdraw-address" + +WALLET_NAME="seed" +WALLET_PASSWORD_FILE="/tmp/seed.pass" +WALLET_MNEMONIC_FILE="/tmp/validator-at-running-wallet.mnemonic" +VALIDATOR_MNEMONIC_FILE="${D23_VALIDATOR_MNEMONIC_FILE:-/tmp/validator-at-running.mnemonic}" +VALIDATOR_MNEMONIC="${D23_VALIDATOR_MNEMONIC:-abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about}" + +NEW_VALIDATOR_DIR="/tmp/new_validators" +VC_HTTP_PORT="5062" +DEPOSIT_RESULT_FILE="/tmp/d23_deposit_result.json" + +export GETH_RPC +export KEYSTORE_PASSWORD="${KEYSTORE_PASSWORD:-admin}" + +######################################## +# Prepare deterministic non-interactive input +######################################## + +if [ ! -s "${VALIDATOR_MNEMONIC_FILE}" ]; then + printf '%s\n' "${VALIDATOR_MNEMONIC}" > "${VALIDATOR_MNEMONIC_FILE}" + chmod 600 "${VALIDATOR_MNEMONIC_FILE}" +fi + +######################################## +# Get FEE_RECIPIENT from first keystore +######################################## + +FEE_RECIPIENT=$( +python3 <<'PY' +import glob, json, os +from eth_account import Account +from web3 import Web3 + +password = os.environ.get("KEYSTORE_PASSWORD", "admin") + +keystore_files = sorted(glob.glob("/tmp/keystore/UTC--*")) + +if not keystore_files: + raise SystemExit("no keystore found in /tmp/keystore") + +ks_path = keystore_files[0] + +ks = json.load(open(ks_path, "r")) +pk = Account.decrypt(ks, password) +acct = Account.from_key(pk) + +print(Web3.to_checksum_address(acct.address)) +PY +) + +echo "[INFO] FEE_RECIPIENT=${FEE_RECIPIENT}" + +######################################## +# Create wallet if not exists +######################################## + +if lighthouse account_manager wallet list --testnet-dir "${TESTNET_DIR}" --datadir "${DATADIR}" 2>/dev/null | grep -qx "${WALLET_NAME}"; then + echo "[INFO] wallet already exists" +else + echo "[INFO] creating lighthouse wallet..." + + lighthouse account_manager wallet create --testnet-dir "${TESTNET_DIR}" --datadir "${DATADIR}" --name "${WALLET_NAME}" --password-file "${WALLET_PASSWORD_FILE}" --mnemonic-output-path "${WALLET_MNEMONIC_FILE}" +fi + +######################################## +# Create validator-manager validator +######################################## + +rm -rf "${NEW_VALIDATOR_DIR}" +mkdir -p "${NEW_VALIDATOR_DIR}" + +echo "[INFO] creating validator-manager validator..." + +lighthouse validator-manager create --testnet-dir "${TESTNET_DIR}" --mnemonic-path "${VALIDATOR_MNEMONIC_FILE}" --stdin-inputs --first-index 0 --count 1 --eth1-withdrawal-address "$(cat ${WITHDRAW_ADDRESS_FILE})" --suggested-fee-recipient "${FEE_RECIPIENT}" --output-path "${NEW_VALIDATOR_DIR}" + +######################################## +# Start VC +######################################## + +echo "[INFO] starting validator client..." + +pkill -f "lighthouse.* vc" || true + +nohup lighthouse --debug-level info vc --datadir "${DATADIR}" --testnet-dir "${TESTNET_DIR}" --init-slashing-protection --beacon-nodes "${BEACON_NODE}" --suggested-fee-recipient "${FEE_RECIPIENT}" --http --http-address 0.0.0.0 --http-port "${VC_HTTP_PORT}" --http-allow-origin "*" --unencrypted-http-transport --enable-doppelganger-protection > /tmp/lighthouse-vc.log 2>&1 & + +for i in $(seq 1 60); do + if [ -s "${DATADIR}/validators/api-token.txt" ] && pgrep -af "lighthouse.* vc" >/dev/null; then + break + fi + if [ "$i" -eq 60 ]; then + echo "[ERROR] validator client did not start or did not create an API token" >&2 + tail -n 80 /tmp/lighthouse-vc.log >&2 || true + exit 1 + fi + sleep 2 +done + +######################################## +# Import validators +######################################## + +echo "[INFO] importing validators into VC..." + +for i in $(seq 1 30); do + if lighthouse validator-manager import --validators-file "${NEW_VALIDATOR_DIR}/validators.json" --vc-url "http://127.0.0.1:${VC_HTTP_PORT}" --vc-token "${DATADIR}/validators/api-token.txt" --ignore-duplicates; then + break + fi + if [ "$i" -eq 30 ]; then + echo "[ERROR] failed to import validator key into validator client" >&2 + tail -n 120 /tmp/lighthouse-vc.log >&2 || true + exit 1 + fi + sleep 2 +done + +######################################## +# Generate deposit script +######################################## + +echo "[INFO] generating deposit python script..." + +cat >/tmp/deposit_from_json_web3v7.py <<'PY' +import glob +import json +import os +from eth_account import Account +from web3 import Web3 + +RPC = os.environ.get("GETH_RPC", "http://10.150.0.72:8545") +KEYSTORE_PASSWORD = os.environ.get("KEYSTORE_PASSWORD", "admin") +RESULT_FILE = "/tmp/d23_deposit_result.json" + +DEPOSIT_CONTRACT = Web3.to_checksum_address( + "0x00000000219ab540356cBB839Cbe05303d7705Fa" +) + +KEYS_DIR = "/tmp/new_validators" + +def log(title, value=""): + print(f"[INFO] {title}: {value}", flush=True) + +def hb(x): + x = x.strip() + if not x.startswith("0x"): + x = "0x" + x + return Web3.to_bytes(hexstr=x) + +def json_default(obj): + if isinstance(obj, bytes): + return Web3.to_hex(obj) + try: + return Web3.to_hex(obj) + except Exception: + return str(obj) + +log("RPC", RPC) +log("deposit contract", DEPOSIT_CONTRACT) +log("deposit json dir", KEYS_DIR) + +deposit_files = sorted(glob.glob(os.path.join(KEYS_DIR, "deposit*.json"))) + +if not deposit_files: + raise SystemExit(f"deposit json not found in {KEYS_DIR}") + +deposit_path = deposit_files[0] + +log("selected deposit_json", deposit_path) + +data = json.load(open(deposit_path, "r")) +entry = data[0] if isinstance(data, list) else data + +pubkey = hb(entry["pubkey"]) +withdrawal_credentials = hb(entry["withdrawal_credentials"]) +signature = hb(entry["signature"]) +deposit_data_root = hb(entry["deposit_data_root"]) + +log("pubkey", entry["pubkey"]) +log("withdrawal_credentials", entry["withdrawal_credentials"]) +log("deposit_data_root", entry["deposit_data_root"]) + +w3 = Web3(Web3.HTTPProvider(RPC)) + +if not w3.is_connected(): + raise SystemExit(f"cannot connect RPC: {RPC}") + +log("connected", True) +log("chain_id", w3.eth.chain_id) +log("latest_block", w3.eth.block_number) +log("gas_price_wei", w3.eth.gas_price) + +code = w3.eth.get_code(DEPOSIT_CONTRACT) + +if not code or code == b"\x00": + raise SystemExit( + f"deposit contract not found at {DEPOSIT_CONTRACT}, eth_getCode is empty" + ) + +log("deposit_contract_code_size", len(code)) + +keystore_files = sorted(glob.glob("/tmp/keystore/UTC--*")) + +if not keystore_files: + raise SystemExit("no keystore found in /tmp/keystore") + +ks_path = keystore_files[0] + +log("selected payer_keystore", ks_path) + +ks = json.load(open(ks_path, "r")) +pk = Account.decrypt(ks, KEYSTORE_PASSWORD) +acct = Account.from_key(pk) + +payer_addr = Web3.to_checksum_address(acct.address) +balance_before = w3.eth.get_balance(payer_addr) + +log("selected payer", payer_addr) +log("payer_balance_before_wei", balance_before) +log("payer_balance_before_eth", w3.from_wei(balance_before, "ether")) + +abi = [{ + "name": "deposit", + "type": "function", + "stateMutability": "payable", + "inputs": [ + {"name": "pubkey", "type": "bytes"}, + {"name": "withdrawal_credentials", "type": "bytes"}, + {"name": "signature", "type": "bytes"}, + {"name": "deposit_data_root", "type": "bytes32"} + ], + "outputs": [] +}] + +contract = w3.eth.contract(address=DEPOSIT_CONTRACT, abi=abi) + +value = w3.to_wei(32, "ether") +nonce = w3.eth.get_transaction_count(payer_addr, "pending") +gas_price = w3.eth.gas_price + +log("deposit_value_eth", 32) +log("nonce_pending", nonce) + +tx = contract.functions.deposit( + pubkey, + withdrawal_credentials, + signature, + deposit_data_root +).build_transaction({ + "from": payer_addr, + "value": value, + "nonce": nonce, + "chainId": w3.eth.chain_id, + "gasPrice": gas_price, +}) + +try: + estimated_gas = w3.eth.estimate_gas(tx) + tx["gas"] = estimated_gas + log("estimated_gas", estimated_gas) +except Exception as e: + tx["gas"] = 250000 + log("estimate_gas_failed_use_default", f"250000 reason={e}") + +max_cost = value + tx["gas"] * gas_price + +log("max_total_cost_wei", max_cost) +log("max_total_cost_eth", w3.from_wei(max_cost, "ether")) + +if balance_before < max_cost: + raise SystemExit( + f"insufficient balance: balance={w3.from_wei(balance_before, 'ether')} ETH, " + f"need about {w3.from_wei(max_cost, 'ether')} ETH" + ) + +signed = w3.eth.account.sign_transaction(tx, pk) +raw = getattr(signed, "raw_transaction", None) or signed.rawTransaction + +tx_hash = w3.eth.send_raw_transaction(raw) +tx_hash_hex = w3.to_hex(tx_hash) + +log("tx_hash", tx_hash_hex) +log("waiting_receipt", "timeout=180s") + +receipt = w3.eth.wait_for_transaction_receipt(tx_hash, timeout=180) + +balance_after = w3.eth.get_balance(payer_addr) + +gas_used = receipt.get("gasUsed", 0) +effective_gas_price = receipt.get("effectiveGasPrice", gas_price) +fee_paid = gas_used * effective_gas_price +success = receipt.status == 1 + +print("\n========== DEPOSIT RESULT ==========") +print("deposit_json:", deposit_path) +print("payer:", payer_addr) +print("payer_keystore:", ks_path) +print("tx_hash:", tx_hash_hex) +print("status:", receipt.status) +print("success:", success) + +print("\n========== BALANCE ==========") +print("balance_before_wei:", balance_before) +print("balance_before_eth:", w3.from_wei(balance_before, "ether")) +print("balance_after_wei:", balance_after) +print("balance_after_eth:", w3.from_wei(balance_after, "ether")) +print("deposit_value_eth:", w3.from_wei(value, "ether")) +print("gas_used:", gas_used) +print("effective_gas_price:", effective_gas_price) +print("fee_paid_wei:", fee_paid) +print("fee_paid_eth:", w3.from_wei(fee_paid, "ether")) + +summary = { + "deposit_json": deposit_path, + "payer": payer_addr, + "payer_keystore": ks_path, + "pubkey": entry["pubkey"], + "tx_hash": tx_hash_hex, + "status": int(receipt.status), + "success": bool(success), + "balance_before_wei": str(balance_before), + "balance_after_wei": str(balance_after), + "deposit_value_wei": str(value), + "gas_used": int(gas_used), + "effective_gas_price": str(effective_gas_price), + "fee_paid_wei": str(fee_paid), + "receipt_block_number": int(receipt.get("blockNumber", 0)), +} +open(RESULT_FILE, "w").write(json.dumps(summary, indent=2, sort_keys=True) + "\n") + +if not success: + raise SystemExit("deposit transaction failed, status != 1") + +print("\n[INFO] deposit success") +PY + +######################################## +# Execute deposit +######################################## + +echo "[INFO] sending 32 ETH deposit..." + +python3 /tmp/deposit_from_json_web3v7.py + +echo +echo "[INFO] ALL DONE" diff --git a/examples/blockchain/D70_solana/README.md b/examples/blockchain/D70_solana/README.md new file mode 100644 index 000000000..55e966c34 --- /dev/null +++ b/examples/blockchain/D70_solana/README.md @@ -0,0 +1,277 @@ +# D70_solana: Solana Blockchain Service Example + +This example deploys a **private, self-contained Solana cluster** on top of an +emulated Internet. It runs the real [Agave](https://github.com/anza-xyz/agave) +validator software (the production Solana client maintained by Anza), so the +same tools and workflows used against real Solana clusters work here unchanged. + +The example mirrors the structure of [`D60_monero`](../D60_monero): build a base +Internet, attach the blockchain service, bind its virtual nodes to hosts, +render, and compile to Docker. + +> **Platform note.** The cluster runs **natively on both amd64 and arm64**. The +> `seedemu-solana` base image downloads Agave's official prebuilt binaries on +> amd64, and (since Anza ships no arm64 Linux binaries) compiles the same pinned +> version from source on arm64. The arm64 source build is slow the first time +> (tens of minutes) but is cached afterward. + +## What gets built + +`solana_basic.py` builds a 10-stub-AS Internet (via `Makers`) and a private +Solana cluster of ten nodes: + +| Virtual node | Role | AS | Host | +| --------------------- | ------------------- | --- | ------- | +| `sol-boot-150` | bootstrap validator | 150 | host_0 | +| `sol-validator-150b` | validator | 150 | host_1 | +| `sol-validator-151` | validator | 151 | host_0 | +| `sol-validator-152` | validator | 152 | host_0 | +| `sol-validator-153` | validator | 153 | host_0 | +| `sol-validator-154` | validator | 154 | host_0 | +| `sol-validator-160` | validator | 160 | host_0 | +| `sol-validator-161` | validator | 161 | host_0 | +| `sol-validator-162` | validator | 162 | host_0 | +| `sol-validator-163` | validator | 163 | host_0 | + +- The **bootstrap validator** creates the genesis ledger with `solana-genesis`, + stakes itself, runs `solana-faucet`, and is the gossip entrypoint. +- The **same-AS validator** (AS150/host_1) joins over the local network. +- The **cross-AS validators** join over inter-domain (BGP) routing, + demonstrating a larger cluster that spans the emulated Internet. + +Each validator waits for the bootstrap RPC, generates its own identity and vote +keypairs, funds its identity with an airdrop (relayed through the bootstrap +RPC), creates a vote account, and joins the cluster via the bootstrap node's +gossip entrypoint. + +All addresses (each node's own IP, and the bootstrap's gossip/RPC endpoint) are +resolved from the virtual-node bindings at render time — nothing is hard-coded +in the service (see the design notes below). + +## Build and run + +If you are at the repository root, enter this example directory first: + +```bash +cd examples/blockchain/D70_solana +``` + +Prepare the emulation first. This builds the Solana base image, generates the +SEED output, builds the local Docker images, starts all containers, and leaves +the cluster running for inspection: + +```bash +./prepare_solana.sh +``` + +The equivalent manual flow, from this directory, is: + +```bash +# 1. Build the Solana base image +docker build -t seedemu-solana ../../../docker_images/seedemu-solana + +# 2. Generate the emulation (defaults to the host architecture; or pass amd|arm) +python3 solana_basic.py + +# 3. Build and start the cluster +cd output +# Build the local base images first. seedemu node images do `FROM `, +# where each base image is built by a "dummy" service (one per file under +# output/dummies/). docker compose v2 builds in parallel, so build these first: +docker compose build $(ls dummies) +docker compose up -d --build +``` + +Give the cluster a couple of minutes to converge the first time. Tear it down +only when you are done: + +```bash +(cd output && docker compose down -v --remove-orphans) +``` + +## Interacting and verifying + +The bootstrap validator's RPC listens on port `8899` inside the emulated +network. The cluster is not port-mapped to the host, so query it by exec-ing +into the bootstrap container (AS150 / host_0). Resolve the container from +`docker ps` each time you start a new shell, because container ids change when +Docker recreates containers: + +```bash +BOOT=$(docker ps --format '{{.Names}}' | grep '^as150h-Solana-Bootstrap-150-' | head -n 1) +test -n "$BOOT" || { echo "bootstrap container is not running"; exit 1; } + +# Cluster software version (proves RPC is up) +docker exec "$BOOT" solana --url http://127.0.0.1:8899 cluster-version + +# Current slot — run twice; it should increase (blocks are being produced) +docker exec "$BOOT" solana --url http://127.0.0.1:8899 slot +docker exec "$BOOT" solana --url http://127.0.0.1:8899 block-height + +# Epoch info (slot, epoch, absolute slot, transaction count) +docker exec "$BOOT" solana --url http://127.0.0.1:8899 epoch-info + +# Validators in the cluster (should list up to ten once converged) +docker exec "$BOOT" solana --url http://127.0.0.1:8899 validators + +# Nodes seen over gossip +docker exec "$BOOT" solana --url http://127.0.0.1:8899 gossip +``` + +Raw JSON-RPC also works (e.g. `getSlot`, `getBlockHeight`, `getEpochInfo`): + +```bash +docker exec "$BOOT" sh -c \ + 'curl -sS --connect-timeout 3 http://127.0.0.1:8899 \ + -X POST -H "Content-Type: application/json" \ + -d "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"getSlot\"}"; + echo' +``` + +## Send transactions + +Use the helper script instead of typing a long `docker exec` command. It finds +the bootstrap container, creates Alice/Bob keypairs on the first run, airdrops +SOL to Alice, transfers SOL to Bob, confirms the signature, and prints the final +balances: + +```bash +python3 send_transaction.py +``` + +You can change the airdrop and transfer amount: + +```bash +python3 send_transaction.py --airdrop 20 --amount 2 +``` + +By default, later runs reuse the same Alice/Bob keypairs, so Bob's balance +increases each time. Start over with fresh accounts when needed: + +```bash +python3 send_transaction.py --reset +``` + +If Alice is running out of SOL, top her up before the transfer: + +```bash +python3 send_transaction.py --top-up --airdrop 10 +``` + +Check the latest Alice/Bob balances created by the demo: + +```bash +python3 show_balances.py +``` + +The script still reaches the RPC through the bootstrap container because this +example does not expose port `8899` on the host. The important Solana flow is +unchanged: Alice's keypair signs the transfer, the RPC receives the signed +transaction, the validator pipeline forwards it to a leader, and the leader +records it in a produced block. + +## Deploy a program + +Solana smart contracts are called **programs**. Deploying a program means +uploading a compiled SBF shared object (`.so`) to a program account. The +`seedemu-solana` runtime image contains the validator and Solana CLI; it does +not contain the Rust/SBF compiler. Build the program outside the emulator, then +copy the resulting `.so` into the bootstrap container for deployment. + +This example includes a minimal Rust program at +`programs/noop_logger`. The program logs its program id, account count, and +instruction data length whenever it is invoked. + +If your host already has the Solana/Agave SBF toolchain, build and deploy in one +step: + +```bash +python3 deploy_program.py --build +``` + +If you already built the `.so`, deploy it directly: + +```bash +python3 deploy_program.py --so programs/noop_logger/dist/seedemu_solana_noop.so +``` + +The script copies the `.so` into the bootstrap container, creates/funds a +deployer account, runs `solana program deploy`, extracts the program id, and +prints `solana program show`. + +What happened during deployment: +- The deployer keypair signs the deployment transactions and pays fees/rent. +- The CLI uploads the SBF bytes into program-data accounts controlled by the + upgradeable loader. +- The loader marks the final program account executable. +- After that, any Solana client can send an instruction addressed to the program + id; validators execute the program while processing that transaction. +- Calling a custom program requires a client that constructs an instruction for + that program id; the Solana CLI manages deployment but does not provide a + generic `program invoke` subcommand. + +## Automated test + +`test_solana.sh` verifies an already-running cluster. It does **not** build +images, generate the emulation, start containers, recreate containers, or tear +anything down. Run `./prepare_solana.sh` first, then run: + +```bash +./test_solana.sh # verify the running cluster +MIN_VALIDATORS=1 ./test_solana.sh # accept a single voting validator +MIN_VALIDATORS=10 ./test_solana.sh # require all ten validators to be voting +``` + +## Logs and troubleshooting + +```bash +BOOT=$(docker ps --format '{{.Names}}' | grep '^as150h-Solana-Bootstrap-150-' | head -n 1) + +# Validator log (bootstrap) +docker exec "$BOOT" tail -n 100 -f /opt/solana/config/bootstrap-validator/*.log + +# Faucet log +docker exec "$BOOT" cat /var/log/solana-faucet.log + +# On a joining validator (AS151) +V=$(docker ps --format '{{.Names}}' | grep '^as151h-Solana-Validator-151-' | head -n 1) +docker exec "$V" tail -n 100 -f /opt/solana/config/validator/*.log +``` + +Notes: +- The first startup is slow: containers build, the bootstrap creates genesis, + then validators fetch genesis/snapshots over gossip and create vote accounts. +- A single staked validator already advances slots, so block production is + visible even before the other validators finish joining. +- Solana validators are CPU/IO intensive (see PRINCIPLES.md P9 on scale being a + real, resource-bounded concern). Give Docker ample CPU/RAM for larger clusters. +- The cluster runs natively on amd64 and arm64, so all ten Solana nodes + (including the cross-AS ones) converge on either architecture once BGP + converges. (Running the amd64 image under emulation on an arm64 host is + **not** recommended: the emulated validator binary cannot complete + non-loopback connects, which blocks the joining validators — build the native + arm64 image instead.) + +## Ports + +| Port | Purpose | +| ----------- | ---------------------------------------- | +| 8899 | JSON-RPC | +| 8900 | RPC PubSub (websocket) | +| 8001 | gossip | +| 9900 | faucet (bootstrap only) | +| 8002–8030 | dynamic range (TPU / turbine / repair) | + +## Design notes (alignment with PRINCIPLES.md) + +- **P1 (execution-agnostic):** the scenario never references Docker; only + `emu.compile(Docker(), ...)` selects the backend. +- **P4 (virtual nodes + late binding):** validators are symbolic vnodes; their + IPs and the bootstrap entrypoint are resolved from `Binding`s during + `configure()`. No IPs/ASNs are hard-coded in the service. +- **P5 (real production software):** runs real `agave-validator` / + `solana-genesis` / `solana-faucet`, pinned to a specific Agave release in + `docker_images/seedemu-solana/Dockerfile`. +- **P6 (self-contained module):** `SolanaService` exposes a small high-level API + (`createBlockchain`, `createBootstrapValidator`, `createValidator`) and hides + all genesis/keypair/faucet boilerplate. diff --git a/examples/blockchain/D70_solana/deploy_program.py b/examples/blockchain/D70_solana/deploy_program.py new file mode 100755 index 000000000..197388733 --- /dev/null +++ b/examples/blockchain/D70_solana/deploy_program.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +"""Build and deploy the example Solana program to the private cluster.""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from decimal import Decimal, InvalidOperation +from pathlib import Path + +from solana_docker import ( + RPC_URL, + docker_cp, + exec_shell, + exit_with_process_error, + find_bootstrap_container, + run_command, +) + +EXAMPLE_DIR = Path(__file__).resolve().parent +PROGRAM_DIR = EXAMPLE_DIR / "programs" / "noop_logger" +DEFAULT_SO = PROGRAM_DIR / "dist" / "seedemu_solana_noop.so" +CONTAINER_SO = "/tmp/seedemu_solana_noop.so" + + +def sol_amount(value: str) -> str: + """Parse and normalize a positive SOL amount.""" + try: + amount = Decimal(value) + except InvalidOperation as exc: + raise argparse.ArgumentTypeError(f"invalid SOL amount: {value}") from exc + if not amount.is_finite() or amount <= 0: + raise argparse.ArgumentTypeError("amount must be positive") + return format(amount, "f") + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Build/copy/deploy the noop_logger SBF program to D70_solana.", + ) + parser.add_argument( + "--build", + action="store_true", + help="run cargo build-sbf before deployment", + ) + parser.add_argument( + "--so", + type=Path, + default=DEFAULT_SO, + help=f"path to the compiled .so (default: {DEFAULT_SO.relative_to(EXAMPLE_DIR)})", + ) + parser.add_argument( + "--airdrop", + type=sol_amount, + default="100", + help="SOL to airdrop to the deployer account (default: 100)", + ) + return parser.parse_args() + + +def build_program() -> None: + """Run cargo build-sbf for the bundled example program.""" + if shutil.which("cargo") is None: + raise SystemExit( + "[fail] cargo was not found. Install the Solana/Agave SBF toolchain, " + "then rerun: python3 deploy_program.py --build" + ) + + proc = run_command( + ["cargo", "build-sbf", "--sbf-out-dir", "dist"], + cwd=PROGRAM_DIR, + ) + if proc.returncode != 0: + exit_with_process_error("cargo build-sbf failed", proc) + if proc.stdout: + print(proc.stdout, end="" if proc.stdout.endswith("\n") else "\n") + if proc.stderr: + print(proc.stderr, file=sys.stderr, end="" if proc.stderr.endswith("\n") else "\n") + + +def main() -> None: + """Deploy the example program.""" + args = parse_args() + if args.build: + build_program() + + so_path = args.so.expanduser() + if not so_path.is_absolute(): + so_path = Path.cwd() / so_path + if not so_path.is_file(): + raise SystemExit( + f"[fail] compiled program not found: {so_path}\n" + "Build it first with: python3 deploy_program.py --build" + ) + + container = find_bootstrap_container() + docker_cp(so_path, container, CONTAINER_SO) + + script = f""" +set -e +U={RPC_URL} +DEPLOYER_KEY=/tmp/seedemu_deployer.json +OUT=/tmp/seedemu_program_deploy.out + +solana-keygen new --no-passphrase -fso "$DEPLOYER_KEY" >/dev/null +DEPLOYER=$(solana-keygen pubkey "$DEPLOYER_KEY") + +echo "bootstrap_container={container}" +echo "deployer=$DEPLOYER" +solana --url "$U" airdrop {args.airdrop} "$DEPLOYER" >/dev/null +echo "deployer_balance=$(solana --url "$U" balance "$DEPLOYER")" + +solana --url "$U" program deploy \\ + --keypair "$DEPLOYER_KEY" \\ + {CONTAINER_SO} | tee "$OUT" + +PROGRAM_ID=$(sed -n 's/^Program Id: //p' "$OUT") +echo "program_id=$PROGRAM_ID" +test -n "$PROGRAM_ID" +solana --url "$U" program show "$PROGRAM_ID" +""" + print(exec_shell(container, script), end="") + + +if __name__ == "__main__": + main() diff --git a/examples/blockchain/D70_solana/prepare_solana.sh b/examples/blockchain/D70_solana/prepare_solana.sh new file mode 100755 index 000000000..223dc243b --- /dev/null +++ b/examples/blockchain/D70_solana/prepare_solana.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# Prepare the D70_solana emulation and leave all containers running. +# +# This script is intentionally separate from test_solana.sh: +# - prepare_solana.sh builds/generates/starts the environment +# - test_solana.sh only verifies an already-running environment +# +# Usage: +# ./prepare_solana.sh +# PLATFORM=linux/amd64 ./prepare_solana.sh + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/../../.." && pwd)" +OUTPUT_DIR="$HERE/output" +COMPOSE="docker compose" +PYTHON_BIN="${PYTHON_BIN:-$(cd "$REPO_ROOT" && command -v python3)}" + +case "$(uname -m)" in + arm64|aarch64) DEFAULT_PLATFORM="linux/arm64" ;; + *) DEFAULT_PLATFORM="linux/amd64" ;; +esac +PLATFORM="${PLATFORM:-$DEFAULT_PLATFORM}" +case "$PLATFORM" in + linux/arm64) EMU_ARCH="arm" ;; + linux/amd64) EMU_ARCH="amd" ;; + *) echo "Unsupported PLATFORM=$PLATFORM (expected linux/amd64 or linux/arm64)" >&2; exit 1 ;; +esac +export DOCKER_DEFAULT_PLATFORM="$PLATFORM" + +log() { echo -e "\033[1;34m[prepare]\033[0m $*"; } +ok() { echo -e "\033[1;32m[ ok ]\033[0m $*"; } +fail() { echo -e "\033[1;31m[fail]\033[0m $*"; } + +log "building seedemu-solana base image ($PLATFORM) ..." +docker build --platform "$PLATFORM" -t seedemu-solana "$REPO_ROOT/docker_images/seedemu-solana" \ + || { fail "base image build failed"; exit 1; } +ok "base image ready." + +log "generating emulation from solana_basic.py (arch: $EMU_ARCH) ..." +( cd "$HERE" && "$PYTHON_BIN" solana_basic.py "$EMU_ARCH" ) \ + || { fail "emulation generation failed"; exit 1; } +[[ -f "$OUTPUT_DIR/docker-compose.yml" ]] || { fail "no docker-compose.yml produced"; exit 1; } +ok "emulation generated." + +# seedemu node images do `FROM `, where each base image is +# built by a "dummy" compose service (one file per digest under output/dummies/). +# docker compose v2 builds services in parallel, so build these first. +dummy_services="$(ls "$OUTPUT_DIR/dummies" 2>/dev/null | tr '\n' ' ')" +if [[ -n "$dummy_services" ]]; then + log "pre-building base images: $dummy_services" + ( cd "$OUTPUT_DIR" && $COMPOSE build $dummy_services ) \ + || { fail "base (dummy) image build failed"; exit 1; } +fi + +log "building containers and starting the cluster ..." +( cd "$OUTPUT_DIR" && $COMPOSE up -d --build ) \ + || { fail "docker compose up failed"; exit 1; } + +ok "cluster containers are running." +echo "Verify with: ./test_solana.sh" +echo "Inspect bootstrap RPC with:" +echo " BOOT=\$(docker ps --format '{{.Names}}' | grep '^as150h-Solana-Bootstrap-150-' | head -n 1)" +echo " docker exec \"\$BOOT\" solana --url http://127.0.0.1:8899 cluster-version" +echo "Tear down manually with:" +echo " (cd output && docker compose down -v --remove-orphans)" diff --git a/examples/blockchain/D70_solana/programs/noop_logger/Cargo.toml b/examples/blockchain/D70_solana/programs/noop_logger/Cargo.toml new file mode 100644 index 000000000..574c317bb --- /dev/null +++ b/examples/blockchain/D70_solana/programs/noop_logger/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "seedemu_solana_noop" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +solana-program = "2.2" diff --git a/examples/blockchain/D70_solana/programs/noop_logger/src/lib.rs b/examples/blockchain/D70_solana/programs/noop_logger/src/lib.rs new file mode 100644 index 000000000..5e16f2371 --- /dev/null +++ b/examples/blockchain/D70_solana/programs/noop_logger/src/lib.rs @@ -0,0 +1,21 @@ +use solana_program::{ + account_info::AccountInfo, + entrypoint, + entrypoint::ProgramResult, + msg, + pubkey::Pubkey, +}; + +entrypoint!(process_instruction); + +pub fn process_instruction( + program_id: &Pubkey, + accounts: &[AccountInfo], + instruction_data: &[u8], +) -> ProgramResult { + msg!("seedemu_solana_noop invoked"); + msg!("program_id: {}", program_id); + msg!("account_count: {}", accounts.len()); + msg!("instruction_data_len: {}", instruction_data.len()); + Ok(()) +} diff --git a/examples/blockchain/D70_solana/send_transaction.py b/examples/blockchain/D70_solana/send_transaction.py new file mode 100755 index 000000000..f6ae386b8 --- /dev/null +++ b/examples/blockchain/D70_solana/send_transaction.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +"""Send a transfer transaction on the private Solana cluster.""" + +from __future__ import annotations + +import argparse +from decimal import Decimal, InvalidOperation + +from solana_docker import RPC_URL, exec_shell, find_bootstrap_container + + +def sol_amount(value: str) -> str: + """Parse and normalize a positive SOL amount.""" + try: + amount = Decimal(value) + except InvalidOperation as exc: + raise argparse.ArgumentTypeError(f"invalid SOL amount: {value}") from exc + if not amount.is_finite() or amount <= 0: + raise argparse.ArgumentTypeError("amount must be positive") + return format(amount, "f") + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Create Alice/Bob accounts and transfer SOL inside the D70_solana cluster.", + ) + parser.add_argument( + "--airdrop", + type=sol_amount, + default="10", + help="SOL to airdrop when Alice is first created, or with --top-up (default: 10)", + ) + parser.add_argument( + "--amount", + type=sol_amount, + default="1", + help="SOL to transfer from Alice to Bob (default: 1)", + ) + parser.add_argument( + "--reset", + action="store_true", + help="discard the previous demo Alice/Bob keypairs and start over", + ) + parser.add_argument( + "--top-up", + action="store_true", + help="airdrop --airdrop SOL to Alice even when reusing an existing keypair", + ) + return parser.parse_args() + + +def main() -> None: + """Run the transaction demo.""" + args = parse_args() + container = find_bootstrap_container() + + script = f""" +set -e +U={RPC_URL} +ALICE_KEY=/tmp/seedemu_alice.json +BOB_KEY=/tmp/seedemu_bob.json +OUT=/tmp/seedemu_transfer.out +RESET={1 if args.reset else 0} +TOP_UP={1 if args.top_up else 0} + +if [ "$RESET" = 1 ]; then + rm -f "$ALICE_KEY" "$BOB_KEY" +fi + +CREATED_ALICE=0 +CREATED_BOB=0 +if [ ! -f "$ALICE_KEY" ]; then + solana-keygen new --no-passphrase -fso "$ALICE_KEY" >/dev/null + CREATED_ALICE=1 +fi +if [ ! -f "$BOB_KEY" ]; then + solana-keygen new --no-passphrase -fso "$BOB_KEY" >/dev/null + CREATED_BOB=1 +fi + +ALICE=$(solana-keygen pubkey "$ALICE_KEY") +BOB=$(solana-keygen pubkey "$BOB_KEY") + +echo "bootstrap_container={container}" +echo "alice=$ALICE" +echo "alice_key_created=$CREATED_ALICE" +echo "bob=$BOB" +echo "bob_key_created=$CREATED_BOB" + +if [ "$CREATED_ALICE" = 1 ] || [ "$TOP_UP" = 1 ]; then + solana --url "$U" airdrop {args.airdrop} "$ALICE" >/dev/null + echo "airdrop_to_alice={args.airdrop} SOL" +else + echo "airdrop_to_alice=skipped" +fi +echo "alice_balance_before=$(solana --url "$U" balance "$ALICE")" + +solana --url "$U" transfer --from "$ALICE_KEY" --fee-payer "$ALICE_KEY" \\ + "$BOB" {args.amount} --allow-unfunded-recipient > "$OUT" +cat "$OUT" + +SIG=$(sed -n 's/^Signature: //p' "$OUT") +test -n "$SIG" +echo "signature=$SIG" +solana --url "$U" confirm -v "$SIG" + +echo "alice_balance_after=$(solana --url "$U" balance "$ALICE")" +echo "bob_balance_after=$(solana --url "$U" balance "$BOB")" +echo "transaction_count=$(solana --url "$U" transaction-count)" +""" + print(exec_shell(container, script), end="") + + +if __name__ == "__main__": + main() diff --git a/examples/blockchain/D70_solana/show_balances.py b/examples/blockchain/D70_solana/show_balances.py new file mode 100755 index 000000000..45b584529 --- /dev/null +++ b/examples/blockchain/D70_solana/show_balances.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +"""Show Alice and Bob balances from the transaction demo.""" + +from __future__ import annotations + +import argparse + +from solana_docker import RPC_URL, exec_shell, find_bootstrap_container + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Show Alice/Bob balances created by send_transaction.py.", + ) + return parser.parse_args() + + +def main() -> None: + """Print the demo account balances.""" + parse_args() + container = find_bootstrap_container() + script = f""" +set -e +U={RPC_URL} +ALICE_KEY=/tmp/seedemu_alice.json +BOB_KEY=/tmp/seedemu_bob.json + +test -f "$ALICE_KEY" || {{ echo "missing $ALICE_KEY; run python3 send_transaction.py first" >&2; exit 1; }} +test -f "$BOB_KEY" || {{ echo "missing $BOB_KEY; run python3 send_transaction.py first" >&2; exit 1; }} + +ALICE=$(solana-keygen pubkey "$ALICE_KEY") +BOB=$(solana-keygen pubkey "$BOB_KEY") + +echo "bootstrap_container={container}" +echo "alice=$ALICE" +echo "alice_balance=$(solana --url "$U" balance "$ALICE")" +echo "bob=$BOB" +echo "bob_balance=$(solana --url "$U" balance "$BOB")" +""" + print(exec_shell(container, script), end="") + + +if __name__ == "__main__": + main() diff --git a/examples/blockchain/D70_solana/solana_basic.py b/examples/blockchain/D70_solana/solana_basic.py new file mode 100644 index 000000000..bd3310ebf --- /dev/null +++ b/examples/blockchain/D70_solana/solana_basic.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +"""SolanaService example: a private, self-contained Solana cluster. + +Topology: a small Internet (10 stub ASes) on top of which we deploy a private +Agave/Solana cluster consisting of one bootstrap (genesis) validator and nine +joining validators distributed across the stub Autonomous Systems. + +This mirrors the structure of the Monero example (D60_monero): build the base +Internet with Makers, attach a blockchain service, bind its virtual nodes to +hosts, render, and compile to Docker. + +The cluster runs natively on both amd64 and arm64: the seedemu-solana base image +downloads Agave's prebuilt binaries on amd64 and compiles them from source on +arm64. Build the base image once before compiling: + + docker build -t seedemu-solana ../../../docker_images/seedemu-solana +""" + +from __future__ import annotations + +import platform as _platform +import sys +from pathlib import Path + +SCRIPT_PATH = Path(__file__).resolve() +EXAMPLE_DIR = SCRIPT_PATH.parent +REPO_ROOT = SCRIPT_PATH.parents[3] +if (REPO_ROOT / "seedemu").is_dir(): + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu import Binding, Filter, Makers +from seedemu.compiler import Docker, Platform +from seedemu.services.SolanaService import SolanaService + + +def _bind(emu, vnode: str, asn: int, host_index: int, display_name: str) -> None: + """Bind a Solana virtual node to host_ and name the host.""" + emu.getLayer('Base').getAutonomousSystem(asn).getHost( + f"host_{host_index}" + ).setDisplayName(display_name) + emu.addBinding(Binding(vnode, filter=Filter(asn=asn, nodeName=f"^host_{host_index}$"))) + + +############################################################################### +# Select the platform. With no argument we detect the host architecture; the +# seedemu-solana image is built natively per-arch (prebuilt on amd64, compiled +# from source on arm64), so both are first-class. +script_name = SCRIPT_PATH.name + + +def _detect_platform() -> Platform: + machine = _platform.machine().lower() + return Platform.ARM64 if machine in ('arm64', 'aarch64') else Platform.AMD64 + + +if len(sys.argv) == 1: + platform = _detect_platform() +elif len(sys.argv) == 2: + arg = sys.argv[1].lower() + if arg == 'amd': + platform = Platform.AMD64 + elif arg == 'arm': + platform = Platform.ARM64 + else: + print(f"Usage: {script_name} [amd|arm] (default: host architecture)") + sys.exit(1) +else: + print(f"Usage: {script_name} [amd|arm] (default: host architecture)") + sys.exit(1) + +############################################################################### +# Build the base topology: 10 stub ASes, each with 2 hosts. +hosts_per_stub_as = 2 +emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as=hosts_per_stub_as) + +############################################################################### +# Create a private Solana cluster. +solana = SolanaService() +blockchain = solana.createBlockchain("seed-solana") + +# AS150 / host_0: the bootstrap (genesis) validator. It builds the genesis +# ledger, runs the faucet, and is the gossip entrypoint for the rest of the +# cluster. +boot = blockchain.createBootstrapValidator("sol-boot-150") +boot.setDisplayName("Solana-Bootstrap-150") +_bind(emu, "sol-boot-150", asn=150, host_index=0, display_name="Solana-Bootstrap-150") + +# Nine joining validators. AS150/host_1 is on-link with the bootstrap; the rest +# join over inter-domain (BGP) routing, demonstrating a larger private Solana +# cluster spread across the emulated Internet. +validator_bindings = [ + ("sol-validator-150b", "Solana-Validator-150b", 150, 1), + ("sol-validator-151", "Solana-Validator-151", 151, 0), + ("sol-validator-152", "Solana-Validator-152", 152, 0), + ("sol-validator-153", "Solana-Validator-153", 153, 0), + ("sol-validator-154", "Solana-Validator-154", 154, 0), + ("sol-validator-160", "Solana-Validator-160", 160, 0), + ("sol-validator-161", "Solana-Validator-161", 161, 0), + ("sol-validator-162", "Solana-Validator-162", 162, 0), + ("sol-validator-163", "Solana-Validator-163", 163, 0), +] + +for vnode, display_name, asn, host_index in validator_bindings: + validator = blockchain.createValidator(vnode) + validator.setDisplayName(display_name) + _bind(emu, vnode, asn=asn, host_index=host_index, display_name=display_name) + +############################################################################### +# Add the Solana service as a layer, then render. +emu.addLayer(solana) +emu.render() + +############################################################################### +# Compile docker-compose and the build context into ./output. +docker = Docker(internetMapEnabled=True, platform=platform) +emu.compile(docker, str(EXAMPLE_DIR / "output"), override=True) diff --git a/examples/blockchain/D70_solana/solana_docker.py b/examples/blockchain/D70_solana/solana_docker.py new file mode 100755 index 000000000..7af08d32e --- /dev/null +++ b/examples/blockchain/D70_solana/solana_docker.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +"""Helpers for driving the private Solana cluster from example scripts.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from typing import Optional + +BOOTSTRAP_PREFIX = "as150h-Solana-Bootstrap-150-" +RPC_URL = "http://127.0.0.1:8899" + + +def run_command( + args: list[str], + *, + cwd: Optional[Path] = None, + input_text: Optional[str] = None, +) -> subprocess.CompletedProcess[str]: + """Run a command and capture stdout/stderr.""" + return subprocess.run( + args, + cwd=cwd, + input=input_text, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def exit_with_process_error(context: str, proc: subprocess.CompletedProcess[str]) -> None: + """Print a failed subprocess result and exit.""" + print(f"[fail] {context}", file=sys.stderr) + if proc.stdout: + print(proc.stdout, file=sys.stderr, end="" if proc.stdout.endswith("\n") else "\n") + if proc.stderr: + print(proc.stderr, file=sys.stderr, end="" if proc.stderr.endswith("\n") else "\n") + raise SystemExit(proc.returncode or 1) + + +def find_bootstrap_container() -> str: + """Return the running bootstrap container name.""" + proc = run_command(["docker", "ps", "--format", "{{.Names}}"]) + if proc.returncode != 0: + exit_with_process_error("docker ps failed", proc) + + for name in proc.stdout.splitlines(): + if name.startswith(BOOTSTRAP_PREFIX): + return name + + print( + f"[fail] no running bootstrap container found ({BOOTSTRAP_PREFIX}...).", + file=sys.stderr, + ) + print("Run ./prepare_solana.sh first.", file=sys.stderr) + raise SystemExit(1) + + +def exec_shell(container: str, script: str) -> str: + """Run a shell script inside the bootstrap container.""" + proc = run_command(["docker", "exec", "-i", container, "sh", "-s"], input_text=script) + if proc.returncode != 0: + exit_with_process_error("docker exec failed", proc) + if proc.stderr: + print(proc.stderr, file=sys.stderr, end="" if proc.stderr.endswith("\n") else "\n") + return proc.stdout + + +def docker_cp(src: Path, container: str, dst: str) -> None: + """Copy a local file into a container.""" + proc = run_command(["docker", "cp", str(src), f"{container}:{dst}"]) + if proc.returncode != 0: + exit_with_process_error("docker cp failed", proc) diff --git a/examples/blockchain/D70_solana/test_solana.sh b/examples/blockchain/D70_solana/test_solana.sh new file mode 100755 index 000000000..a9b88a933 --- /dev/null +++ b/examples/blockchain/D70_solana/test_solana.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# +# Verify an already-running D70_solana private Solana cluster. +# +# This script intentionally does not build images, generate the emulation, +# start containers, or tear containers down. Prepare the environment first with +# ./prepare_solana.sh, then run this script as many times as needed while you +# inspect the live cluster. +# +# Usage: +# ./prepare_solana.sh # one-time setup/start +# ./test_solana.sh # verify the running cluster +# MIN_VALIDATORS=1 ./test_solana.sh + +set -uo pipefail + +# Bootstrap validator: AS150 / host_0. Its RPC is reachable on the emulated +# network; we reach it by exec-ing into the container (no host port mapping). +BOOT_NAME_PREFIX="as150h-Solana-Bootstrap-150-" + +# How long to wait (seconds) for the chain to start producing blocks. +MAX_WAIT="${MAX_WAIT:-420}" +# Minimum number of *voting* validators required for a hard pass. On a native +# image for your host architecture (amd64 or arm64) and with enough CPU/RAM, +# all validators can converge. Set REQUIRE_MULTI=1 to make this a hard +# requirement. +MIN_VALIDATORS="${MIN_VALIDATORS:-2}" + +log() { echo -e "\033[1;34m[test]\033[0m $*"; } +ok() { echo -e "\033[1;32m[ ok ]\033[0m $*"; } +fail() { echo -e "\033[1;31m[fail]\033[0m $*"; } + +if [[ $# -gt 0 ]]; then + fail "test_solana.sh does not take lifecycle arguments; it only verifies a running cluster." + echo "Prepare the cluster with ./prepare_solana.sh, then run ./test_solana.sh." + exit 2 +fi + +find_boot_container() { + docker ps --format '{{.Names}}' | grep "^$BOOT_NAME_PREFIX" | head -n 1 +} + +boot_cid="$(find_boot_container)" +if [[ -z "$boot_cid" ]]; then + fail "could not find a running bootstrap container (${BOOT_NAME_PREFIX}...)." + docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Status}}' + echo "Run ./prepare_solana.sh first." + exit 1 +fi + +ok "bootstrap container is running: $boot_cid" + +# Helper: resolve the current bootstrap container name before every exec, so a +# recreated container does not leave the test script holding a stale reference. +RPC_URL_LOCAL="http://127.0.0.1:8899" +exec_boot() { + local cid + cid="$(find_boot_container)" + [[ -n "$cid" ]] || return 1 + docker exec "$cid" "$@" +} + +sol() { + local tries=0 out + while (( tries < 6 )); do + if out=$(exec_boot solana --url "$RPC_URL_LOCAL" "$@" 2>/dev/null); then + echo "$out"; return 0 + fi + tries=$((tries+1)); sleep 2 + done + return 1 +} + +# --------------------------------------------------------------------------- +# 4) Verify the chain is alive and producing blocks. +# --------------------------------------------------------------------------- +log "waiting for the bootstrap RPC to come online ..." +deadline=$(( $(date +%s) + MAX_WAIT )) +rpc_up=0 +ver="" +while (( $(date +%s) < deadline )); do + if ver=$(sol cluster-version) && [[ -n "$ver" ]]; then + rpc_up=1; break + fi + sleep 10 +done +if (( rpc_up != 1 )); then + fail "RPC never came online within ${MAX_WAIT}s" + exec_boot sh -c 'tail -n 40 /opt/solana/config/bootstrap-validator/*.log' 2>/dev/null || true + exit 1 +fi +ok "RPC online: cluster-version $ver" + +log "checking that slots are advancing (block production) ..." +slot1=$(sol slot | tr -dc '0-9'); slot1=${slot1:-0} +log "slot now: $slot1 ; waiting ~30s ..." +sleep 30 +slot2=$(sol slot | tr -dc '0-9'); slot2=${slot2:-0} +log "slot now: $slot2" + +if [[ "$slot2" -gt "$slot1" ]]; then + ok "slots are advancing ($slot1 -> $slot2): the chain is producing blocks." +else + fail "slots did not advance ($slot1 -> $slot2): chain is not producing blocks." + exec_boot sh -c 'tail -n 60 /opt/solana/config/bootstrap-validator/*.log' 2>/dev/null || true + exit 1 +fi + +# Confirm block height is non-zero too. +bh=$(sol block-height | tr -dc '0-9') +log "block-height: ${bh:-unknown}" + +# --------------------------------------------------------------------------- +# 4b) Verify the chain actually processes transactions (airdrop -> balance). +# --------------------------------------------------------------------------- +log "verifying transaction processing (airdrop to a fresh account) ..." +tx_ok=0 +if exec_boot bash -c ' + set -e + U=http://127.0.0.1:8899 + solana-keygen new --no-passphrase -fso /tmp/seedemu_test_payer.json >/dev/null 2>&1 + PK=$(solana-keygen pubkey /tmp/seedemu_test_payer.json) + solana --url $U airdrop 5 "$PK" >/dev/null 2>&1 + for i in $(seq 1 10); do + bal=$(solana --url $U balance "$PK" 2>/dev/null | grep -oE "^[0-9.]+") + awk "BEGIN{exit !($bal > 0)}" && exit 0 + sleep 2 + done + exit 1 + ' 2>/dev/null; then + tx_ok=1 + ok "transaction processed: airdrop credited a new account (ledger is executing transactions)." +else + fail "transaction did not confirm (airdrop balance stayed 0)." + exec_boot sh -c 'tail -n 40 /opt/solana/config/bootstrap-validator/*.log' 2>/dev/null || true + exit 1 +fi + +# --------------------------------------------------------------------------- +# 5) Verify the joining validators register and vote in the cluster. +# +# This is the multi-host part of the cluster. It converges when the image is +# native to the host architecture and Docker has enough CPU/RAM. Under +# cross-architecture emulation, joining validators may not converge; that is an +# emulation-host limitation, not a defect in the generated topology. It is +# therefore best-effort by default; set REQUIRE_MULTI=1 to make it a hard +# requirement. +# --------------------------------------------------------------------------- +log "waiting for joining validators to vote (up to ${MAX_WAIT}s; need >= $MIN_VALIDATORS) ..." +deadline=$(( $(date +%s) + MAX_WAIT )) +nval=0 +while (( $(date +%s) < deadline )); do + nval=$(sol validators 2>/dev/null | grep -oE '[0-9]+ current validators' | grep -oE '^[0-9]+' | head -1) + nval=${nval:-0} + log "voting validators in cluster: $nval (target >= $MIN_VALIDATORS)" + (( nval >= MIN_VALIDATORS )) && break + sleep 15 +done + +echo +log "validator set:" +sol validators || true +echo + +multi_ok=0 +if (( nval >= MIN_VALIDATORS )); then + (( nval >= 2 )) && multi_ok=1 + if (( multi_ok == 1 )); then + ok "$nval validators are voting (multi-validator consensus working)." + else + ok "$nval validator is voting." + fi +elif [[ "${REQUIRE_MULTI:-0}" == "1" ]]; then + fail "only $nval voting validator(s); expected >= $MIN_VALIDATORS (REQUIRE_MULTI=1)." + exec_boot sh -c 'tail -n 40 /opt/solana/config/bootstrap-validator/*.log' 2>/dev/null || true + exit 1 +else + echo -e "\033[1;33m[warn]\033[0m only $nval voting validator(s) converged here." + echo " Multi-host convergence needs native images for your host architecture and enough CPU/RAM." +fi + +# --------------------------------------------------------------------------- +# Verdict. The core requirement — a working Solana chain that produces blocks +# and processes transactions — is verified by the hard checks above. +# --------------------------------------------------------------------------- +echo +ok "SUCCESS: the private Solana chain is live, producing blocks, and processing transactions." +echo " cluster-version : $ver" +echo " slot : $slot1 -> $slot2" +echo " block-height : ${bh:-unknown}" +echo " transaction test : $([[ $tx_ok == 1 ]] && echo passed || echo failed)" +echo " voting validators : $nval $([[ $multi_ok == 1 ]] && echo '(multi-validator consensus)' || echo '(single voting validator)')" +echo " cluster state : left running" diff --git a/examples/blockchain/R00_ethereum_pos_test/batch_docker_compose_build.sh b/examples/blockchain/R00_ethereum_pos_test/batch_docker_compose_build.sh new file mode 100644 index 000000000..e1ee8a253 --- /dev/null +++ b/examples/blockchain/R00_ethereum_pos_test/batch_docker_compose_build.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Disable Docker BuildKit to avoid multi-thread conflicts +export DOCKER_BUILDKIT=0 + +# =============================================== +# Batch docker-compose build script +# Default batch size: 50 services per batch +# Includes logging, retry on failure, and detailed output +# =============================================== + +BATCH_SIZE=50 +LOG_FILE="batch_compose_build.log" +COMPOSE_FILE="./output/docker-compose.yml" + +echo "===== Executing docker compose build in batches =====" | tee "$LOG_FILE" +echo "Batch size: $BATCH_SIZE" | tee -a "$LOG_FILE" +echo "Start time: $(date)" | tee -a "$LOG_FILE" + +if [ ! -f "$COMPOSE_FILE" ]; then + echo "No compose file at $COMPOSE_FILE. Exiting script." | tee -a "$LOG_FILE" + exit 1 +fi +services=($(docker compose -f "$COMPOSE_FILE" config --services)) +total=${#services[@]} +echo "Total number of services: $total" | tee -a "$LOG_FILE" + +if [ $total -eq 0 ]; then + echo "? No services found in $COMPOSE_FILE. Exiting script." | tee -a "$LOG_FILE" + exit 1 +fi + +batch_count=0 + +# =============================================== +# Batch build +for ((i=0; i [amd|arm]") + sys.exit(1) + +# Read total number of Ethereum beaconnodes from the command line argument +try: + total_number_of_beaconnodes = int(sys.argv[1]) +except ValueError: + print(f"Invalid number of Ethereum beaconnodes: {sys.argv[1]}") + sys.exit(1) + +# Optional platform argument +if len(sys.argv) == 3: + if sys.argv[2].lower() == 'amd': + platform = Platform.AMD64 + elif sys.argv[2].lower() == 'arm': + platform = Platform.ARM64 + else: + print(f"Usage: {script_name} amd|arm") + sys.exit(1) +else: + platform = Platform.AMD64 # Default platform is AMD64 + +# Calculate the number of each type of node +# 1. Geth nodes: one per beacon node +# 2. Beacon nodes: one per VC node +# 3. VC nodes: three per beacon node +# 4. Beacon setup node: one +geth_node_number = total_number_of_beaconnodes +beacon_node_number = total_number_of_beaconnodes +vc_node_number = 3 * total_number_of_beaconnodes +beaconsetup_node_number = 1 + +total_number_of_nodes = geth_node_number + beacon_node_number + vc_node_number + beaconsetup_node_number + + +# Calculate how many hosts per stub AS are needed +# We know we have 10 stub AS (150-154, 160-164), and at least one node per AS is required +# to host a node (Beacon or Ethereum node). +total_stub_as = 10 # We have 10 ASNs available +hosts_per_stub_as = math.ceil(total_number_of_nodes / total_stub_as) + +# Create Emulator Base with the calculated number of hosts per stub AS +emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as=hosts_per_stub_as) + +print(f"Number of eth nodes per stub AS: {hosts_per_stub_as}") + +# Create the Ethereum layer +eth = EthereumService() +### EthereumService -> Blockchain -> EthServer (GethServer/BeaconServer/BeaconSetupServer) +# Create the Blockchain layer as a sub-layer of the Ethereum layer; configure it for POS consensus +blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) + + + + +# Generate a list of accounts and prefund them +accounts_total = 1000 +pre_funded_amount = 1000000 +mnemonic = "gentle always fun glass foster produce north tail security list example gain" +Account.enable_unaudited_hdwallet_features() +for i in range(accounts_total): + account = Account.from_mnemonic(mnemonic, account_path=f"m/44'/60'/0'/0/{i}") + blockchain.addLocalAccount(address=account.address, balance=pre_funded_amount) + + +asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] + +################################################### + + + + +geth_nodes: List[PoSGethServer] = [] +beacon_nodes: List[PoSBeaconServer] = [] + +vc_nodes: List[PoSVcServer] =[] + +### Create beacon setup node +beaconsetupServer: PoSBeaconSetupServer = blockchain.createBeaconSetupNode(f"BeaconSetupNode") +emu.getVirtualNode(f'BeaconSetupNode').setDisplayName('Ethereum-POS-BeaconSetup') +### Create geth nodes +for i in range(geth_node_number): + gethServer: PoSGethServer = blockchain.createGethNode(f"gethnode{i}") + gethServer.enableGethHttp() + gethServer.appendClassName(f'Ethereum-POS-Geth-{i+1}') + geth_nodes.append(gethServer) + emu.getVirtualNode(f'gethnode{i}').setDisplayName(f'Ethereum-POS-Geth-{i+1}') +## Create beacon nodes +for i in range(beacon_node_number): + beaconServer: PoSBeaconServer = blockchain.createBeaconNode(f"beaconnode{i}") + beaconServer.appendClassName(f'Ethereum-POS-Beacon-{i+1}') + beaconServer.connectToGethNode(f"gethnode{(i+1)%len(geth_nodes)}") + beacon_nodes.append(beaconServer) + emu.getVirtualNode(f'beaconnode{i}').setDisplayName(f'Ethereum-POS-Beacon-{i+1}') + # beaconServer.enablePOSValidatorAtGenesis() +# Set boot nodes +geth_nodes[0].setBootNode(True) +beacon_nodes[0].setBootNode(True) +# Set beacon_node cmd for eth viewer +# beacon_nodes[1].addViewercmd(True) +# beacon_nodes[1].setDisplayName('Ethereum-POS-Beacon-Viewer') + +for i in range(vc_node_number): + VcServer: PoSVcServer=blockchain.createVcNode(f"vcnode{i}") + VcServer.appendClassName(f'Ethereum-POS-Validator-{i+1}') + + VcServer.connectToBeaconNode(f"beaconnode{(i+1)%len(beacon_nodes)}") + VcServer.enablePOSValidatorAtGenesis() + vc_nodes.append(VcServer) + emu.getVirtualNode(f'vcnode{i}').setDisplayName(f'Ethereum-POS-Validator-{i+1}') + + +assign_index = 0 +total_nodes = len(geth_nodes) + len(beacon_nodes) + len(vc_nodes) +for asn in asns: + for id in range(hosts_per_stub_as): + if asn == 152 and id == 0: + emu.addBinding(Binding('BeaconSetupNode', + filter=Filter(asn=asn, nodeName=f'^host_{id}$'), + action=Action.FIRST)) + else: + if assign_index >= total_nodes: + continue + if assign_index < len(geth_nodes): + name = f'gethnode{assign_index}' + elif assign_index < len(geth_nodes) + len(beacon_nodes): + name = f'beaconnode{assign_index - len(geth_nodes)}' + else: + name = f'vcnode{assign_index - len(geth_nodes) - len(beacon_nodes)}' + emu.addBinding(Binding(name, + filter=Filter(asn=asn, nodeName=f'^host_{id}$'), + action=Action.FIRST)) + assign_index += 1 + + + +# Add Ethereum layer to the emulator +emu.addLayer(eth) +base_layer = emu.getLayer('Base') +for asn in asns: + as_obj = base_layer.getAutonomousSystem(asn) + net = as_obj.getNetwork('net0') + # Extend host IP range from 71-99 to 71-199 (supports 129 hosts, enough for 50 per AS) + # Router range is 200-254, so host must end at 199 to avoid conflict + # Move DHCP range to 51-70 to avoid conflict with extended host range (71-199) + net.setHostIpRange(hostStart=71, hostEnd=199, hostStep=1) + +emu.render() + +# Enable internetMap and etherView for visualization +docker = Docker(internetMapEnabled=True, etherViewEnabled=True, platform=platform) + +# Compile the emulator to output +emu.compile(docker, './output', override=True) diff --git a/examples/internet/B00_mini_internet/README.md b/examples/internet/B00_mini_internet/README.md index f4d4185b5..84ed480dc 100644 --- a/examples/internet/B00_mini_internet/README.md +++ b/examples/internet/B00_mini_internet/README.md @@ -11,6 +11,18 @@ inside the emulator. The emulator generated from this example is saved to a component file, and be used by several other examples as the basis. +The public Python entrypoint is still: + +```python +from examples.internet.B00_mini_internet import mini_internet + +mini_internet.run(dumpfile='./base_internet.bin') +mini_internet.run(dumpfile='./base_internet.bin', hosts_per_as=2) +``` + +Other examples use this API to build their underlying network topology, so the +`run(dumpfile=None, hosts_per_as=2, ...)` signature is kept compatible. + ## Using Utility Functions @@ -36,3 +48,51 @@ two not running any service. Makers.makeStubAs(emu, base, 153, 101, [web, None, None]) ``` +## Standard Arguments + +The example accepts both the legacy platform argument and the newer named +arguments: + +```sh +python examples/internet/B00_mini_internet/mini_internet.py amd +python examples/internet/B00_mini_internet/mini_internet.py --platform amd --output examples/internet/B00_mini_internet/output +python examples/internet/B00_mini_internet/mini_internet.py --dumpfile examples/internet/B00_mini_internet/base_internet.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--hosts-per-as N`: number of hosts created in each stub AS. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +This example includes an `example.yaml` manifest for `seedemu.testing`. Run +these commands from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/internet/B00_mini_internet/example.yaml +python seedemu/testing/cli.py compile examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +python seedemu/testing/cli.py build examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +python seedemu/testing/cli.py up examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +python seedemu/testing/cli.py probe examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +python seedemu/testing/cli.py test examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +python seedemu/testing/cli.py down examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B00_mini_internet/example.yaml --artifact-dir ci-artifacts/b00 +``` + +The manifest declares `runner: internet` because it uses Internet-style probes. +The readiness stage checks representative transit routers, stub routers, and +stub hosts. The probe stage checks cross-AS reachability across different parts +of the mini Internet. The `test_runtime.py` program demonstrates custom runtime +validation, including a check for the AS154 host with the customized IP address. + diff --git a/examples/internet/B00_mini_internet/example.yaml b/examples/internet/B00_mini_internet/example.yaml new file mode 100644 index 000000000..592965be2 --- /dev/null +++ b/examples/internet/B00_mini_internet/example.yaml @@ -0,0 +1,79 @@ +id: internet-b00-mini-internet +name: Mini Internet +description: A reusable mini-Internet topology with transit ASes, route-server peerings, and stub ASes. +runner: internet +script: mini_internet.py +platform: amd +features: + - ipv4-default + - mini-internet + - ebgp-route-server + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: representative B00 services are running + type: compose-ps + services: + - brdnode_2_r100 + - brdnode_2_r101 + - brdnode_3_r103 + - brdnode_4_r104 + - brdnode_11_r102 + - brdnode_12_r101 + - brdnode_150_router0 + - brdnode_152_router0 + - brdnode_160_router0 + - brdnode_171_router0 + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_160_host_0 + - hnode_171_host_0 + retries: 40 + interval: 3 + +probes: + - name: AS150 reaches AS152 across tier-1 and tier-2 transit + type: ping + service: hnode_150_host_0 + target: 10.152.0.71 + count: 3 + retries: 40 + interval: 5 + + - name: AS150 reaches AS160 through AS3 + type: ping + service: hnode_150_host_0 + target: 10.160.0.71 + count: 3 + retries: 40 + interval: 5 + + - name: AS171 reaches AS154 customized host + type: ping + service: hnode_171_host_0 + target: 10.154.0.129 + count: 3 + retries: 40 + interval: 5 + +test_programs: + - name: B00 custom runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/internet/B00_mini_internet/mini_internet.py b/examples/internet/B00_mini_internet/mini_internet.py index b584d64e7..d4d67a996 100755 --- a/examples/internet/B00_mini_internet/mini_internet.py +++ b/examples/internet/B00_mini_internet/mini_internet.py @@ -1,36 +1,48 @@ #!/usr/bin/env python3 # encoding: utf-8 -from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf, PeerRelationship +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu.compiler import Docker, Platform from seedemu.core import Emulator +from seedemu.layers import Base, Ebgp, Ibgp, Ospf, PeerRelationship, Routing from seedemu.utilities import Makers -import os, sys -def run(dumpfile=None, hosts_per_as=2): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - - emu = Emulator() - ebgp = Ebgp() - base = Base() - + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B00 mini Internet example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator(hosts_per_as=2) -> Emulator: + emu = Emulator() + ebgp = Ebgp() + base = Base() + ############################################################################### # Create internet exchanges ix100 = base.createInternetExchange(100) @@ -39,39 +51,46 @@ def run(dumpfile=None, hosts_per_as=2): ix103 = base.createInternetExchange(103) ix104 = base.createInternetExchange(104) ix105 = base.createInternetExchange(105) - + # Customize names (for visualization purpose) - ix100.getPeeringLan().setDisplayName('NYC-100') - ix101.getPeeringLan().setDisplayName('San Jose-101') - ix102.getPeeringLan().setDisplayName('Chicago-102') - ix103.getPeeringLan().setDisplayName('Miami-103') - ix104.getPeeringLan().setDisplayName('Boston-104') - ix105.getPeeringLan().setDisplayName('Huston-105') - - + ix100.getPeeringLan().setDisplayName("NYC-100") + ix101.getPeeringLan().setDisplayName("San Jose-101") + ix102.getPeeringLan().setDisplayName("Chicago-102") + ix103.getPeeringLan().setDisplayName("Miami-103") + ix104.getPeeringLan().setDisplayName("Boston-104") + ix105.getPeeringLan().setDisplayName("Huston-105") + ############################################################################### - # Create Transit Autonomous Systems - + # Create Transit Autonomous Systems + ## Tier 1 ASes - Makers.makeTransitAs(base, 2, [100, 101, 102, 105], - [(100, 101), (101, 102), (100, 105)] + Makers.makeTransitAs( + base, + 2, + [100, 101, 102, 105], + [(100, 101), (101, 102), (100, 105)], ) - - Makers.makeTransitAs(base, 3, [100, 103, 104, 105], - [(100, 103), (100, 105), (103, 105), (103, 104)] + + Makers.makeTransitAs( + base, + 3, + [100, 103, 104, 105], + [(100, 103), (100, 105), (103, 105), (103, 104)], ) - - Makers.makeTransitAs(base, 4, [100, 102, 104], - [(100, 104), (102, 104)] + + Makers.makeTransitAs( + base, + 4, + [100, 102, 104], + [(100, 104), (102, 104)], ) - + ## Tier 2 ASes Makers.makeTransitAs(base, 11, [102, 105], [(102, 105)]) Makers.makeTransitAs(base, 12, [101, 104], [(101, 104)]) - - + ############################################################################### - # Create single-homed stub ASes. + # Create single-homed stub ASes. Makers.makeStubAsWithHosts(emu, base, 150, 100, hosts_per_as) Makers.makeStubAsWithHosts(emu, base, 151, 100, hosts_per_as) Makers.makeStubAsWithHosts(emu, base, 152, 101, hosts_per_as) @@ -84,70 +103,93 @@ def run(dumpfile=None, hosts_per_as=2): Makers.makeStubAsWithHosts(emu, base, 164, 104, hosts_per_as) Makers.makeStubAsWithHosts(emu, base, 170, 105, hosts_per_as) Makers.makeStubAsWithHosts(emu, base, 171, 105, hosts_per_as) - - # An example to show how to add a host with customized IP address + + # An example to show how to add a host with customized IP address. as154 = base.getAutonomousSystem(154) - new_host = as154.createHost('host_new').joinNetwork('net0', address = '10.154.0.129') - from seedemu.core import OptionRegistry, OptionMode + new_host = as154.createHost("host_new").joinNetwork("net0", address="10.154.0.129") + from seedemu.core import OptionMode, OptionRegistry - - o = OptionRegistry().sysctl_netipv4_conf_rp_filter({'all': False, 'default': False, 'net0': False}, mode = OptionMode.RUN_TIME) + o = OptionRegistry().sysctl_netipv4_conf_rp_filter( + {"all": False, "default": False, "net0": False}, + mode=OptionMode.RUN_TIME, + ) new_host.setOption(o) - o = OptionRegistry().sysctl_netipv4_udp_rmem_min(5000, mode = OptionMode.RUN_TIME) + o = OptionRegistry().sysctl_netipv4_udp_rmem_min(5000, mode=OptionMode.RUN_TIME) new_host.setOption(o) - + ############################################################################### - # Peering via RS (route server). The default peering mode for RS is PeerRelationship.Peer, - # which means each AS will only export its customers and their own prefixes. - # We will use this peering relationship to peer all the ASes in an IX. - # None of them will provide transit service for others. - + # Peering via route servers. ebgp.addRsPeers(100, [2, 3, 4]) ebgp.addRsPeers(102, [2, 4]) ebgp.addRsPeers(104, [3, 4]) ebgp.addRsPeers(105, [2, 3]) - - # To buy transit services from another autonomous system, - # we will use private peering - - ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) - ebgp.addPrivatePeerings(100, [3], [150], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) + + # Private peerings for transit service. + ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) + ebgp.addPrivatePeerings(100, [3], [150], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) ebgp.addPrivatePeerings(101, [12], [152, 153], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) ebgp.addPrivatePeerings(102, [11], [154], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(103, [3], [160, 161, 162], PeerRelationship.Provider) - + + ebgp.addPrivatePeerings(103, [3], [160, 161, 162], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [3, 4], [12], PeerRelationship.Provider) - ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) ebgp.addPrivatePeerings(104, [12], [164], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) ebgp.addPrivatePeerings(105, [11], [171], PeerRelationship.Provider) - - + ############################################################################### # Add layers to the emulator emu.addLayer(base) emu.addLayer(Routing()) - emu.addLayer(ebgp) + emu.addLayer(ebgp) emu.addLayer(Ibgp()) emu.addLayer(Ospf()) + return emu - if dumpfile is not None: - # Save it to a file, so it can be used by other emulators + +def run( + dumpfile=None, + hosts_per_as=2, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator(hosts_per_as=hosts_per_as) + if dumpfile is not None: + # Save it to a file, so it can be used by other emulators. emu.dump(dumpfile) - else: + return + + if render: emu.render() - # Attach the Internet Map container to the emulator - docker = Docker(platform=platform) - emu.compile(docker, './output', override=True) + docker = Docker(platform=platform) + emu.compile(docker, output or "./output", override=override) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(output_dir), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + if __name__ == "__main__": - run() + raise SystemExit(main()) diff --git a/examples/internet/B00_mini_internet/test_runtime.py b/examples/internet/B00_mini_internet/test_runtime.py new file mode 100644 index 000000000..16fb11acf --- /dev/null +++ b/examples/internet/B00_mini_internet/test_runtime.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + host150 = test.require_service(150, "host_0") + host152 = test.require_service(152, "host_0") + host160 = test.require_service(160, "host_0") + host171 = test.require_service(171, "host_0") + host154_new = test.require_service(154, "host_new") + + if host150 and host152: + test.exec_check("AS150 reaches AS152 across the mini Internet", host150, "ping -c 3 {} >/dev/null".format(host152.address)) + if host150 and host160: + test.exec_check("AS150 reaches AS160 through AS3", host150, "ping -c 3 {} >/dev/null".format(host160.address)) + if host171 and host154_new: + test.exec_check("AS171 reaches AS154 customized host", host171, "ping -c 3 {} >/dev/null".format(host154_new.address)) + if host154_new: + test.exec_check( + "AS154 customized host has the expected address", + host154_new, + "ip addr show net0 | grep -q '10.154.0.129'", + ) + + test.write_summary("b00-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B01_dns_component/README.md b/examples/internet/B01_dns_component/README.md index bf39286fd..4687a3c0a 100644 --- a/examples/internet/B01_dns_component/README.md +++ b/examples/internet/B01_dns_component/README.md @@ -3,19 +3,24 @@ This example demonstrates how we can build a DNS infrastructure as a component. We generate a mini DNS infrastructure and save it into a file as a DNS component. This component can be loaded into other emulators, which -means deploying the DNS infrastructure in those emulators. +means deploying the DNS infrastructure in those emulators. + +This example is not a standalone emulator. It does not create a complete +network topology, compile Docker files, or run containers. Its only output is +the serialized DNS component file, `dns_component.bin`. Runtime testing should +be done by an emulator that loads and deploys this component, such as a later +example built for that purpose. + In this mini DNS, we created the following nameservers: - Root servers: `a-root-server` and `b-root-server` - COM servers: `a-com-server` and `b-com-server` - NET server: `a-net-server` - EDU server: `a-edu-server` -- Two ccTLD servers: `a-cn-server` and `b-cn-server` - `twitter.com` nameserver: `ns-twitter-com` - `google.com` nameserver: `ns-google-com` - `example.net` nameserver: `ns-example-net` - `syr.edu` nameserver: `ns-syr-edu` -- `weibo.cn` nameserver: `ns-weibo-cn` ## Creating Virtual Nameserver @@ -70,5 +75,21 @@ infrastructure in their emulation by loading this component. ``` emu.addLayer(dns) -emu.dump('dns-component.bin') +emu.dump('dns_component.bin') ``` + +## Testing this component + +This example only verifies the component-building workflow. A good runtime test +should be written in an emulator that consumes this component. Such a test can +load `dns_component.bin`, bind the virtual DNS nodes to physical nodes, start +the emulator, and then run DNS queries for records such as: + +- `twitter.com. A 1.1.1.1` +- `google.com. A 2.2.2.2` +- `example.net. A 3.3.3.3` +- `syr.edu. A 128.230.18.63` + +Keeping the runtime test in a consuming example is intentional, because it tests +the real purpose of this component: whether it can be reused by another +emulator. diff --git a/examples/internet/B02_mini_internet_with_dns/README.md b/examples/internet/B02_mini_internet_with_dns/README.md index a6e9737c0..0e7390d7c 100644 --- a/examples/internet/B02_mini_internet_with_dns/README.md +++ b/examples/internet/B02_mini_internet_with_dns/README.md @@ -42,10 +42,10 @@ from examples.internet.B00_mini_internet import mini_internet from examples.internet.B01_dns_component import dns_component mini_internet.run(dumpfile='./base_internet.bin') -dns_component.run(dumpfile='./dns-component.bin') +dns_component.run(dumpfile='./dns_component.bin') emuA.load('./base_internet.bin') -emuB.load('./dns-component.bin') +emuB.load('./dns_component.bin') emu = emuA.merge(emuB, DEFAULT_MERGERS) ``` @@ -148,3 +148,65 @@ customization will be carried to the physical nodes. This way, the customization can be done when a virtual node is created, instead of waiting for it to be bound to a physical node. Our DNS component is already modified to use the new design. + + +## Interacting with Nameservers Using `nsupdate` + +After the emulator starts running, we can use the `nsupdate` command to interact with the nameservers. All the nameservers inside the emulator are configured with the following setting inside the `/etc/bind/named.conf.options` file: +``` +allow-update { any; }; +``` + +This allows anyone to update the DNS records on this server. We have provided an example `add_record.sh` in the current folder to demonstrate how to add a record to the `example.net` server. Run the following: +``` +./add_record.sh 1.2.3.4 +``` + +After running it, go to any container, and run `"dig www.example.net"`, you will see that the IP address of `www.example.net` becomes `1.2.3.4`. Users can refer to the manual of `nsupdate` for more examples. + + +## Standardized TestRunner lifecycle + +This example includes an `example.yaml` manifest so it can be compiled, built, +started, probed, tested, and stopped by `seedemu.testing`. + +Run the full lifecycle from the repository root: + +```sh +python seedemu/testing/cli.py all examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +``` + +The lifecycle can also be run step by step: + +```sh +python seedemu/testing/cli.py clean examples/internet/B02_mini_internet_with_dns/example.yaml +python seedemu/testing/cli.py compile examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +python seedemu/testing/cli.py build examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +python seedemu/testing/cli.py up examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +python seedemu/testing/cli.py probe examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +python seedemu/testing/cli.py test examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +python seedemu/testing/cli.py down examples/internet/B02_mini_internet_with_dns/example.yaml --artifact-dir ci-artifacts/b02-mini-internet-with-dns +``` + +The tests focus only on DNS behavior introduced by this example. They do not +repeat B00's mini-Internet reachability tests. + +The declarative probes check that representative hosts can resolve records from +the DNS component: + +- `twitter.com` resolves to `1.1.1.1` +- `google.com` resolves to `2.2.2.2` +- `example.net` resolves to `3.3.3.3` +- `syr.edu` resolves to `128.230.18.63` + +The custom `test_runtime.py` program checks DNS-specific deployment details: + +- `local-dns-1` is created in AS152 with address `10.152.0.53`. +- `local-dns-2` is created in AS153 with address `10.153.0.53`. +- AS160 and AS170 hosts use `10.152.0.53` as their resolver. +- Representative hosts in other ASes use `10.153.0.53` as their resolver. +- The B01 DNS component records resolve through those local DNS cache servers. +- The `add_record.sh` dynamic-update helper exists. +- An `nsupdate` command adds `www.example.net A 5.6.7.8`, and a client can + resolve the dynamically added record through the local DNS cache. + diff --git a/examples/internet/B02_mini_internet_with_dns/add_record.sh b/examples/internet/B02_mini_internet_with_dns/add_record.sh new file mode 100755 index 000000000..345ebdcc7 --- /dev/null +++ b/examples/internet/B02_mini_internet_with_dns/add_record.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# This shell script shows how to use the nsupdate command +# to add records to a nameserver. The nameserver in our emulator +# is configured to allow remote update. + +if [ -z "$1" ]; then + echo "Error: Missing required argumetns." + echo "Usage: $0 " + exit 1 +fi + +# This is the nameserver of example.net +dns_server=10.163.0.71 + +new_ip=$1 + +update_command=$(cat </dev/null" + expect_exit: 0 + retries: 30 + interval: 3 + + - name: AS153 local DNS cache is ready + type: exec + service: hnode_153_local-dns-2 + command: "pgrep named >/dev/null" + expect_exit: 0 + retries: 30 + interval: 3 + +probes: + - name: AS160 resolves twitter.com through assigned local DNS + type: dns + service: hnode_160_host_0 + query: twitter.com + record_type: A + expect_answer: 1.1.1.1 + retries: 30 + interval: 5 + + - name: AS170 resolves google.com through assigned local DNS + type: dns + service: hnode_170_host_0 + query: google.com + record_type: A + expect_answer: 2.2.2.2 + retries: 30 + interval: 5 + + - name: AS150 resolves example.net through default local DNS + type: dns + service: hnode_150_host_0 + query: example.net + record_type: A + expect_answer: 3.3.3.3 + retries: 30 + interval: 5 + + - name: AS164 resolves syr.edu through default local DNS + type: dns + service: hnode_164_host_0 + query: syr.edu + record_type: A + expect_answer: 128.230.18.63 + retries: 30 + interval: 5 + +test_programs: + - name: B02 DNS runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py b/examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py index bdcfc4de8..f8797ec1d 100755 --- a/examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py +++ b/examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py @@ -1,6 +1,18 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu.core import Emulator, Binding, Filter, Action from seedemu.mergers import DEFAULT_MERGERS from seedemu.compiler import Docker, Platform @@ -9,38 +21,39 @@ from seedemu.layers import Base from examples.internet.B00_mini_internet import mini_internet from examples.internet.B01_dns_component import dns_component -import os, sys - -def run(dumpfile=None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B02 mini Internet with DNS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: emuA = Emulator() emuB = Emulator() # Run the pre-built components - mini_internet.run(dumpfile='./base_internet.bin') - dns_component.run(dumpfile='./dns_component.bin') + base_component = SCRIPT_DIR / "base_internet.bin" + dns_component_file = SCRIPT_DIR / "dns_component.bin" + mini_internet.run(dumpfile=str(base_component)) + dns_component.run(dumpfile=str(dns_component_file)) # Load and merge the pre-built components - emuA.load('./base_internet.bin') - emuB.load('./dns_component.bin') + emuA.load(str(base_component)) + emuB.load(str(dns_component_file)) emu = emuA.merge(emuB, DEFAULT_MERGERS) @@ -89,15 +102,41 @@ def run(dumpfile=None): # Add the ldns layer emu.addLayer(ldns) - + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() if dumpfile is not None: - # Save it to a file, so it can be used by other emulators - emu.dump(dumpfile) - else: - # Rendering compilation - emu.render() - emu.compile(Docker(platform=platform), './output', override=True) + # Save it to a file, so it can be used by other emulators + emu.dump(dumpfile) + return -if __name__ == "__main__": - run() + # Rendering compilation + if render: + emu.render() + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B02_mini_internet_with_dns/test_runtime.py b/examples/internet/B02_mini_internet_with_dns/test_runtime.py new file mode 100644 index 000000000..e8d616c72 --- /dev/null +++ b/examples/internet/B02_mini_internet_with_dns/test_runtime.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + local_dns_1 = test.require_service(152, "local-dns-1") + local_dns_2 = test.require_service(153, "local-dns-2") + host150 = test.require_service(150, "host_0") + host160 = test.require_service(160, "host_0") + ns_example_net = test.require_service(163, "host_0", "example.net authoritative server is generated") + host164 = test.require_service(164, "host_0") + host170 = test.require_service(170, "host_0") + add_record = Path(__file__).resolve().parent / "add_record.sh" + + test.structural_check( + "add_record.sh helper exists for dynamic DNS updates", + add_record.is_file(), + "found {}".format(add_record), + ) + + if local_dns_1: + test.structural_check( + "AS152 local DNS cache has the expected fixed address", + local_dns_1.address == "10.152.0.53", + "observed {}".format(local_dns_1.address), + ) + test.exec_check("AS152 local DNS cache is running named", local_dns_1, "pgrep named >/dev/null") + + if local_dns_2: + test.structural_check( + "AS153 local DNS cache has the expected fixed address", + local_dns_2.address == "10.153.0.53", + "observed {}".format(local_dns_2.address), + ) + test.exec_check("AS153 local DNS cache is running named", local_dns_2, "pgrep named >/dev/null") + + if host160: + test.exec_check( + "AS160 host uses global-dns-1", + host160, + "grep -q 'nameserver[[:space:]]*10.152.0.53' /etc/resolv.conf", + ) + test.exec_check( + "AS160 resolves twitter.com from the B01 DNS component", + host160, + "getent hosts twitter.com | grep -q '1.1.1.1'", + ) + + if host170: + test.exec_check( + "AS170 host uses global-dns-1", + host170, + "grep -q 'nameserver[[:space:]]*10.152.0.53' /etc/resolv.conf", + ) + test.exec_check( + "AS170 resolves google.com from the B01 DNS component", + host170, + "getent hosts google.com | grep -q '2.2.2.2'", + ) + + if host150: + test.exec_check( + "AS150 host uses global-dns-2 by default", + host150, + "grep -q 'nameserver[[:space:]]*10.153.0.53' /etc/resolv.conf", + ) + test.exec_check( + "AS150 resolves example.net from the B01 DNS component", + host150, + "getent hosts example.net | grep -q '3.3.3.3'", + ) + + if host164: + test.exec_check( + "AS164 host uses global-dns-2 by default", + host164, + "grep -q 'nameserver[[:space:]]*10.153.0.53' /etc/resolv.conf", + ) + test.exec_check( + "AS164 resolves syr.edu from the B01 DNS component", + host164, + "getent hosts syr.edu | grep -q '128.230.18.63'", + ) + + if ns_example_net and host150 and add_record.is_file(): + script_text = add_record.read_text(encoding="utf-8") + if not script_text.endswith("\n"): + script_text += "\n" + run_add_record = ( + "cat > /tmp/add_record.sh <<'__SEED_ADD_RECORD_SCRIPT__'\n" + "{}" + "__SEED_ADD_RECORD_SCRIPT__\n" + "chmod +x /tmp/add_record.sh\n" + "/tmp/add_record.sh 5.6.7.8" + ).format(script_text) + test.exec_check( + "add_record.sh dynamically adds www.example.net", + ns_example_net, + run_add_record, + ) + test.exec_check( + "AS150 resolves dynamically added www.example.net record", + host150, + "getent hosts www.example.net | grep -q '5.6.7.8'", + ) + + test.write_summary("b02-dns-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B03_hybrid_internet/README.md b/examples/internet/B03_hybrid_internet/README.md index 33416c9d4..3616f1331 100644 --- a/examples/internet/B03_hybrid_internet/README.md +++ b/examples/internet/B03_hybrid_internet/README.md @@ -1,63 +1,137 @@ # Hybrid Internet -Most part of this example is similar to the mini-internet example, -but several things are added to this example to enable -the emulator to communicate with the real world. With these -additions, nodes inside the emulator can communicate with -the machines in the real Internet, and the machines in the -real Internet can VPN into the emulator to become a node -in the emulation. +This example extends `B00_mini_internet` with real-world connectivity features. +The base mini Internet is built by `examples/internet/B00_mini_internet`; B03 +then adds only the hybrid-specific pieces. +This keeps B00 as the single source of truth for the mini-Internet topology. +When the base topology changes, B03 inherits those changes instead of carrying +a second copy of the same IX, AS, host, and peering code. -## Real-World Autonomous System 11872 +## What B03 Adds -The example creates a real-world AS (`AS-11872`), which is -Syracuse University's autonomous system number. It will collect -the network prefixes announced by this autonomous system in the real -world, and announce them inside the emulator. Packets (from inside -the emulator) going to these networks will be routed to this AS, and -then be forwarded to the real world. Returning packets -will come back from the outside, enter the emulator at -this AS, and be routed to its final destination inside -the emulator. +B03 starts with: +```python +emu = mini_internet.build_emulator(hosts_per_as=hosts_per_as) +base = emu.getLayer("Base") +ebgp = emu.getLayer("Ebgp") ``` + +Then it adds three hybrid features. + +## Real-World AS11872 + +The example creates a real-world AS for Syracuse University, `AS11872`, and +connects it to `IX102`: + +```python as11872 = base.createAutonomousSystem(11872) -as11872.createRealWorldRouter('rw').joinNetwork('ix102', '10.102.0.118') -``` +as11872.createRealWorldRouter( + "rw-11872-syr", + prefixes=["128.230.0.0/16"], +).joinNetwork("ix102", "10.102.0.118") -## Real-World Autonomous System 99999 +ebgp.addPrivatePeerings(102, [11], [11872], PeerRelationship.Provider) +``` -The example creates a real-world AS (`AS-99999`), which is -a hybrid autonomous system for the emulator. The prefixes of the AS -are configured as [`0.0.0.0/1`, `128.0.0.0/1`] and announce -them inside the emulator. These two IP prefixes cover the -entire IPv4 address space. Therefore, if the destination of a -packet does not match any network inside the emulator, -the packet will be routed to this AS, and -then be forwarded to the real world (via NAT). Returning packets -will come back from the outside, enter the emulator at -this AS, and be routed to its final destination inside -the emulator. +By default, the example uses a deterministic prefix so CI and agent-driven tests +do not depend on live Internet prefix lookup. To fetch live prefixes for +`AS11872`, use: +```sh +python examples/internet/B03_hybrid_internet/hybrid_internet.py --live-prefixes ``` + +## Default Real-World Gateway + +The example also creates `AS99999`, a hybrid AS that routes traffic toward the +real Internet. It announces two split default prefixes: + +```python as99999 = base.createAutonomousSystem(99999) -as99999.createRealWorldRouter('rw-real-world', - prefixes=['0.0.0.0/1', '128.0.0.0/1']).joinNetwork('ix100', '10.100.0.99') +as99999.createRealWorldRouter( + "rw-real-world", + prefixes=["0.0.0.0/1", "128.0.0.0/1"], +).joinNetwork("ix100", "10.100.0.99") + +ebgp.addPrivatePeerings(100, [3], [99999], PeerRelationship.Provider) +``` + +The two prefixes cover the IPv4 address space without using `0.0.0.0/0` +directly. Packets that do not match an emulated prefix can be sent to this AS +and then forwarded to the real world through NAT. + +## OpenVPN Remote Access + +`AS152` is configured to allow a real-world machine to VPN into its local +network: + +```python +ovpn = OpenVpnRemoteAccessProvider() +base.getAutonomousSystem(152).getNetwork("net0").enableRemoteAccess(ovpn) ``` +This allows an outside host to become a participant in the emulated network. +See [the OpenVPN remote access documentation](../../../misc/openvpn-remote-access/README.md) +for client-side connection details. -## Allow Real-World Access +## Standard Arguments -The `AS-152` is configured to allow real-world access. This means -a machine from outside of the emulator can VPN into `AS-152`'s network, -and essentially becomes a node of the emulator. This allows outside -real-world machines to participate in the emulation. -Please refer to [this document](../../../misc/openvpn-remote-access/README.md) -for instructions on how to VPN into this host. +From the repository root: +```sh +python examples/internet/B03_hybrid_internet/hybrid_internet.py --platform amd --output examples/internet/B03_hybrid_internet/output ``` -as152 = base.getAutonomousSystem(152) -as152.getNetwork('net0').enableRemoteAccess(ovpn) + +The legacy platform argument is also accepted: + +```sh +python examples/internet/B03_hybrid_internet/hybrid_internet.py amd +``` + +Useful options: + +```text +--platform amd|arm +--output PATH +--dumpfile PATH +--hosts-per-as N +--live-prefixes +--skip-render +--no-override ``` +## Test Runner + +Use the standardized runner from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/internet/B03_hybrid_internet/example.yaml +python seedemu/testing/cli.py compile examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +python seedemu/testing/cli.py build examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +python seedemu/testing/cli.py up examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +python seedemu/testing/cli.py probe examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +python seedemu/testing/cli.py test examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +python seedemu/testing/cli.py down examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B03_hybrid_internet/example.yaml --artifact-dir ci-artifacts/b03 +``` + +## Runtime Checks + +The declarative probes verify that normal B00 mini-Internet reachability still +works after the hybrid features are added. + +The custom `test_runtime.py` validates the B03-specific additions: + +- `AS11872` real-world router is generated; +- `AS11872` announces the deterministic example prefix; +- `AS99999` default real-world gateway is generated; +- `AS99999` announces the split default prefixes; +- both real-world routers have NAT setup scripts; +- an OpenVPN remote access bridge is generated for `AS152`. diff --git a/examples/internet/B03_hybrid_internet/example.yaml b/examples/internet/B03_hybrid_internet/example.yaml new file mode 100644 index 000000000..781f4d40c --- /dev/null +++ b/examples/internet/B03_hybrid_internet/example.yaml @@ -0,0 +1,67 @@ +id: internet-b03-hybrid-internet +name: Hybrid Internet +description: A B00-based mini Internet extended with real-world routers and OpenVPN remote access. +runner: internet +script: hybrid_internet.py +platform: amd +features: + - ipv4-default + - mini-internet + - real-world-router + - default-real-world-gateway + - openvpn-remote-access + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B03 representative services are running + type: compose-ps + services: + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_154_host_new + - hnode_170_host_0 + - brdnode_11872_rw-11872-syr + - brdnode_99999_rw-real-world + retries: 40 + interval: 3 + +probes: + - name: AS150 reaches AS152 through the B00 base Internet + type: ping + service: hnode_150_host_0 + target: 10.152.0.71 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS170 reaches AS154 through the B00 base Internet + type: ping + service: hnode_170_host_0 + target: 10.154.0.129 + count: 3 + retries: 40 + interval: 5 + timeout: 45 + +test_programs: + - name: B03 hybrid Internet structural validation + script: test_runtime.py + timeout: 300 diff --git a/examples/internet/B03_hybrid_internet/hybrid_internet.py b/examples/internet/B03_hybrid_internet/hybrid_internet.py index a9fe562a6..03f2ba782 100755 --- a/examples/internet/B03_hybrid_internet/hybrid_internet.py +++ b/examples/internet/B03_hybrid_internet/hybrid_internet.py @@ -1,166 +1,126 @@ #!/usr/bin/env python3 # encoding: utf-8 - -from seedemu import * -import os, sys - -def run(dumpfile = None, hosts_per_as=2): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - - - ############################################################################### - emu = Emulator() - base = Base() - routing = Routing() - ebgp = Ebgp() - ibgp = Ibgp() - ospf = Ospf() - web = WebService() - ovpn = OpenVpnRemoteAccessProvider() - - - ############################################################################### - # Create Internet Exchanges - ix100 = base.createInternetExchange(100) - ix101 = base.createInternetExchange(101) - ix102 = base.createInternetExchange(102) - ix103 = base.createInternetExchange(103) - ix104 = base.createInternetExchange(104) - ix105 = base.createInternetExchange(105) - - # Customize names (for visualization purpose) - ix100.getPeeringLan().setDisplayName('NYC-100') - ix101.getPeeringLan().setDisplayName('San Jose-101') - ix102.getPeeringLan().setDisplayName('Chicago-102') - ix103.getPeeringLan().setDisplayName('Miami-103') - ix104.getPeeringLan().setDisplayName('Boston-104') - ix105.getPeeringLan().setDisplayName('Huston-105') - - - ############################################################################### - # Create Transit Autonomous Systems - - ## Tier 1 ASes - Makers.makeTransitAs(base, 2, [100, 101, 102, 105], - [(100, 101), (101, 102), (100, 105)] - ) - - Makers.makeTransitAs(base, 3, [100, 103, 104, 105], - [(100, 103), (100, 105), (103, 105), (103, 104)] - ) - - Makers.makeTransitAs(base, 4, [100, 102, 104], - [(100, 104), (102, 104)] +# +# Purpose: build the B03 hybrid Internet example. Inputs are standard +# TestRunner CLI arguments. Outputs are Docker compiler files under --output. + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from examples.internet.B00_mini_internet import mini_internet +from seedemu.compiler import Docker, Platform +from seedemu.core import Emulator +from seedemu.layers import Base, Ebgp, PeerRelationship +from seedemu.raps import OpenVpnRemoteAccessProvider + + +SYRACUSE_ASN = 11872 +SYRACUSE_EXAMPLE_PREFIXES = ["128.230.0.0/16"] +HYBRID_ASN = 99999 +HYBRID_DEFAULT_PREFIXES = ["0.0.0.0/1", "128.0.0.0/1"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B03 hybrid Internet example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument( + "--live-prefixes", + action="store_true", + help="Fetch live prefixes for AS11872 instead of using deterministic example prefixes.", ) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args - ## Tier 2 ASes - Makers.makeTransitAs(base, 11, [102, 105], [(102, 105)]) - Makers.makeTransitAs(base, 12, [101, 104], [(101, 104)]) - - - ############################################################################### - # Create single-homed stub ASes. "None" means create a host only - - Makers.makeStubAsWithHosts(emu, base, 150, 100, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 151, 100, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 152, 101, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 153, 101, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 154, 102, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 160, 103, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 161, 103, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 162, 103, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 163, 104, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 164, 104, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 170, 105, hosts_per_as) - Makers.makeStubAsWithHosts(emu, base, 171, 105, hosts_per_as) - # Allow outside computers to VPN into AS-152's network - as152 = base.getAutonomousSystem(152) - as152.getNetwork('net0').enableRemoteAccess(ovpn) +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 - ############################################################################### - # Create real-world AS. - # AS11872 is the Syracuse University's autonomous system - as11872 = base.createAutonomousSystem(11872) - as11872.createRealWorldRouter('rw-11872-syr').joinNetwork('ix102', '10.102.0.118') +def add_remote_access(base: Base) -> None: + ovpn = OpenVpnRemoteAccessProvider() + base.getAutonomousSystem(152).getNetwork("net0").enableRemoteAccess(ovpn) - ############################################################################### - # Create hybrid AS. - # AS99999 is the emulator's autonomous system that routes the traffics - # to the real-world internet - as99999 = base.createAutonomousSystem(99999) - as99999.createRealWorldRouter('rw-real-world', - prefixes=['0.0.0.0/1', '128.0.0.0/1']).joinNetwork('ix100', '10.100.0.99') +def add_real_world_as(base: Base, ebgp: Ebgp, use_live_prefixes: bool = False) -> None: + prefixes = None if use_live_prefixes else SYRACUSE_EXAMPLE_PREFIXES + as11872 = base.createAutonomousSystem(SYRACUSE_ASN) + as11872.createRealWorldRouter("rw-11872-syr", prefixes=prefixes).joinNetwork("ix102", "10.102.0.118") + ebgp.addPrivatePeerings(102, [11], [SYRACUSE_ASN], PeerRelationship.Provider) - ############################################################################### - # Peering via RS (route server). The default peering mode for RS is - # PeerRelationship.Peer, which means each AS will only export its customers - # and their own prefixes. - # We will use this peering relationship to peer all the ASes in an IX. - # None of them will provide transit service for others. +def add_default_real_world_gateway(base: Base, ebgp: Ebgp) -> None: + as99999 = base.createAutonomousSystem(HYBRID_ASN) + as99999.createRealWorldRouter( + "rw-real-world", + prefixes=HYBRID_DEFAULT_PREFIXES, + ).joinNetwork("ix100", "10.100.0.99") + ebgp.addPrivatePeerings(100, [3], [HYBRID_ASN], PeerRelationship.Provider) - ebgp.addRsPeers(100, [2, 3, 4]) - ebgp.addRsPeers(102, [2, 4]) - ebgp.addRsPeers(104, [3, 4]) - ebgp.addRsPeers(105, [2, 3]) - # To buy transit services from another autonomous system, - # we will use private peering +def build_emulator(hosts_per_as: int = 2, use_live_prefixes: bool = False) -> Emulator: + emu = mini_internet.build_emulator(hosts_per_as=hosts_per_as) - ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) - ebgp.addPrivatePeerings(100, [3], [150, 99999], PeerRelationship.Provider) + base: Base = emu.getLayer("Base") + ebgp: Ebgp = emu.getLayer("Ebgp") - ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) - ebgp.addPrivatePeerings(101, [12], [152, 153], PeerRelationship.Provider) + add_remote_access(base) + add_real_world_as(base, ebgp, use_live_prefixes=use_live_prefixes) + add_default_real_world_gateway(base, ebgp) + return emu - ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) - ebgp.addPrivatePeerings(102, [11], [154, 11872], PeerRelationship.Provider) - ebgp.addPrivatePeerings(103, [3], [160, 161, 162 ], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(104, [3, 4], [12], PeerRelationship.Provider) - ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) - ebgp.addPrivatePeerings(104, [12], [164], PeerRelationship.Provider) - - ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) - ebgp.addPrivatePeerings(105, [11], [171], PeerRelationship.Provider) +def run( + dumpfile=None, + hosts_per_as: int = 2, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, + use_live_prefixes: bool = False, +): + emu = build_emulator(hosts_per_as=hosts_per_as, use_live_prefixes=use_live_prefixes) + if dumpfile is not None: + emu.dump(dumpfile) + return + if render: + emu.render() - ############################################################################### - # Add layers to the emulator - emu.addLayer(base) - emu.addLayer(routing) - emu.addLayer(ebgp) - emu.addLayer(ibgp) - emu.addLayer(ospf) - emu.addLayer(web) + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + use_live_prefixes=args.live_prefixes, + ) + return 0 - if dumpfile is not None: - # Save it to a component file, so it can be used by other emulators - emu.dump(dumpfile) - else: - emu.render() - emu.compile(Docker(platform=platform), './output', override=True) if __name__ == "__main__": - run() + raise SystemExit(main()) diff --git a/examples/internet/B03_hybrid_internet/test_runtime.py b/examples/internet/B03_hybrid_internet/test_runtime.py new file mode 100644 index 000000000..8d876202d --- /dev/null +++ b/examples/internet/B03_hybrid_internet/test_runtime.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# +# Purpose: validate the running B03 hybrid Internet example after TestRunner +# starts Docker Compose. Inputs come from TestRunner environment variables and +# generated docker-compose.yml labels. Outputs are JSON runtime summaries. + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def dockerfiles_containing(test: ComposeRuntimeTest, *patterns: str) -> list[str]: + matches = [] + for dockerfile in test.compose_file.parent.glob("*/Dockerfile"): + text = dockerfile.read_text(encoding="utf-8", errors="replace") + if all(pattern in text for pattern in patterns): + matches.append(dockerfile.parent.name) + return sorted(matches) + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + host150 = test.require_service(150, "host_0") + host152 = test.require_service(152, "host_0") + host170 = test.require_service(170, "host_0") + host154_new = test.require_service(154, "host_new") + syracuse = test.require_service(11872, "rw-11872-syr", "AS11872 real-world router is generated") + hybrid = test.require_service(99999, "rw-real-world", "AS99999 default real-world gateway is generated") + + if host150 and host152: + test.exec_check( + "AS150 reaches AS152 through the reused B00 base", + host150, + "ping -c 3 {} >/dev/null".format(host152.address), + retries=30, + interval=3, + ) + + if host170 and host154_new: + test.exec_check( + "AS170 reaches AS154 customized B00 host", + host170, + "ping -c 3 {} >/dev/null".format(host154_new.address), + retries=30, + interval=3, + ) + + if syracuse: + test.exec_check( + "AS11872 announces the deterministic example prefix", + syracuse, + "grep -q '128.230.0.0/16' /etc/bird/bird.conf", + ) + test.exec_check( + "AS11872 has real-world router NAT setup", + syracuse, + "test -s /rw_configure_script && grep -q 'MASQUERADE' /rw_configure_script", + ) + + if hybrid: + test.exec_check( + "AS99999 announces split default prefixes", + hybrid, + "grep -q '0.0.0.0/1' /etc/bird/bird.conf && grep -q '128.0.0.0/1' /etc/bird/bird.conf", + ) + test.exec_check( + "AS99999 has real-world router NAT setup", + hybrid, + "test -s /rw_configure_script && grep -q 'MASQUERADE' /rw_configure_script", + ) + + openvpn_outputs = dockerfiles_containing(test, "/ovpn-server.conf", "/ovpn_startup") + test.structural_check( + "OpenVPN remote access bridge is generated for AS152", + len(openvpn_outputs) >= 1, + ", ".join(openvpn_outputs), + ) + + test.write_summary("b03-hybrid-internet-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B06_internet_map/README.md b/examples/internet/B06_internet_map/README.md index 2d33ea1a0..80606e142 100644 --- a/examples/internet/B06_internet_map/README.md +++ b/examples/internet/B06_internet_map/README.md @@ -47,7 +47,7 @@ docker.attachInternetMap( **Please start the emulator and map container in `examples/internet/B06_internet_map` first** -1. visit [http://localhost:8080/plugin.html](http://localhost:8080/plugin.html), click "install" on the "submit_event" line to install "submit_event". +1. visit [http://localhost:8080/pro/plugin](http://localhost:8080/pro/plugin), click "install" on the "submit_event" line to install "submit_event". 2. enter any host container, (e.g., `docker exec -it as150h-host_1-10.150.0.72 bash`) 3. execute the submit_event.sh script in the container - `bash /map-plugins/submit_event.sh -a flash`, the container will flash. diff --git a/examples/internet/B20_dhcp/README.md b/examples/internet/B20_dhcp/README.md index ee8134792..aeb2b8f0c 100644 --- a/examples/internet/B20_dhcp/README.md +++ b/examples/internet/B20_dhcp/README.md @@ -1,68 +1,137 @@ # DHCP -In this example, we show how to deploy a dhcp server inside the -SEED Emulator and how to set a host's ip with the installed dhcp server. -We first create dhcp servers on `AS151` and `AS152` controller. -We then 2 hosts in each AS to get ip address from the installed dhcp server. +This example demonstrates how to deploy DHCP servers inside the SEED Emulator +and how to configure hosts to obtain their IP addresses through DHCP. -See the comments in the code for detailed explanation. +The example starts from the `B00_mini_internet` topology, then adds DHCP service +to two stub ASes: -We can utilize DHCP on `C03-bring-your-own-internet`. When we connect external -devices such as computer, smartphone, and IoT device to a internet emulator, -the installed dhcp will assign ip address to the newly attached devices so that -they can communicate with the nodes inside the emulator and use internet service. +- AS151 has DHCP server `dhcp-server-01`. +- AS161 has DHCP server `dhcp-server-02`. +- AS151 has two DHCP clients: `dhcp-client-01` and `dhcp-client-02`. +- AS161 has two DHCP clients: `dhcp-client-03` and `dhcp-client-04`. -## Step 1) Deploy a dhcp +The tests for this example focus only on DHCP behavior. They do not repeat B00's +mini-Internet reachability tests. + +## Step 1: Deploy DHCP Servers + +Create the DHCP service and install two DHCP server virtual nodes: ```python -# Create a DHCP server (virtual node). dhcp = DHCPService() -# Default DhcpIpRange : x.x.x.101 ~ x.x.x.120 -# Set DhcpIpRange : x.x.x.125 ~ x.x.x.140 -dhcp.install('dhcp-01').setIpRange(125, 140) -dhcp.install('dhcp-02') +# Default DHCP range: x.x.x.101 - x.x.x.120. +# Custom AS151 DHCP range: x.x.x.125 - x.x.x.140. +dhcp.install("dhcp-01").setIpRange(125, 140) +dhcp.install("dhcp-02") +``` +Customize their display names for visualization: -# Customize the display name (for visualization purpose) -emu.getVirtualNode('dhcp-01').setDisplayName('DHCP Server 1') -emu.getVirtualNode('dhcp-02').setDisplayName('DHCP Server 2') +```python +emu.getVirtualNode("dhcp-01").setDisplayName("DHCP Server 1") +emu.getVirtualNode("dhcp-02").setDisplayName("DHCP Server 2") +``` +Create physical hosts to run the DHCP servers: -# Create new host in AS-151 and AS-161, use them to host the DHCP servers. -# We can also host it on an existing node. +```python as151 = base.getAutonomousSystem(151) -as151.createHost('dhcp-server-01').joinNetwork('net0') +as151.createHost("dhcp-server-01").joinNetwork("net0") as161 = base.getAutonomousSystem(161) -as161.createHost('dhcp-server-02').joinNetwork('net0') - -# Bind the DHCP virtual node to the physical node. -emu.addBinding(Binding('dhcp-01', filter = Filter(asn=151, nodeName='dhcp-server-01'))) -emu.addBinding(Binding('dhcp-02', filter = Filter(asn=161, nodeName='dhcp-server-02'))) +as161.createHost("dhcp-server-02").joinNetwork("net0") ``` -Use method `DHCPServer:setIpRange` to set the ip range to assign. -The default IP range of Emulator is as below. -- host ip range : 71-99 -- dhcp ip range : 101-120 -- router ip range : 254-200 +Bind the DHCP virtual nodes to those physical hosts: -`DHCPServer:setIpRange` can change dhcp ip range. To change entire ip range, -we can use `Network:setHostIpRange()`, `Network:setDhcpIpRange()`, and -`Network:setRouterIpRange()`. +```python +emu.addBinding(Binding("dhcp-01", filter=Filter(asn=151, nodeName="dhcp-server-01"))) +emu.addBinding(Binding("dhcp-02", filter=Filter(asn=161, nodeName="dhcp-server-02"))) +``` +## Step 2: Create DHCP Clients -### Step 2) Set host to use dhcp +To make a host use DHCP instead of a static address, join the network with +`address="dhcp"`: ```python -# Create new hosts in AS-151, use it to host the Host which use dhcp instead of static ip -as151.createHost('dhcp-client-01').joinNetwork('net0', address = "dhcp") -as151.createHost('dhcp-client-02').joinNetwork('net0', address = "dhcp") +as151.createHost("dhcp-client-01").joinNetwork("net0", address="dhcp") +as151.createHost("dhcp-client-02").joinNetwork("net0", address="dhcp") + +as161.createHost("dhcp-client-03").joinNetwork("net0", address="dhcp") +as161.createHost("dhcp-client-04").joinNetwork("net0", address="dhcp") +``` + +The SEED Emulator adds the DHCP client software and a startup helper script to +request a lease when the container starts. + +## DHCP Address Ranges + +The default address ranges are: + +- Host static address range: `.71` to `.99` +- DHCP address range: `.101` to `.120` +- Router address range: `.254` downward -# Create new hosts in AS-161, use it to host the Host which use dhcp instead of static ip -as161.createHost('dhcp-client-01').joinNetwork('net0', address = "dhcp") -as161.createHost('dhcp-client-01').joinNetwork('net0', address = "dhcp") +`DHCPServer.setIpRange()` changes the DHCP range for the network served by that +DHCP server. In this example: +- AS151 uses the custom DHCP range `10.151.0.125` to `10.151.0.140`. +- AS161 uses the default DHCP range `10.161.0.101` to `10.161.0.120`. + +To change the entire network allocation policy, use `Network.setHostIpRange()`, +`Network.setDhcpIpRange()`, and `Network.setRouterIpRange()`. + +## Standard Arguments + +```sh +python examples/internet/B20_dhcp/dhcp.py amd +python examples/internet/B20_dhcp/dhcp.py --platform amd --output examples/internet/B20_dhcp/output +python examples/internet/B20_dhcp/dhcp.py --dumpfile examples/internet/B20_dhcp/dhcp.bin ``` +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## Standardized TestRunner Lifecycle + +Run the full lifecycle from the repository root: + +```sh +python seedemu/testing/cli.py all examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +``` + +The lifecycle can also be run step by step: + +```sh +python seedemu/testing/cli.py clean examples/internet/B20_dhcp/example.yaml +python seedemu/testing/cli.py compile examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +python seedemu/testing/cli.py build examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +python seedemu/testing/cli.py up examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +python seedemu/testing/cli.py probe examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +python seedemu/testing/cli.py test examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +python seedemu/testing/cli.py down examples/internet/B20_dhcp/example.yaml --artifact-dir ci-artifacts/b20-dhcp +``` + +The declarative probes check representative DHCP clients: + +- An AS151 client receives an address from `10.151.0.125` to `10.151.0.140`. +- An AS161 client receives an address from `10.161.0.101` to `10.161.0.120`. +- DHCP clients can reach their local default routers. + +The custom `test_runtime.py` program checks DHCP-specific deployment details: + +- DHCP server containers are generated and `dhcpd` is running. +- AS151's DHCP server config contains the custom range. +- AS161's DHCP server config contains the default range. +- All four DHCP clients have the generated DHCP client helper. +- All four DHCP clients receive addresses from the expected ranges. +- All four DHCP clients install the expected default route. diff --git a/examples/internet/B20_dhcp/dhcp.py b/examples/internet/B20_dhcp/dhcp.py index a83d3c8ed..b5c8793fb 100755 --- a/examples/internet/B20_dhcp/dhcp.py +++ b/examples/internet/B20_dhcp/dhcp.py @@ -1,77 +1,115 @@ #!/usr/bin/env python3 # encoding: utf-8 -from seedemu import * +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base +from seedemu.services import DHCPService from examples.internet.B00_mini_internet import mini_internet -import os, sys -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B20 DHCP example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + base_component = SCRIPT_DIR / "base_internet.bin" + mini_internet.run(dumpfile=str(base_component)) + + emu = Emulator() + emu.load(str(base_component)) -mini_internet.run(dumpfile='./base_internet.bin') + base: Base = emu.getLayer("Base") -emu = Emulator() + # Create DHCP servers as virtual nodes. + dhcp = DHCPService() -# Load the pre-built component -emu.load('./base_internet.bin') + # Default DHCP range: x.x.x.101 - x.x.x.120. + # Custom AS151 DHCP range: x.x.x.125 - x.x.x.140. + dhcp.install("dhcp-01").setIpRange(125, 140) + dhcp.install("dhcp-02") -base:Base = emu.getLayer('Base') + emu.getVirtualNode("dhcp-01").setDisplayName("DHCP Server 1") + emu.getVirtualNode("dhcp-02").setDisplayName("DHCP Server 2") -# Create a DHCP server (virtual node). -dhcp = DHCPService() + # Create hosts in AS151 and AS161 to run the DHCP servers. + as151 = base.getAutonomousSystem(151) + as151.createHost("dhcp-server-01").joinNetwork("net0") -# Default DhcpIpRange : x.x.x.101 ~ x.x.x.120 -# Set DhcpIpRange : x.x.x.125 ~ x.x.x.140 -dhcp.install('dhcp-01').setIpRange(125, 140) -dhcp.install('dhcp-02') + as161 = base.getAutonomousSystem(161) + as161.createHost("dhcp-server-02").joinNetwork("net0") + emu.addBinding(Binding("dhcp-01", filter=Filter(asn=151, nodeName="dhcp-server-01"))) + emu.addBinding(Binding("dhcp-02", filter=Filter(asn=161, nodeName="dhcp-server-02"))) -# Customize the display name (for visualization purpose) -emu.getVirtualNode('dhcp-01').setDisplayName('DHCP Server 1') -emu.getVirtualNode('dhcp-02').setDisplayName('DHCP Server 2') + # Create DHCP clients. They use DHCP instead of static addresses. + as151.createHost("dhcp-client-01").joinNetwork("net0", address="dhcp") + as151.createHost("dhcp-client-02").joinNetwork("net0", address="dhcp") + as161.createHost("dhcp-client-03").joinNetwork("net0", address="dhcp") + as161.createHost("dhcp-client-04").joinNetwork("net0", address="dhcp") -# Create new hosts in AS-151 and AS-161, use them to host the DHCP servers. -# We can also host it on an existing node. -as151 = base.getAutonomousSystem(151) -as151.createHost('dhcp-server-01').joinNetwork('net0') + emu.addLayer(dhcp) + return emu -as161 = base.getAutonomousSystem(161) -as161.createHost('dhcp-server-02').joinNetwork('net0') -# Bind the DHCP virtual node to the physical node. -emu.addBinding(Binding('dhcp-01', filter = Filter(asn=151, nodeName='dhcp-server-01'))) -emu.addBinding(Binding('dhcp-02', filter = Filter(asn=161, nodeName='dhcp-server-02'))) +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + emu.dump(dumpfile) + return + if render: + emu.render() -# Create new hosts in AS-151 and AS-161 -# Make them to use dhcp instead of static ip -as151.createHost('dhcp-client-01').joinNetwork('net0', address = "dhcp") -as151.createHost('dhcp-client-02').joinNetwork('net0', address = "dhcp") + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) -as161.createHost('dhcp-client-03').joinNetwork('net0', address = "dhcp") -as161.createHost('dhcp-client-04').joinNetwork('net0', address = "dhcp") -# Add the dhcp layer -emu.addLayer(dhcp) +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 -# Render the emulation -emu.render() -# Compil the emulation -emu.compile(Docker(platform=platform), './output', override=True) \ No newline at end of file +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B20_dhcp/example.yaml b/examples/internet/B20_dhcp/example.yaml new file mode 100644 index 000000000..6928ecbfb --- /dev/null +++ b/examples/internet/B20_dhcp/example.yaml @@ -0,0 +1,94 @@ +id: internet-b20-dhcp +name: DHCP +description: A B00 mini-Internet deployment that adds DHCP servers and DHCP clients in AS151 and AS161. +runner: internet +script: dhcp.py +platform: amd +features: + - ipv4-default + - mini-internet + - dhcp + +compile: + enabled: true + output: output + clean: + - output + - base_internet.bin + expected: + - output/docker-compose.yml + - base_internet.bin + timeout: 900 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B20 DHCP services and clients are running + type: compose-ps + services: + - hnode_151_dhcp-server-01 + - hnode_161_dhcp-server-02 + - hnode_151_dhcp-client-01 + - hnode_151_dhcp-client-02 + - hnode_161_dhcp-client-03 + - hnode_161_dhcp-client-04 + retries: 40 + interval: 3 + + - name: AS151 DHCP server is ready + type: exec + service: hnode_151_dhcp-server-01 + command: "pgrep dhcpd >/dev/null" + expect_exit: 0 + retries: 30 + interval: 3 + + - name: AS161 DHCP server is ready + type: exec + service: hnode_161_dhcp-server-02 + command: "pgrep dhcpd >/dev/null" + expect_exit: 0 + retries: 30 + interval: 3 + +probes: + - name: AS151 DHCP client receives an address from custom range + type: exec + service: hnode_151_dhcp-client-01 + command: "ip -4 addr show net0 | grep -Eq '10\\.151\\.0\\.(12[5-9]|13[0-9]|140)/'" + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS161 DHCP client receives an address from default range + type: exec + service: hnode_161_dhcp-client-03 + command: "ip -4 addr show net0 | grep -Eq '10\\.161\\.0\\.(10[1-9]|11[0-9]|120)/'" + expect_exit: 0 + retries: 30 + interval: 5 + + - name: AS151 DHCP client can reach its default router + type: ping + service: hnode_151_dhcp-client-01 + target: 10.151.0.254 + count: 3 + retries: 20 + interval: 5 + + - name: AS161 DHCP client can reach its default router + type: ping + service: hnode_161_dhcp-client-03 + target: 10.161.0.254 + count: 3 + retries: 20 + interval: 5 + +test_programs: + - name: B20 DHCP runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/internet/B20_dhcp/test_runtime.py b/examples/internet/B20_dhcp/test_runtime.py new file mode 100644 index 000000000..3908a13ff --- /dev/null +++ b/examples/internet/B20_dhcp/test_runtime.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +AS151_RANGE = r"10\.151\.0\.(12[5-9]|13[0-9]|140)" +AS161_RANGE = r"10\.161\.0\.(10[1-9]|11[0-9]|120)" + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + server151 = test.require_service(151, "dhcp-server-01") + server161 = test.require_service(161, "dhcp-server-02") + client151_1 = test.require_service(151, "dhcp-client-01") + client151_2 = test.require_service(151, "dhcp-client-02") + client161_1 = test.require_service(161, "dhcp-client-03") + client161_2 = test.require_service(161, "dhcp-client-04") + + if server151: + test.exec_check("AS151 DHCP server is running", server151, "pgrep dhcpd >/dev/null") + test.exec_check( + "AS151 DHCP server advertises custom range", + server151, + "grep -q 'range 10.151.0.125 10.151.0.140;' /etc/dhcp/dhcpd.conf", + ) + test.exec_check( + "AS151 DHCP server advertises the subnet router", + server151, + "grep -q 'option routers 10.151.0.254;' /etc/dhcp/dhcpd.conf", + ) + + if server161: + test.exec_check("AS161 DHCP server is running", server161, "pgrep dhcpd >/dev/null") + test.exec_check( + "AS161 DHCP server advertises default range", + server161, + "grep -q 'range 10.161.0.101 10.161.0.120;' /etc/dhcp/dhcpd.conf", + ) + test.exec_check( + "AS161 DHCP server advertises the subnet router", + server161, + "grep -q 'option routers 10.161.0.254;' /etc/dhcp/dhcpd.conf", + ) + + for client in (client151_1, client151_2): + if client: + test.exec_check( + "{} has the DHCP client helper".format(client.name), + client, + "test -x dhclient.sh", + ) + test.exec_check( + "{} received an AS151 DHCP lease".format(client.name), + client, + "ip -4 addr show net0 | grep -Eq '{}/'".format(AS151_RANGE), + ) + test.exec_check( + "{} installed the AS151 DHCP default route".format(client.name), + client, + "ip route | grep -q '^default via 10.151.0.254'", + ) + + for client in (client161_1, client161_2): + if client: + test.exec_check( + "{} has the DHCP client helper".format(client.name), + client, + "test -x dhclient.sh", + ) + test.exec_check( + "{} received an AS161 DHCP lease".format(client.name), + client, + "ip -4 addr show net0 | grep -Eq '{}/'".format(AS161_RANGE), + ) + test.exec_check( + "{} installed the AS161 DHCP default route".format(client.name), + client, + "ip route | grep -q '^default via 10.161.0.254'", + ) + + test.write_summary("b20-dhcp-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B21_etc_hosts/README.md b/examples/internet/B21_etc_hosts/README.md index 13bdb006f..14fc1cd96 100644 --- a/examples/internet/B21_etc_hosts/README.md +++ b/examples/internet/B21_etc_hosts/README.md @@ -1,39 +1,89 @@ -# Enable local DNS service using `EtcHosts` layer +# EtcHosts -This example demonstrates how we can enable a local DNS service using the `EtcHosts` layer. -This layer will add the IP address and hostnames of all the nodes in the emulator to the `/etc/hosts` file. -The `/etc/hosts` file is a simple text file that associates IP addresses with hostnames. -It is used to resolve hostnames to IP addresses. +This example demonstrates how to enable local name resolution using the +`EtcHosts` layer. The layer writes emulator node names and custom hostnames into +each container's `/etc/hosts` file, so hosts can resolve names without deploying +DNS. -The default hostname of a node is of the form `-`. For example, the default hostname of a node named `node1` in the autonomous system `154` is `154-node1`. However, we can add additional hostnames to a node using the `custom_host_names` parameter of the `Node` constructor or the `addHostName` method of the `Node` class. The `EtcHosts` layer will add these custom hostnames to the `/etc/hosts` file. +The example starts from the `B00_mini_internet` topology, then adds one new host +in AS152: -Following is an example of how to add a custom hostname to a node: +- Host name: `database` +- IP address: `10.152.0.4` +- Custom hostname: `database.com` -- Add a new custom hostname using the `addHostName` method: -``` -node.addHostName('custom_hostname3') -as152.createHost('database').joinNetwork('net0', address = '10.152.0.4').addHostName('database.com') -``` +The tests for this example focus only on `/etc/hosts` behavior. They do not +repeat B00's mini-Internet reachability tests. -In this example, we will create a new host with custom host name on top of the mini-internet component and then enable `EtcHosts` layer. +## Create A Host With A Custom Hostname -## Load the Mini-Internet Component +The default hostname of a node is based on its scope and node name. For example, +a node named `host_0` in AS154 has a generated hostname such as `154-host_0`. -``` -emu.load('../B00-mini-internet/base_component.bin') +We can add additional hostnames using `addHostName()`: + +```python +base: Base = emu.getLayer("Base") +as152 = base.getAutonomousSystem(152) +as152.createHost("database").joinNetwork("net0", address="10.152.0.4").addHostName("database.com") ``` -## Create a New Host with Custom Hostname +## Add The EtcHosts Layer +After the custom hostname is configured, add the `EtcHosts` layer: + +```python +emu.addLayer(EtcHosts()) ``` -base: Base = emu.getLayer('Base') -as152 = base.getAutonomousSystem(152) -as152.createHost('database').joinNetwork('net0', address = '10.152.0.4').addHostName('database.com') + +When the emulator is rendered, the layer creates `/etc/hosts` entries for the +emulator hosts. Every generated container should then be able to resolve +`database.com` to `10.152.0.4`. + +## Standard Arguments + +```sh +python examples/internet/B21_etc_hosts/etc_hosts.py amd +python examples/internet/B21_etc_hosts/etc_hosts.py --platform amd --output examples/internet/B21_etc_hosts/output +python examples/internet/B21_etc_hosts/etc_hosts.py --dumpfile examples/internet/B21_etc_hosts/etc_hosts.bin ``` -## Add EtcHosts Layer +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## Standardized TestRunner Lifecycle +Run the full lifecycle from the repository root: + +```sh +python seedemu/testing/cli.py all examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts ``` -etc_hosts = EtcHosts() -emu.addLayer(etc_hosts) + +The lifecycle can also be run step by step: + +```sh +python seedemu/testing/cli.py clean examples/internet/B21_etc_hosts/example.yaml +python seedemu/testing/cli.py compile examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts +python seedemu/testing/cli.py build examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts +python seedemu/testing/cli.py up examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts +python seedemu/testing/cli.py probe examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts +python seedemu/testing/cli.py test examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts +python seedemu/testing/cli.py down examples/internet/B21_etc_hosts/example.yaml --artifact-dir ci-artifacts/b21-etc-hosts ``` + +The declarative probes check that representative hosts can resolve +`database.com` to `10.152.0.4`. + +The custom `test_runtime.py` program checks `/etc/hosts`-specific behavior: + +- The `database` host is generated in AS152. +- The `database` host has address `10.152.0.4`. +- Representative containers contain a `database.com` entry in `/etc/hosts`. +- Representative containers resolve `database.com` to `10.152.0.4`. +- A representative host can reach the database host by custom hostname. diff --git a/examples/internet/B21_etc_hosts/etc_hosts.py b/examples/internet/B21_etc_hosts/etc_hosts.py index 73c02aeed..3657fc34b 100755 --- a/examples/internet/B21_etc_hosts/etc_hosts.py +++ b/examples/internet/B21_etc_hosts/etc_hosts.py @@ -1,45 +1,88 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from seedemu.compiler import Docker, Platform -from seedemu.layers import Base, EtcHosts from seedemu.core import Emulator +from seedemu.layers import Base, EtcHosts from examples.internet.B00_mini_internet import mini_internet -import os, sys - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -mini_internet.run('base_internet.bin') - -emu = Emulator() -emu.load('base_internet.bin') - -etc_hosts = EtcHosts() - -# Create a new host in AS-152 with custom host name -base: Base = emu.getLayer('Base') -as152 = base.getAutonomousSystem(152) -as152.createHost('database').joinNetwork('net0', address = '10.152.0.4').addHostName('database.com') - -# Add the etc_hosts layer -emu.addLayer(etc_hosts) - -# Render the emulation and further customization -emu.render() -emu.compile(Docker(platform=platform), './output', override=True) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B21 /etc/hosts example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + base_component = SCRIPT_DIR / "base_internet.bin" + mini_internet.run(dumpfile=str(base_component)) + + emu = Emulator() + emu.load(str(base_component)) + + base: Base = emu.getLayer("Base") + as152 = base.getAutonomousSystem(152) + as152.createHost("database").joinNetwork("net0", address="10.152.0.4").addHostName("database.com") + + emu.addLayer(EtcHosts()) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B21_etc_hosts/example.yaml b/examples/internet/B21_etc_hosts/example.yaml new file mode 100644 index 000000000..9c84f232c --- /dev/null +++ b/examples/internet/B21_etc_hosts/example.yaml @@ -0,0 +1,68 @@ +id: internet-b21-etc-hosts +name: EtcHosts +description: A B00 mini-Internet deployment that adds an AS152 database host and resolves it through the EtcHosts layer. +runner: internet +script: etc_hosts.py +platform: amd +features: + - ipv4-default + - mini-internet + - etc-hosts + +compile: + enabled: true + output: output + clean: + - output + - base_internet.bin + expected: + - output/docker-compose.yml + - base_internet.bin + timeout: 900 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B21 EtcHosts services are running + type: compose-ps + services: + - hnode_152_database + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_160_host_0 + retries: 40 + interval: 3 + +probes: + - name: AS150 resolves database.com through /etc/hosts + type: exec + service: hnode_150_host_0 + command: "getent hosts database.com | grep -q '10.152.0.4'" + expect_exit: 0 + retries: 20 + interval: 3 + + - name: AS160 resolves database.com through /etc/hosts + type: exec + service: hnode_160_host_0 + command: "getent hosts database.com | grep -q '10.152.0.4'" + expect_exit: 0 + retries: 20 + interval: 3 + + - name: Database host resolves its custom hostname locally + type: exec + service: hnode_152_database + command: "getent hosts database.com | grep -q '10.152.0.4'" + expect_exit: 0 + retries: 20 + interval: 3 + +test_programs: + - name: B21 EtcHosts runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/internet/B21_etc_hosts/test_runtime.py b/examples/internet/B21_etc_hosts/test_runtime.py new file mode 100644 index 000000000..c4014b7c6 --- /dev/null +++ b/examples/internet/B21_etc_hosts/test_runtime.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + database = test.require_service(152, "database") + host150 = test.require_service(150, "host_0") + host152 = test.require_service(152, "host_0") + host160 = test.require_service(160, "host_0") + + if database: + test.structural_check( + "AS152 database host has the expected fixed address", + database.address == "10.152.0.4", + "observed {}".format(database.address), + ) + + for service in (database, host150, host152, host160): + if service: + test.exec_check( + "{} has database.com in /etc/hosts".format(service.name), + service, + "grep -Eq '^10\\.152\\.0\\.4[[:space:]].*database\\.com([[:space:]]|$)' /etc/hosts", + ) + test.exec_check( + "{} resolves database.com through /etc/hosts".format(service.name), + service, + "getent hosts database.com | grep -q '10.152.0.4'", + ) + + if host150: + test.exec_check( + "AS150 can reach the database host by custom hostname", + host150, + "ping -c 3 database.com >/dev/null", + ) + + test.write_summary("b21-etc-hosts-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B22_botnet/README.md b/examples/internet/B22_botnet/README.md index 8d71992fe..6f4dca527 100644 --- a/examples/internet/B22_botnet/README.md +++ b/examples/internet/B22_botnet/README.md @@ -1,68 +1,110 @@ # Botnet -In this example, we show how to deploy a botnet inside the -SEED Emulator. We first create a bot controller and 6 bots. -We then deploy the controller on `10.150.0.66`, and -deploy the bots in randomly selected autonomous systems. -See the comments in the code for detailed explanation. +This example demonstrates how to deploy a botnet inside the SEED Emulator. It +uses the `B00_mini_internet` topology as the base Internet, then adds one BYOB +controller and six bot clients. -We have also recorded a -[video](https://www.youtube.com/watch?v=5FYWB-b21bg&list=PLwCoMLt7WGjan54CuqeYGnJuMqA-RzQwD) -to explain the details of this example. -We have modified the code (`botnet-base.py`) since the video was recorded, -so the code used in the video is slightly different from the code here. +The controller is deployed at `10.150.0.66`. The bot clients are placed across +several ASes selected from the B00 stub ASes. By default, placement is +reproducible because the script uses a fixed random seed; this keeps CI and +agent-driven testing stable while still demonstrating distributed bot placement. +## Files -## Start the Controller +- `botnet_basic.py`: topology entrypoint. Supports the standard example + arguments used by `seedemu/testing/cli.py`. +- `ddos.py`: helper script copied to the controller as `/tmp/ddos.py` for the + manual denial-of-service demonstration. +- `example.yaml`: test manifest for compile, build, runtime readiness, probes, + and the custom runtime test. +- `test_runtime.py`: custom runtime validation using `ComposeRuntimeTest`. +- `not_ready/botnet-with-dga.py`: unfinished DGA-based variant; it is not part + of this example's automated lifecycle. -We will find the Bot controller container, and get a shell on it. -We have customized the display names of the controller and all the bot nodes -with a prefix `Bot-`, so they are quite easy to find. -Once we are inside the controller, we can start the `byob` -server. +## Standard Arguments +From the repository root: + +```sh +python examples/internet/B22_botnet/botnet_basic.py --platform amd --output examples/internet/B22_botnet/output +``` + +The legacy platform argument is also accepted: + +```sh +python examples/internet/B22_botnet/botnet_basic.py amd ``` -# cd /tmp/byob/byob -# python3 server.py --port 445 + +Useful options: + +```text +--platform amd|arm +--output PATH +--dumpfile PATH +--hosts-per-as N +--bot-count N +--seed N +--skip-render +--no-override ``` -We will then wait for the bot nodes to connect to the server. We have deployed -6 bot nodes in the emulator, so we should see 6 clients. We can type the -`sessions` command to see their information. +## Test Runner +Use the standardized runner from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/internet/B22_botnet/example.yaml +python seedemu/testing/cli.py compile examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 +python seedemu/testing/cli.py build examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 +python seedemu/testing/cli.py up examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 +python seedemu/testing/cli.py probe examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 +python seedemu/testing/cli.py test examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 +python seedemu/testing/cli.py down examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 ``` -[byob @ /tmp/byob/byob]>sessions -0 - public_ip 10.151.0.73 - local_ip 10.151.0.73 - platform linux - mac_address 24:20:A9:70:04:9 - architecture 64 - username user - administrator True - device 721736c3959c - owner None - latitude 0 - longitude 0 - uid b4a453a1fd66f9108e23c96716455f3b - joined 2021-08-08 14:20:01.213183 - online True - sessions True - last_online 2021-08-08 14:20:01.213259 - -1 - ... + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B22_botnet/example.yaml --artifact-dir ci-artifacts/b22 ``` -## Launch Attacks +## Automated Testing Strategy + +The BYOB shell is interactive, so the CI test does not try to type commands +into it. Instead, the test verifies the non-interactive parts that must work +before a manual BYOB session can be useful: + +- the controller container exists and uses `10.150.0.66`; +- BYOB is installed on the controller; +- `/bin/start-byob-shell` exists for manual use; +- `/tmp/ddos.py` is copied to the controller; +- the controller exposes the generated BYOB dropper endpoint on port `446`; +- six bot client containers are generated; +- each bot has the BYOB client startup runner; +- each bot can reach the controller by ICMP and can fetch the dropper endpoint. + +The automated test intentionally does not launch the DDoS command. That behavior +is better suited for a manual classroom demonstration, where students can watch +the traffic in the emulator visualization. -You can find the online manuals regarding how to use `byob`. Here, we will -just do a simple testing. We broadcast a command to all the bots, asking them -to ping `10.161.0.71` 10 times, with each ICMP packet carrying a large payload. -From the visualization map, we should clear see the attack traffic. -This demonstrates a denial-of-service attack. +## Manual Demonstration +Find the bot controller container and enter a shell. The controller has display +name `Bot-Controller`, and the bots have display names such as `Bot-000`. + +Start the BYOB shell: + +```sh +start-byob-shell ``` -[byob @ /tmp/byob/byob]>broadcast ping -c 10 -s 5000 10.161.0.71 + +After the bots connect, use the BYOB `sessions` command to list them. For a +simple traffic demonstration, broadcast a ping command to all bots: + +```text +broadcast ping -c 10 -s 5000 10.161.0.71 ``` +The visualization should show traffic from multiple bot ASes toward the victim +host. This demonstrates the command-and-control structure and the effect of a +distributed denial-of-service attack inside the emulator. diff --git a/examples/internet/B22_botnet/botnet_basic.py b/examples/internet/B22_botnet/botnet_basic.py index d25a5ae90..f9f59f717 100755 --- a/examples/internet/B22_botnet/botnet_basic.py +++ b/examples/internet/B22_botnet/botnet_basic.py @@ -1,86 +1,138 @@ #!/usr/bin/env python3 # encoding: utf-8 +# +# Purpose: build the B22 botnet example. Inputs are standard TestRunner CLI +# arguments. Outputs are Docker compiler files under --output. -import random -from seedemu.core import Emulator, Binding, Filter, Action -from seedemu.services import BotnetService, BotnetClientService -from seedemu.compiler import Docker, Platform -from examples.internet.B00_mini_internet import mini_internet -import os, sys - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -mini_internet.run(dumpfile='./base_internet.bin') - -emu = Emulator() - -# Load the pre-built component -emu.load('./base_internet.bin') - -############################################################################### -# Build a botnet - -# Create two service layers -bot = BotnetService() -botClient = BotnetClientService() - -# Create a virtual node for bot controller, -# and customize its display name -bot.install('bot-controller') -emu.getVirtualNode('bot-controller').setDisplayName('Bot-Controller') - -# Install a file to this node -f = open("./ddos.py", "r") -emu.getVirtualNode('bot-controller').setFile(content=f.read(), path="/tmp/ddos.py") - -# Create 6 bot nodes -for counter in range(6): - vname = 'bot-node-%.3d'%(counter) - - # Create a virtual node for each bot client, - # tell them which node is the controller node, - # and customize its display name. - botClient.install(vname).setServer('bot-controller') - emu.getVirtualNode(vname).setDisplayName('Bot-%.3d'%(counter)) - - -############################################################################### -# Bind the virtual nodes to physical nodes - -# Bind the controller node -emu.addBinding(Binding('bot-controller', - filter = Filter(ip='10.150.0.66'), action=Action.NEW)) - -as_list = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164, 170, 171] -for counter in range(6): - vname = 'bot-node-%.3d'%(counter) - - # Pick an autonomous system randomly from the list, - # and create a new host for each bot. - asn = random.choice(as_list) - emu.addBinding(Binding(vname, filter=Filter(asn=asn), action=Action.NEW)) +from __future__ import annotations +import argparse +import random +from pathlib import Path +import sys -############################################################################### -emu.addLayer(bot) -emu.addLayer(botClient) +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) -emu.render() -emu.compile(Docker(platform=platform), './output', override=True) +from examples.internet.B00_mini_internet import mini_internet +from seedemu.compiler import Docker, Platform +from seedemu.core import Action, Binding, Emulator, Filter +from seedemu.services import BotnetClientService, BotnetService + + +BOT_CONTROLLER_IP = "10.150.0.66" +BOT_CANDIDATE_ASNS = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164, 170, 171] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B22 botnet example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--bot-count", type=int, default=6) + parser.add_argument("--seed", type=int, default=22) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def select_bot_asns(bot_count: int, seed: int) -> list[int]: + if bot_count > len(BOT_CANDIDATE_ASNS): + raise ValueError("bot_count cannot exceed {}".format(len(BOT_CANDIDATE_ASNS))) + rng = random.Random(seed) + return rng.sample(BOT_CANDIDATE_ASNS, bot_count) + + +def build_emulator(hosts_per_as: int = 2, bot_count: int = 6, seed: int = 22) -> Emulator: + base_component = SCRIPT_DIR / "base_internet.bin" + mini_internet.run(dumpfile=str(base_component), hosts_per_as=hosts_per_as) + + emu = Emulator() + emu.load(str(base_component)) + + bot = BotnetService() + bot_client = BotnetClientService() + + bot.install("bot-controller") + emu.getVirtualNode("bot-controller").setDisplayName("Bot-Controller") + ddos_script = (SCRIPT_DIR / "ddos.py").read_text(encoding="utf-8") + emu.getVirtualNode("bot-controller").setFile(content=ddos_script, path="/tmp/ddos.py") + + for counter, asn in enumerate(select_bot_asns(bot_count, seed)): + vnode = "bot-node-{:03d}".format(counter) + bot_client.install(vnode).setServer("bot-controller") + emu.getVirtualNode(vnode).setDisplayName("Bot-{:03d}".format(counter)) + emu.addBinding( + Binding( + vnode, + filter=Filter(asn=asn, nodeName=vnode), + action=Action.NEW, + ) + ) + + emu.addBinding( + Binding( + "bot-controller", + filter=Filter(ip=BOT_CONTROLLER_IP, nodeName="bot-controller"), + action=Action.NEW, + ) + ) + + emu.addLayer(bot) + emu.addLayer(bot_client) + return emu + + +def run( + dumpfile=None, + hosts_per_as: int = 2, + bot_count: int = 6, + seed: int = 22, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, +): + emu = build_emulator(hosts_per_as=hosts_per_as, bot_count=bot_count, seed=seed) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + emu.compile(Docker(platform=platform), output or str(SCRIPT_DIR / "output"), override=override) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + bot_count=args.bot_count, + seed=args.seed, + output=str(output_dir), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B22_botnet/example.yaml b/examples/internet/B22_botnet/example.yaml new file mode 100644 index 000000000..3056a3bfc --- /dev/null +++ b/examples/internet/B22_botnet/example.yaml @@ -0,0 +1,70 @@ +id: internet-b22-botnet +name: Botnet +description: A B00-based mini Internet that deploys one BYOB controller and six bot clients across multiple ASes. +runner: internet +script: botnet_basic.py +platform: amd +features: + - ipv4-default + - mini-internet + - botnet + - byob + - command-and-control + +compile: + enabled: true + output: output + clean: + - output + - base_internet.bin + expected: + - output/docker-compose.yml + - base_internet.bin + timeout: 600 + +build: + enabled: true + timeout: 2400 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B22 controller and representative network services are running + type: compose-ps + services: + - hnode_150_bot-controller + - brdnode_150_router0 + - hnode_161_host_0 + retries: 40 + interval: 3 + +probes: + - name: controller has the DDoS helper script + type: exec + service: hnode_150_bot-controller + command: test -f /tmp/ddos.py + expect_exit: 0 + retries: 20 + interval: 3 + + - name: controller has the BYOB shell helper + type: exec + service: hnode_150_bot-controller + command: test -x /bin/start-byob-shell + expect_exit: 0 + retries: 20 + interval: 3 + + - name: controller exposes the BYOB dropper endpoint + type: exec + service: hnode_150_bot-controller + command: curl -fsS http://127.0.0.1:446/clients/droppers/client.py >/dev/null + expect_exit: 0 + retries: 60 + interval: 5 + timeout: 45 + +test_programs: + - name: B22 custom runtime validation + script: test_runtime.py + timeout: 900 diff --git a/examples/internet/B22_botnet/test_runtime.py b/examples/internet/B22_botnet/test_runtime.py new file mode 100644 index 000000000..f12df5ddc --- /dev/null +++ b/examples/internet/B22_botnet/test_runtime.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# +# Purpose: validate the running B22 botnet example without scripting the +# interactive BYOB shell. Inputs come from TestRunner environment variables and +# generated docker-compose.yml labels. Outputs are JSON runtime summaries. + +from __future__ import annotations + +from typing import List + +from seedemu.testing import ComposeRuntimeTest, ComposeService +from seedemu.testing.runtime import ADDRESS_LABEL, NODE_LABEL + + +BOT_CONTROLLER_IP = "10.150.0.66" +EXPECTED_BOT_COUNT = 6 + + +def discover_bots(test: ComposeRuntimeTest) -> List[ComposeService]: + bots: List[ComposeService] = [] + for name, service in test.compose.get("services", {}).items(): + labels = dict(service.get("labels", {})) + node_name = str(labels.get(NODE_LABEL, "")) + if node_name.startswith("bot-node-"): + address = str(labels.get(ADDRESS_LABEL, "")).split("/", 1)[0] + bots.append(ComposeService(name=str(name), address=address, labels=labels)) + + bots.sort(key=lambda service: str(service.labels.get(NODE_LABEL, service.name))) + return bots + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + controller = test.require_service(150, "bot-controller") + bots = discover_bots(test) + + test.structural_check( + "B22 creates six bot clients", + len(bots) == EXPECTED_BOT_COUNT, + "found {} bot clients".format(len(bots)), + ) + + if controller: + test.structural_check( + "bot controller uses the expected address", + controller.address == BOT_CONTROLLER_IP, + "controller address={}".format(controller.address), + ) + test.exec_check( + "controller has BYOB installed", + controller, + "test -d /tmp/byob/byob && test -f /tmp/byob/byob/server.py", + retries=20, + interval=3, + ) + test.exec_check( + "controller has the manual BYOB shell helper", + controller, + "test -x /bin/start-byob-shell", + retries=20, + interval=3, + ) + test.exec_check( + "controller has the DDoS helper script", + controller, + "test -f /tmp/ddos.py", + retries=20, + interval=3, + ) + test.exec_check( + "controller exposes BYOB dropper endpoint", + controller, + "curl -fsS http://127.0.0.1:446/clients/droppers/client.py >/dev/null", + retries=60, + interval=5, + timeout=45, + ) + + for bot in bots: + label = bot.labels.get(NODE_LABEL, bot.name) + test.exec_check( + "{} has BYOB client startup runner".format(label), + bot, + "test -x /tmp/byob_client_dropper_runner", + retries=20, + interval=3, + ) + test.exec_check( + "{} reaches controller ICMP".format(label), + bot, + "ping -c 3 {} >/dev/null".format(BOT_CONTROLLER_IP), + retries=30, + interval=3, + timeout=45, + ) + test.exec_check( + "{} reaches controller dropper HTTP endpoint".format(label), + bot, + "curl -fsS http://{}:446/clients/droppers/client.py >/dev/null".format(BOT_CONTROLLER_IP), + retries=60, + interval=5, + timeout=45, + ) + + test.write_summary("b22-botnet-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B24_ip_anycast/README.md b/examples/internet/B24_ip_anycast/README.md index 60a9cd8bc..1ddf2481e 100644 --- a/examples/internet/B24_ip_anycast/README.md +++ b/examples/internet/B24_ip_anycast/README.md @@ -1,83 +1,128 @@ -# IP Anycast +# IP Anycast -The purpose of this example is to demonstrate the IP anycast feature. -We will use the emulator from example-B00 as the base. -The IP anycast technology allows multiple computers on the Internet to have -the same IP address. When another machine sends a packet to this IP address, -one of the computers will get the packet. Exactly which one will -get the packet depends on BGP routing. IP anycast is naturally supported by BGP. +This example demonstrates IP anycast using the `B00_mini_internet` topology as +the base Internet. IP anycast lets multiple sites use the same IP address. BGP +then decides which site receives traffic for that address. -One of the well-known applications of IP anycast is the DNS root servers. -Some of the DNS root servers, such as the F server, -have multiple machines geographically located in many different places around the world, -but they have the same IP address. -For example, all the F servers have the same IP address `192.5.5.241`. -When a DNS client sends a request to this -IP address, one of the F servers will get the request. +One well-known use case is DNS root service. For example, one logical DNS root +server can be deployed at many physical locations, while clients still send +queries to one stable IP address. The network routes each client to one of the +available sites. +## Topology -## Create Two Hosts with Same IP Address +The example adds `AS180` to the B00 mini Internet. `AS180` has two disconnected +sites: -We will first create an autonomous system (`AS-180`). We create two networks -inside this AS, but we give them the same network prefix. We also create -a host on each network, and assign the same IP address to them. +```text +IX100 side: + host-0: 10.180.0.100 + router0: connected to IX100 +IX105 side: + host-1: 10.180.0.100 + router1: connected to IX105 ``` -as180 = base.createAutonomousSystem(180) -as180.createNetwork('net0', '10.180.0.0/24') -as180.createNetwork('net1', '10.180.0.0/24') -as180.createHost('host-0').joinNetwork('net0', address = '10.180.0.100') -as180.createHost('host-1').joinNetwork('net1', address = '10.180.0.100') -``` -It should be noted that this AS has two disjoint parts: one has `net0`, and the -other has `net1`. These two networks are not connected by any router. -We then connect one part of the AS to `ix100`, and connect the other part -of the AS to `ix105`, and peer the AS with other ASes at these two locations. +Both hosts use the same address, `10.180.0.100`, but they are on different +internal networks: + +```python +as180.createNetwork("net0", "10.180.0.0/24") +as180.createNetwork("net1", "10.180.0.0/24") +as180.createHost("host-0").joinNetwork("net0", address="10.180.0.100") +as180.createHost("host-1").joinNetwork("net1", address="10.180.0.100") ``` -as180.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') -ebgp.addPrivatePeerings(100, [3, 4], [180], PeerRelationship.Provider) -as180.createRouter('router1').joinNetwork('net1').joinNetwork('ix105') -ebgp.addPrivatePeerings(105, [2, 3], [180], PeerRelationship.Provider) +The two sites connect to the Internet at different IXes: + +```python +as180.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") +ebgp.addPrivatePeerings(100, [3, 4], [180], PeerRelationship.Provider) + +as180.createRouter("router1").joinNetwork("net1").joinNetwork("ix105") +ebgp.addPrivatePeerings(105, [2, 3], [180], PeerRelationship.Provider) ``` -When compiling the emulation to generate the docker files, we need -to set the `selfManagedNetwork` option to True. Without this option, -the network will be entirely managed by Docker, which will not -allow us to have two networks with the same IP prefix. Using -this option will solve this problem. +Each anycast host runs a small web server with different content. This makes +the selected anycast site visible during testing: +```text +host-0 response contains: ix100-west +host-1 response contains: ix105-east ``` -emu.compile(Docker(selfManagedNetwork=True), './output') + +## Self-Managed Docker Networks + +The Docker compiler must use `selfManagedNetwork=True` for this example. +Without this option, Docker manages the networks and does not allow two +different Docker networks to use the same IP prefix. The example entrypoint +sets this option automatically: + +```python +Docker(selfManagedNetwork=True, platform=platform) ``` +## Standard Arguments -Now, we have deployed two hosts using the IP anycast technology. -When other machines send a packet to `10.180.0.100`, one of these two -hosts will get the packet. +The entrypoint supports the standard example arguments used by +`seedemu/testing/cli.py`: + +```sh +python examples/internet/B24_ip_anycast/ip_anycast.py --platform amd --output examples/internet/B24_ip_anycast/output +``` +The legacy platform argument is also accepted: -## ICMP Testing +```sh +python examples/internet/B24_ip_anycast/ip_anycast.py amd +``` -We can ping `10.180.0.100` from one of the hosts in the emulator, -we set the filter to `icmp`. We should be able to see the -traffic going to one of the `10.180.0.100` hosts. If we disable -this location's BGP session, we will see that the traffic -immediately switches to the other `10.180.0.100` host. -We should also be able to find two hosts inside the emulator, -such that they talk to a different `10.180.0.100` host. +Useful options: +```text +--platform amd|arm +--output PATH +--dumpfile PATH +--hosts-per-as N +--skip-render +--no-override +``` -## UDP Testing +## Test Runner -We can run a UDP server on each of the `10.180.0.100` host using `nc -luk 9090`. -Then from another machine, we send a UDP message to `10.180.0.100`. -One of the hosts will receive the message, depending on where the client -is located. If we disable the BGP session at one of the locations, -we can see that the UDP message will go to the other server. +Use the standardized runner from the repository root: +```sh +python seedemu/testing/cli.py clean examples/internet/B24_ip_anycast/example.yaml +python seedemu/testing/cli.py compile examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 +python seedemu/testing/cli.py build examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 +python seedemu/testing/cli.py up examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 +python seedemu/testing/cli.py probe examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 +python seedemu/testing/cli.py test examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 +python seedemu/testing/cli.py down examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 ``` -# echo hello | nc -u 10.180.0.100 9090 + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B24_ip_anycast/example.yaml --artifact-dir ci-artifacts/b24 ``` + +## Runtime Checks + +The declarative probes check the externally visible anycast behavior: + +- a host in `AS150` reaches `10.180.0.100` and receives the IX100-side web + response; +- a host in `AS170` reaches `10.180.0.100` and receives the IX105-side web + response. + +The custom `test_runtime.py` adds more specific validation: + +- both AS180 hosts exist and have the same anycast address; +- both local web servers are running and expose their site markers; +- both AS180 routers have the anycast prefix in BIRD; +- representative clients from different parts of the mini Internet reach + different anycast sites. diff --git a/examples/internet/B24_ip_anycast/example.yaml b/examples/internet/B24_ip_anycast/example.yaml new file mode 100644 index 000000000..bf6c02b4b --- /dev/null +++ b/examples/internet/B24_ip_anycast/example.yaml @@ -0,0 +1,66 @@ +id: internet-b24-ip-anycast +name: IP Anycast +description: A B00-based mini Internet that demonstrates two AS180 sites announcing and serving the same anycast IP address. +runner: internet +script: ip_anycast.py +platform: amd +features: + - ipv4-default + - mini-internet + - ebgp-private-peering + - ip-anycast + - web-service + +compile: + enabled: true + output: output + clean: + - output + - base_internet.bin + expected: + - output/docker-compose.yml + - base_internet.bin + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B24 anycast services are running + type: compose-ps + services: + - hnode_150_host_0 + - hnode_170_host_0 + - hnode_180_host-0 + - hnode_180_host-1 + - brdnode_180_router0 + - brdnode_180_router1 + retries: 40 + interval: 3 + +probes: + - name: AS150 reaches the IX100-side anycast site + type: exec + service: hnode_150_host_0 + command: curl -fsS http://10.180.0.100 | grep -q 'ix100-west' + expect_exit: 0 + retries: 40 + interval: 5 + timeout: 45 + + - name: AS170 reaches the IX105-side anycast site + type: exec + service: hnode_170_host_0 + command: curl -fsS http://10.180.0.100 | grep -q 'ix105-east' + expect_exit: 0 + retries: 40 + interval: 5 + timeout: 45 + +test_programs: + - name: B24 custom runtime validation + script: test_runtime.py + timeout: 600 diff --git a/examples/internet/B24_ip_anycast/ip_anycast.py b/examples/internet/B24_ip_anycast/ip_anycast.py index fe046388b..925cfc3d7 100755 --- a/examples/internet/B24_ip_anycast/ip_anycast.py +++ b/examples/internet/B24_ip_anycast/ip_anycast.py @@ -1,70 +1,121 @@ #!/usr/bin/env python3 # encoding: utf-8 +# +# Purpose: build the B24 IP anycast example. Inputs are standard TestRunner +# CLI arguments. Outputs are Docker compiler files under --output. -from seedemu.core import Emulator +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from examples.internet.B00_mini_internet import mini_internet from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter from seedemu.layers import Base, Ebgp, PeerRelationship -from examples.internet.B00_mini_internet import mini_internet -import sys, os - -def run(dumpfile=None): - ############################################################################### - # Set the platform information - if dumpfile is None: - script_name = os.path.basename(__file__) - - if len(sys.argv) == 1: - platform = Platform.AMD64 - elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) +from seedemu.services import WebService + +ANYCAST_ADDRESS = "10.180.0.100" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B24 IP anycast example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator(hosts_per_as: int = 2) -> Emulator: emu = Emulator() - # Run the pre-built component; load it into the current emulator - mini_internet.run(dumpfile='./base_internet.bin') - emu.load('./base_internet.bin') - - base: Base = emu.getLayer('Base') - ebgp: Ebgp = emu.getLayer('Ebgp') - - # Create a new AS with two disjoint networks, but the - # IP prefix of these two networks are the same. + base_component = SCRIPT_DIR / "base_internet.bin" + mini_internet.run(dumpfile=str(base_component), hosts_per_as=hosts_per_as) + emu.load(str(base_component)) + + base: Base = emu.getLayer("Base") + ebgp: Ebgp = emu.getLayer("Ebgp") + web = WebService() + + # AS180 has two disconnected sites that both announce the same /24. as180 = base.createAutonomousSystem(180) - as180.createNetwork('net0', '10.180.0.0/24') - as180.createNetwork('net1', '10.180.0.0/24') - - # Create a host on each network, but assign them the same IP address - as180.createHost('host-0').joinNetwork('net0', address = '10.180.0.100') - as180.createHost('host-1').joinNetwork('net1', address = '10.180.0.100') - - # Attach one network to IX-100 (via BGP router) - # Peer AS-180 with AS-3 and AS-4 - as180.createRouter('router0').joinNetwork('net0').joinNetwork('ix100') - ebgp.addPrivatePeerings(100, [3, 4], [180], PeerRelationship.Provider) - - # Attach the other network to IX-105 (via a different BGP router) - # Peer AS-180 with AS-2 and AS-3 - as180.createRouter('router1').joinNetwork('net1').joinNetwork('ix105') - ebgp.addPrivatePeerings(105, [2, 3], [180], PeerRelationship.Provider) - + as180.createNetwork("net0", "10.180.0.0/24") + as180.createNetwork("net1", "10.180.0.0/24") + + as180.createHost("host-0").joinNetwork("net0", address=ANYCAST_ADDRESS) + as180.createHost("host-1").joinNetwork("net1", address=ANYCAST_ADDRESS) + + as180.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + ebgp.addPrivatePeerings(100, [3, 4], [180], PeerRelationship.Provider) + + as180.createRouter("router1").joinNetwork("net1").joinNetwork("ix105") + ebgp.addPrivatePeerings(105, [2, 3], [180], PeerRelationship.Provider) + + web.install("anycast-west").setIndexContent("B24 anycast site: ix100-west\n") + web.install("anycast-east").setIndexContent("B24 anycast site: ix105-east\n") + emu.addBinding(Binding("anycast-west", filter=Filter(asn=180, nodeName="host-0"))) + emu.addBinding(Binding("anycast-east", filter=Filter(asn=180, nodeName="host-1"))) + emu.addLayer(web) + + return emu + + +def run( + dumpfile=None, + hosts_per_as: int = 2, + output=None, + platform=Platform.AMD64, + override: bool = True, + render: bool = True, +): + emu = build_emulator(hosts_per_as=hosts_per_as) if dumpfile is not None: - # Save it to a file, so it can be used by other emulators - emu.dump(dumpfile) - else: - emu.render() - # We need to set the selfManagedNetwork option to True (see README) - emu.compile(Docker(selfManagedNetwork=True, platform=platform), './output', override=True) + emu.dump(dumpfile) + return + + if render: + emu.render() + + # selfManagedNetwork is required because AS180 intentionally has two Docker + # networks with the same IP prefix. + docker = Docker(selfManagedNetwork=True, platform=platform) + emu.compile(docker, output or str(SCRIPT_DIR / "output"), override=override) -if __name__ == "__main__": - run() +def main() -> int: + args = parse_args() + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(output_dir), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B24_ip_anycast/test_runtime.py b/examples/internet/B24_ip_anycast/test_runtime.py new file mode 100644 index 000000000..db215279c --- /dev/null +++ b/examples/internet/B24_ip_anycast/test_runtime.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Purpose: validate the running B24 IP anycast example after TestRunner starts +# Docker Compose. Inputs come from TestRunner environment variables and compose +# labels. Outputs are JSON runtime summaries. + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +ANYCAST_ADDRESS = "10.180.0.100" + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + west_site = test.require_service(180, "host-0") + east_site = test.require_service(180, "host-1") + west_router = test.require_service(180, "router0") + east_router = test.require_service(180, "router1") + west_client = test.require_service(150, "host_0") + east_client = test.require_service(170, "host_0") + + if west_site and east_site: + test.structural_check( + "AS180 anycast sites share one service address", + west_site.address == ANYCAST_ADDRESS and east_site.address == ANYCAST_ADDRESS, + "host-0 address={}, host-1 address={}".format(west_site.address, east_site.address), + ) + test.exec_check( + "IX100-side anycast web server has west marker", + west_site, + "curl -fsS http://127.0.0.1 | grep -q 'ix100-west'", + ) + test.exec_check( + "IX105-side anycast web server has east marker", + east_site, + "curl -fsS http://127.0.0.1 | grep -q 'ix105-east'", + ) + + if west_router: + test.exec_check( + "AS180 router0 advertises the anycast prefix", + west_router, + "birdc show route 10.180.0.0/24 | grep -q '10.180.0.0/24'", + retries=30, + interval=3, + timeout=45, + ) + + if east_router: + test.exec_check( + "AS180 router1 advertises the anycast prefix", + east_router, + "birdc show route 10.180.0.0/24 | grep -q '10.180.0.0/24'", + retries=30, + interval=3, + timeout=45, + ) + + if west_client: + test.exec_check( + "AS150 reaches the IX100-side anycast site", + west_client, + "curl -fsS http://{} | grep -q 'ix100-west'".format(ANYCAST_ADDRESS), + retries=40, + interval=5, + timeout=45, + ) + + if east_client: + test.exec_check( + "AS170 reaches the IX105-side anycast site", + east_client, + "curl -fsS http://{} | grep -q 'ix105-east'".format(ANYCAST_ADDRESS), + retries=40, + interval=5, + timeout=45, + ) + + test.write_summary("b24-ip-anycast-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B25_pki/README.md b/examples/internet/B25_pki/README.md index b283f1cee..fc6de569b 100644 --- a/examples/internet/B25_pki/README.md +++ b/examples/internet/B25_pki/README.md @@ -45,8 +45,8 @@ as151.createHost('web2').joinNetwork('net0').addHostName('bank32.com') This approach is much more complicated, as it needs to set up a DNS infrastructure, consisting of multiple nameservers, including the root servers, TLD name servers, and specific domain name servers. -The example `pki-with-dns.py` uses this approach. Part of the -DNS set up is in `basenetwithDNS.py`. For detailed instructions on +The example `pki_with_dns.py` uses this approach. Part of the +DNS set up is in `base_internet_with_dns.py`. For detailed instructions on how to create a DNS, please see `examples/B01-dns-component`. In this example, we create three physical nodes, and then @@ -211,3 +211,63 @@ Certificate: 55:83:bd:ab:26:0f:66:1f:38:5f:24:67:17:d3:e0:32 ``` +## Standard Arguments + +The main testable variant for this example is `pki.py`, which uses the +`EtcHosts` layer for name resolution. + +```sh +python examples/internet/B25_pki/pki.py amd +python examples/internet/B25_pki/pki.py --platform amd --output examples/internet/B25_pki/output +python examples/internet/B25_pki/pki.py --dumpfile examples/internet/B25_pki/pki.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## Standardized TestRunner Lifecycle + +Run the full lifecycle from the repository root: + +```sh +python seedemu/testing/cli.py all examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +``` + +The lifecycle can also be run step by step: + +```sh +python seedemu/testing/cli.py clean examples/internet/B25_pki/example.yaml +python seedemu/testing/cli.py compile examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +python seedemu/testing/cli.py build examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +python seedemu/testing/cli.py up examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +python seedemu/testing/cli.py probe examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +python seedemu/testing/cli.py test examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +python seedemu/testing/cli.py down examples/internet/B25_pki/example.yaml --artifact-dir ci-artifacts/b25-pki +``` + +The tests focus on PKI behavior introduced by this example. They do not repeat +the base Internet routing tests. + +The declarative probes check: + +- `example32.com` and `bank32.com` resolve through `/etc/hosts`. +- Both web servers serve HTTPS content. +- The HTTPS connections are trusted by a representative client. + +The custom `test_runtime.py` program checks: + +- CA server containers are generated and `step-ca` is running. +- CA ACME directory endpoints are reachable. +- Internal root CA certificates are installed. +- Web server containers are generated with the expected fixed addresses. +- Web servers obtain ACME certificates under `/etc/letsencrypt/live/...`. +- A representative client can resolve CA and web names. +- A representative client can fetch both HTTPS sites without certificate errors. +- The issued certificates contain the expected DNS subject alternative names. + diff --git a/examples/internet/B25_pki/example.yaml b/examples/internet/B25_pki/example.yaml new file mode 100644 index 000000000..33edec3c5 --- /dev/null +++ b/examples/internet/B25_pki/example.yaml @@ -0,0 +1,111 @@ +id: internet-b25-pki +name: PKI With ACME +description: A small Internet deployment with two internal CAs issuing trusted HTTPS certificates to two web servers. +runner: internet +script: pki.py +platform: amd +features: + - ipv4-default + - pki + - acme + - https + - etc-hosts + +compile: + enabled: true + output: output + clean: + - output + - base_internet.bin + expected: + - output/docker-compose.yml + - base_internet.bin + timeout: 900 + +build: + enabled: true + timeout: 2400 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: B25 PKI services are running + type: compose-ps + services: + - hnode_150_ca1 + - hnode_150_ca2 + - hnode_151_web1 + - hnode_151_web2 + - hnode_150_host_0 + retries: 40 + interval: 3 + + - name: CA1 ACME endpoint is ready + type: exec + service: hnode_150_ca1 + command: "curl -kfsS https://seedCA.net/acme/acme/directory >/dev/null" + expect_exit: 0 + retries: 60 + interval: 5 + + - name: CA2 ACME endpoint is ready + type: exec + service: hnode_150_ca2 + command: "curl -kfsS https://seedCA.com/acme/acme/directory >/dev/null" + expect_exit: 0 + retries: 60 + interval: 5 + + - name: example32.com HTTPS endpoint is ready + type: exec + service: hnode_150_host_0 + command: "curl -fsS https://example32.com >/dev/null" + expect_exit: 0 + retries: 60 + interval: 5 + + - name: bank32.com HTTPS endpoint is ready + type: exec + service: hnode_150_host_0 + command: "curl -fsS https://bank32.com >/dev/null" + expect_exit: 0 + retries: 60 + interval: 5 + +probes: + - name: example32.com resolves through EtcHosts + type: exec + service: hnode_150_host_0 + command: "getent hosts example32.com | grep -q '10.151.0.7'" + expect_exit: 0 + retries: 20 + interval: 3 + + - name: bank32.com resolves through EtcHosts + type: exec + service: hnode_150_host_0 + command: "getent hosts bank32.com | grep -q '10.151.0.8'" + expect_exit: 0 + retries: 20 + interval: 3 + + - name: example32.com serves trusted HTTPS content + type: exec + service: hnode_150_host_0 + command: "curl -fsS https://example32.com | grep -q 'Web server at example32.com'" + expect_exit: 0 + retries: 30 + interval: 5 + + - name: bank32.com serves trusted HTTPS content + type: exec + service: hnode_150_host_0 + command: "curl -fsS https://bank32.com | grep -q 'Web server at bank32.com'" + expect_exit: 0 + retries: 30 + interval: 5 + +test_programs: + - name: B25 PKI runtime validation + script: test_runtime.py + timeout: 600 diff --git a/examples/internet/B25_pki/pki.py b/examples/internet/B25_pki/pki.py index 4a660696f..8c9bec585 100755 --- a/examples/internet/B25_pki/pki.py +++ b/examples/internet/B25_pki/pki.py @@ -1,86 +1,134 @@ #!/usr/bin/env python3 # encoding: utf-8 +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + from seedemu.compiler import Docker, Platform from seedemu.core import Binding, Emulator, Filter, Action from seedemu.layers import Base from seedemu.services import CAService, CAServer, WebService, WebServer, RootCAStore import base_internet -import os, sys - -############################################################################### -# Set the platform information -script_name = os.path.basename(__file__) - -if len(sys.argv) == 1: - platform = Platform.AMD64 -elif len(sys.argv) == 2: - if sys.argv[1].lower() == 'amd': - platform = Platform.AMD64 - elif sys.argv[1].lower() == 'arm': - platform = Platform.ARM64 - else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) -else: - print(f"Usage: {script_name} amd|arm") - sys.exit(1) - -base_internet.run(dumpfile='./base_internet.bin') - -emu = Emulator() -emu.load('./base_internet.bin') - -base: Base = emu.getLayer('Base') - -# Create a physical node for CA servers -as150 = base.getAutonomousSystem(150) -as150.createHost('ca1').joinNetwork('net0').addHostName('seedCA.net') -as150.createHost('ca2').joinNetwork('net0').addHostName('seedCA.com') - -# Create two physical nodes for web servers -as151 = base.getAutonomousSystem(151) -as151.createHost('web1').joinNetwork('net0', address='10.151.0.7') \ - .addHostName('example32.com') -as151.createHost('web2').joinNetwork('net0', address='10.151.0.8') \ - .addHostName('bank32.com') - -# Create and configure CA server vnodes -caStore1 = RootCAStore(caDomain='seedCA.net') -caStore2 = RootCAStore(caDomain='seedCA.com') -ca = CAService() -caServer1: CAServer = ca.install('ca1-vnode') -caServer1.setCAStore(caStore1) -caServer1.setCertDuration("2160h") -caServer1.installCACert() - -caServer2: CAServer = ca.install('ca2-vnode') -caServer2.setCAStore(caStore2) -caServer2.setCertDuration("2160h") -caServer2.installCACert() - -# Create and configure web server vnodes -web = WebService() -webServer1: WebServer = web.install('web1-vnode') -webServer1.setServerNames(['example32.com']) -webServer1.setCAServer(caServer1).enableHTTPS() -webServer1.setIndexContent("

Web server at example32.com

") - -webServer2: WebServer = web.install('web2-vnode') -webServer2.setServerNames(['bank32.com']) -webServer2.setCAServer(caServer2).enableHTTPS() -webServer2.setIndexContent("

Web server at bank32.com

") - -# Bind vnodes to physical nodes -emu.addBinding(Binding('ca1-vnode', filter=Filter(nodeName='ca1'), action=Action.FIRST)) -emu.addBinding(Binding('ca2-vnode', filter=Filter(nodeName='ca2'), action=Action.FIRST)) -emu.addBinding(Binding('web1-vnode', filter=Filter(nodeName='web1'), action=Action.FIRST)) -emu.addBinding(Binding('web2-vnode', filter=Filter(nodeName='web2'), action=Action.FIRST)) - - -# Add layers, render and compile -emu.addLayer(ca) -emu.addLayer(web) - -emu.render() -emu.compile(Docker(platform=platform), './output', override=True) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B25 PKI example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + base_component = SCRIPT_DIR / "base_internet.bin" + base_internet.run(dumpfile=str(base_component)) + + emu = Emulator() + emu.load(str(base_component)) + + base: Base = emu.getLayer("Base") + + # Create physical nodes for CA servers. + as150 = base.getAutonomousSystem(150) + as150.createHost("ca1").joinNetwork("net0").addHostName("seedCA.net") + as150.createHost("ca2").joinNetwork("net0").addHostName("seedCA.com") + + # Create physical nodes for HTTPS web servers. + as151 = base.getAutonomousSystem(151) + as151.createHost("web1").joinNetwork("net0", address="10.151.0.7").addHostName("example32.com") + as151.createHost("web2").joinNetwork("net0", address="10.151.0.8").addHostName("bank32.com") + + # Create and configure CA server virtual nodes. + ca_store_1 = RootCAStore(caDomain="seedCA.net") + ca_store_2 = RootCAStore(caDomain="seedCA.com") + ca = CAService() + + ca_server_1: CAServer = ca.install("ca1-vnode") + ca_server_1.setCAStore(ca_store_1) + ca_server_1.setCertDuration("2160h") + ca_server_1.installCACert() + + ca_server_2: CAServer = ca.install("ca2-vnode") + ca_server_2.setCAStore(ca_store_2) + ca_server_2.setCertDuration("2160h") + ca_server_2.installCACert() + + # Create and configure HTTPS web server virtual nodes. + web = WebService() + + web_server_1: WebServer = web.install("web1-vnode") + web_server_1.setServerNames(["example32.com"]) + web_server_1.setCAServer(ca_server_1).enableHTTPS() + web_server_1.setIndexContent("

Web server at example32.com

") + + web_server_2: WebServer = web.install("web2-vnode") + web_server_2.setServerNames(["bank32.com"]) + web_server_2.setCAServer(ca_server_2).enableHTTPS() + web_server_2.setIndexContent("

Web server at bank32.com

") + + # Bind virtual nodes to physical nodes. + emu.addBinding(Binding("ca1-vnode", filter=Filter(nodeName="ca1"), action=Action.FIRST)) + emu.addBinding(Binding("ca2-vnode", filter=Filter(nodeName="ca2"), action=Action.FIRST)) + emu.addBinding(Binding("web1-vnode", filter=Filter(nodeName="web1"), action=Action.FIRST)) + emu.addBinding(Binding("web2-vnode", filter=Filter(nodeName="web2"), action=Action.FIRST)) + + emu.addLayer(ca) + emu.addLayer(web) + return emu + + +def run( + dumpfile=None, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator() + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B25_pki/test_runtime.py b/examples/internet/B25_pki/test_runtime.py new file mode 100644 index 000000000..adf3a084a --- /dev/null +++ b/examples/internet/B25_pki/test_runtime.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + ca1 = test.require_service(150, "ca1") + ca2 = test.require_service(150, "ca2") + web1 = test.require_service(151, "web1") + web2 = test.require_service(151, "web2") + client = test.require_service(150, "host_0", "representative PKI client is generated") + + if ca1: + test.exec_check("CA1 step-ca process is running", ca1, "pgrep step-ca >/dev/null") + test.exec_check( + "CA1 ACME directory is reachable", + ca1, + "curl -kfsS https://seedCA.net/acme/acme/directory | grep -q 'newNonce'", + retries=60, + interval=5, + ) + test.exec_check( + "CA1 root certificate is installed locally", + ca1, + "test -s /usr/local/share/ca-certificates/SEEDEMU_Internal_Root_CA_0.crt", + ) + + if ca2: + test.exec_check("CA2 step-ca process is running", ca2, "pgrep step-ca >/dev/null") + test.exec_check( + "CA2 ACME directory is reachable", + ca2, + "curl -kfsS https://seedCA.com/acme/acme/directory | grep -q 'newNonce'", + retries=60, + interval=5, + ) + test.exec_check( + "CA2 root certificate is installed locally", + ca2, + "test -s /usr/local/share/ca-certificates/SEEDEMU_Internal_Root_CA_1.crt", + ) + + if web1: + test.structural_check( + "web1 has the expected fixed address", + web1.address == "10.151.0.7", + "observed {}".format(web1.address), + ) + test.exec_check("web1 nginx process is running", web1, "pgrep nginx >/dev/null") + test.exec_check( + "web1 obtained an ACME certificate for example32.com", + web1, + "test -s /etc/letsencrypt/live/example32.com/fullchain.pem", + retries=60, + interval=5, + ) + + if web2: + test.structural_check( + "web2 has the expected fixed address", + web2.address == "10.151.0.8", + "observed {}".format(web2.address), + ) + test.exec_check("web2 nginx process is running", web2, "pgrep nginx >/dev/null") + test.exec_check( + "web2 obtained an ACME certificate for bank32.com", + web2, + "test -s /etc/letsencrypt/live/bank32.com/fullchain.pem", + retries=60, + interval=5, + ) + + if client: + test.exec_check( + "client resolves CA and web hostnames through EtcHosts", + client, + "getent hosts seedCA.net | grep -q '10.150.' && " + "getent hosts seedCA.com | grep -q '10.150.' && " + "getent hosts example32.com | grep -q '10.151.0.7' && " + "getent hosts bank32.com | grep -q '10.151.0.8'", + ) + test.exec_check( + "client trusts example32.com HTTPS certificate", + client, + "curl -fsS https://example32.com | grep -q 'Web server at example32.com'", + retries=60, + interval=5, + ) + test.exec_check( + "client trusts bank32.com HTTPS certificate", + client, + "curl -fsS https://bank32.com | grep -q 'Web server at bank32.com'", + retries=60, + interval=5, + ) + test.exec_check( + "example32.com certificate has expected SAN", + client, + "step certificate inspect https://example32.com | grep -q 'DNS:example32.com'", + retries=30, + interval=5, + ) + test.exec_check( + "bank32.com certificate has expected SAN", + client, + "step certificate inspect https://bank32.com | grep -q 'DNS:bank32.com'", + retries=30, + interval=5, + ) + + test.write_summary("b25-pki-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B26_ipfs_kubo/README.md b/examples/internet/B26_ipfs_kubo/README.md index 71930e141..c34fb11d2 100644 --- a/examples/internet/B26_ipfs_kubo/README.md +++ b/examples/internet/B26_ipfs_kubo/README.md @@ -69,7 +69,7 @@ of a node to a physical node in the Emulator. One way to do all of this is as fo emu.compile(docker, OUTPUDIR, override = True) ``` Here, we additionally enable the *Internet Map*, an emulator functionality that displays a - visualization of the emulation in your browser at http://localhost:8080/map.html. We also enable override for our compiler, so that it will overwrite previous compiles each time we + visualization of the emulation in your browser at http://localhost:8080/pro/map. We also enable override for our compiler, so that it will overwrite previous compiles each time we run this script. 7. Finally, go ahead and run the emulation script, in this case, `kubo.py`. This will compile @@ -85,7 +85,7 @@ the emulation. ``` - Now that the emulation is running, you can view the network topology and access each host -via the Internet Map at http://localhost:8080/map.html. Alternatively, you can use the `docker +via the Internet Map at http://localhost:8080/pro/map. Alternatively, you can use the `docker exec` command to attach a CLI session to a given Docker container. - The best way to interact with IPFS here, is to use the Kubo CLI on any host that has Kubo installed. diff --git a/examples/internet/B29_email_dns/README.md b/examples/internet/B29_email_dns/README.md index 267598c37..e1b2a2b81 100644 --- a/examples/internet/B29_email_dns/README.md +++ b/examples/internet/B29_email_dns/README.md @@ -2,7 +2,7 @@ A realistic multi-ISP, multi-IX email system using DNS-based MX routing with a Roundcube webmail frontend. This is the single source of truth for running and validating the B29 scenario. -- Internet Map: http://localhost:8080/map.html +- Internet Map: http://localhost:8080/pro/map - Roundcube: http://localhost:8082 ## Status diff --git a/examples/internet/B29_email_dns/b29ctl.sh b/examples/internet/B29_email_dns/b29ctl.sh index fe11f4cb5..1ef603b01 100755 --- a/examples/internet/B29_email_dns/b29ctl.sh +++ b/examples/internet/B29_email_dns/b29ctl.sh @@ -52,7 +52,7 @@ b29_up() { "$SCRIPT_DIR/manage_roundcube.sh" accounts || true log "Starting Roundcube ..." "$SCRIPT_DIR/manage_roundcube.sh" start - log "Done. Map: http://localhost:8080/map.html Roundcube: http://localhost:8082" + log "Done. Map: http://localhost:8080/pro/map Roundcube: http://localhost:8082" } b29_down() { diff --git a/examples/internet/B29_email_dns/docs/testing_guide.md b/examples/internet/B29_email_dns/docs/testing_guide.md index 4de2c842c..cc51b8bb7 100644 --- a/examples/internet/B29_email_dns/docs/testing_guide.md +++ b/examples/internet/B29_email_dns/docs/testing_guide.md @@ -2,7 +2,7 @@ This guide provides a thorough checklist and runnable commands to validate the B29 DNS-first email example end-to-end, including: DNS, BGP, email flows (intra- and cross-domain), logging/record verification, resilience, and large-scale/roster-based automation. -- Internet Map: http://localhost:8080/map.html +- Internet Map: http://localhost:8080/pro/map - Roundcube: http://localhost:8082 ## 1) Environment pre-checks diff --git a/examples/internet/B29_email_dns/email_realistic.py b/examples/internet/B29_email_dns/email_realistic.py index 21297ae54..60b9d9c07 100644 --- a/examples/internet/B29_email_dns/email_realistic.py +++ b/examples/internet/B29_email_dns/email_realistic.py @@ -504,7 +504,7 @@ def run(platform="auto"): SMTP: localhost:2205 | IMAP: localhost:1405 🌐 Monitoring: - Internet Map: http://localhost:8080/map.html + Internet Map: http://localhost:8080/pro/map ====================================================================== "") diff --git a/examples/internet/B30_CDN/README.md b/examples/internet/B30_CDN/README.md index 2d712e6fc..09ba3af63 100644 --- a/examples/internet/B30_CDN/README.md +++ b/examples/internet/B30_CDN/README.md @@ -1,87 +1,115 @@ -# Simple CDN +# CDNService Baseline Example -This example builds a simple CDN on top of the mini Internet example. +This example validates the new `CDNService` and its integration with +`DomainNameService`. -The deployment includes: +The current design keeps the two services loosely coupled: -- Three DNS servers, one in each region -- Three CDN edge nodes -- Three CDN origin nodes -- A single service domain: `www.example.com` +- DNS explicitly exposes an include file through `setInclude(...)` +- CDN explicitly writes policy-specific include content through `setIncludeContent(...)` -Each client uses a region-aware DNS server order. The DNS servers rewrite -`www.example.com` based on the client's source subnet and return the edge IP of -the East, West, or Central CDN site. Each edge node runs `nginx` and reverse -proxies requests to its local origin node, which also runs `nginx` and returns -a small JSON payload identifying the selected site. +The topology is built on top of the mini Internet and models a simple CDN with: +- one authoritative DNS server +- three edge sites +- one shared origin +- one service domain: `www.example.com` -## Topology +The CDN uses region-aware DNS answers: + +- East clients resolve to `10.181.0.100` +- West clients resolve to `10.182.0.100` +- Central clients resolve to `10.183.0.100` + +All three edge nodes proxy to the same origin at `10.184.0.10`. -This example reuses the base topology from -[B00_mini_internet](/Users/bruce/seed-emulator/examples/internet/B00_mini_internet). -The CDN sites are: +## Topology -- `nyc` for East -- `sjc` for West -- `chi` for Central +The example reuses the base topology from +`examples/internet/B00_mini_internet`. -The DNS servers are placed in: +CDN components: -- `AS150` at `10.150.0.53` -- `AS152` at `10.152.0.53` -- `AS154` at `10.154.0.53` +- DNS: `AS150`, `10.150.0.53` +- East edge: `AS181`, `10.181.0.100` +- West edge: `AS182`, `10.182.0.100` +- Central edge: `AS183`, `10.183.0.100` +- Shared origin: `AS184`, `10.184.0.10` ## Running -Generate the Docker output with: +Generate the Docker output: ```bash -python ./cdn.py amd +conda run -n seedpy310 python examples/internet/B30_CDN/cdn.py amd ``` -This creates the output directory: +This creates: ```bash -./output +examples/internet/B30_CDN/output ``` -Start the emulation manually: +Build and start the topology manually: ```bash -cd output -docker compose build +cd examples/internet/B30_CDN/output +DOCKER_BUILDKIT=0 docker compose build docker compose up -d ``` -Wait about 1-2 minutes for BGP, DNS, and `nginx` to settle. +Wait about 10-20 seconds for services to start. ## Verifying -From an East client: +East client: ```bash -docker exec -it as150h-host_0-10.150.0.71 sh -lc "cat /etc/resolv.conf" -docker exec -it as150h-host_0-10.150.0.71 sh -lc "getent ahostsv4 www.example.com" -docker exec -it as150h-host_0-10.150.0.71 sh -lc "curl --noproxy '*' -s http://www.example.com:8080/" +docker exec as150h-host_0-10.150.0.71 sh -lc 'getent ahostsv4 www.example.com' +docker exec as150h-host_0-10.150.0.71 sh -lc 'env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY -u all_proxy -u ALL_PROXY curl --noproxy "*" -s -D - http://www.example.com:8080/ -o /dev/null' ``` -From a West client: +West client: ```bash -docker exec -it as152h-host_0-10.152.0.71 sh -lc "getent ahostsv4 www.example.com" -docker exec -it as152h-host_0-10.152.0.71 sh -lc "curl --noproxy '*' -s http://www.example.com:8080/" +docker exec as152h-host_0-10.152.0.71 sh -lc 'getent ahostsv4 www.example.com' +docker exec as152h-host_0-10.152.0.71 sh -lc 'env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY -u all_proxy -u ALL_PROXY curl --noproxy "*" -s -D - http://www.example.com:8080/ -o /dev/null' ``` -From a Central client: +Central client: ```bash -docker exec -it as154h-host_0-10.154.0.71 sh -lc "getent ahostsv4 www.example.com" -docker exec -it as154h-host_0-10.154.0.71 sh -lc "curl --noproxy '*' -s http://www.example.com:8080/" +docker exec as154h-host_0-10.154.0.71 sh -lc 'getent ahostsv4 www.example.com' +docker exec as154h-host_0-10.154.0.71 sh -lc 'env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY -u all_proxy -u ALL_PROXY curl --noproxy "*" -s -D - http://www.example.com:8080/ -o /dev/null' ``` -The DNS answers should point to different edge IPs for different regions, and -the JSON response should identify the selected CDN site. +Expected results: + +- East resolves to `10.181.0.100` +- West resolves to `10.182.0.100` +- Central resolves to `10.183.0.100` +- HTTP responses include: + - `X-CDN-Origin: global_origin` + - `X-CDN-Origin-IP: 10.184.0.10` + - `X-CDN-Edge` + - `X-CDN-Region` + - `X-CDN-Edge-IP` + + +## What This Example Exercises + +This example checks that: + +- `CDNService` can bind virtual origin and edge nodes to concrete hosts +- final edge and origin configuration is rendered before container startup +- `DomainNameService` can host CDN-generated BIND views through an include file +- region-aware DNS steering and HTTP proxying work end to end + + +## Files + +- `examples/internet/B30_CDN/cdn.py`: baseline CDN built with `CDNService` +- `examples/internet/B30_CDN/output`: generated Docker deployment diff --git a/examples/internet/B30_CDN/cdn.py b/examples/internet/B30_CDN/cdn.py index 4579d5409..86a66ee5d 100644 --- a/examples/internet/B30_CDN/cdn.py +++ b/examples/internet/B30_CDN/cdn.py @@ -5,7 +5,6 @@ import argparse from pathlib import Path import sys -import textwrap SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[3] @@ -13,300 +12,203 @@ sys.path.insert(0, str(REPO_ROOT)) from seedemu.compiler import Docker, Platform -from seedemu.core import Emulator +from seedemu.core import Action, Binding, Emulator, Filter from seedemu.layers import Ebgp, PeerRelationship -from examples.internet.B00_mini_internet import mini_internet - - -SERVICE_DOMAIN = "www.example.com" -SERVICE_ZONE = "example.com" -SERVICE_ZONE_FQDN = "example.com." +from seedemu.services import CDNService, DomainNameService -CLIENT_REGION_BY_ASN = { - 150: "east", - 151: "east", - 152: "west", - 153: "west", - 154: "central", - 160: "east", - 161: "east", - 162: "east", - 163: "east", - 164: "east", - 170: "central", - 171: "central", -} +from examples.internet.B00_mini_internet import mini_internet -DNS_SERVERS = [ - {"id": "dns_east", "asn": 150, "node_name": "cdn_dns_east", "ip": "10.150.0.53"}, - {"id": "dns_west", "asn": 152, "node_name": "cdn_dns_west", "ip": "10.152.0.53"}, - {"id": "dns_central", "asn": 154, "node_name": "cdn_dns_central", "ip": "10.154.0.53"}, -] -DNS_ORDER_BY_REGION = { - "east": ["10.150.0.53", "10.154.0.53", "10.152.0.53"], - "west": ["10.152.0.53", "10.154.0.53", "10.150.0.53"], - "central": ["10.154.0.53", "10.150.0.53", "10.152.0.53"], +SERVICE_DOMAIN = 'www.example.com' +SERVICE_ZONE = 'example.com.' +OUTPUT_RELATIVE_PATH = 'examples/internet/B30_CDN/output' +DNS_INCLUDE_PATH = '/etc/bind/include/cdn_views.local' + +DNS_VNODE = 'dns-auth' +DNS_NODE_NAME = 'cdn_dns' +DNS_IP = '10.150.0.53' + +ORIGIN_VNODE = 'origin-main' +ORIGIN_SITE = { + 'asn': 184, + 'node_name': 'global_origin', + 'router_name': 'origin_br', + 'net_name': 'origin_net', + 'prefix': '10.184.0.0/24', + 'ip': '10.184.0.10', + 'ix': 102, + 'upstreams': [2, 4, 11], } -CDN_SITES = [ +EDGE_SITES = [ { - "site_id": "nyc", - "region": "east", - "ix": 100, - "asn": 181, - "prefix": "10.181.0.0/24", - "edge_ip": "10.181.0.100", - "origin_ip": "10.181.0.10", - "upstreams": [2, 3, 4], + 'vnode': 'edge-east', + 'region': 'east', + 'asn': 181, + 'node_name': 'nyc_edge', + 'router_name': 'nyc_br', + 'net_name': 'nyc_net', + 'prefix': '10.181.0.0/24', + 'ip': '10.181.0.100', + 'ix': 100, + 'upstreams': [2, 3, 4], }, { - "site_id": "sjc", - "region": "west", - "ix": 101, - "asn": 182, - "prefix": "10.182.0.0/24", - "edge_ip": "10.182.0.100", - "origin_ip": "10.182.0.10", - "upstreams": [2, 12], + 'vnode': 'edge-west', + 'region': 'west', + 'asn': 182, + 'node_name': 'sjc_edge', + 'router_name': 'sjc_br', + 'net_name': 'sjc_net', + 'prefix': '10.182.0.0/24', + 'ip': '10.182.0.100', + 'ix': 101, + 'upstreams': [2, 12], }, { - "site_id": "chi", - "region": "central", - "ix": 102, - "asn": 183, - "prefix": "10.183.0.0/24", - "edge_ip": "10.183.0.100", - "origin_ip": "10.183.0.10", - "upstreams": [2, 4, 11], + 'vnode': 'edge-central', + 'region': 'central', + 'asn': 183, + 'node_name': 'chi_edge', + 'router_name': 'chi_br', + 'net_name': 'chi_net', + 'prefix': '10.183.0.0/24', + 'ip': '10.183.0.100', + 'ix': 102, + 'upstreams': [2, 4, 11], }, ] -EDGE_IP_BY_REGION = {site["region"]: site["edge_ip"] for site in CDN_SITES} +CLIENT_REGION_BY_ASN = { + 150: 'east', + 151: 'east', + 152: 'west', + 153: 'west', + 154: 'central', + 160: 'east', + 161: 'east', + 162: 'east', + 163: 'east', + 164: 'east', + 170: 'central', + 171: 'central', +} def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Build a simple CDN example on top of mini_internet") - parser.add_argument("platform", nargs="?", default="amd", choices=["amd", "arm"]) + parser = argparse.ArgumentParser(description='Build a baseline CDN using CDNService') + parser.add_argument('platform', nargs='?', default='amd', choices=['amd', 'arm']) return parser.parse_args() def resolve_platform(name: str) -> Platform: - return Platform.AMD64 if name == "amd" else Platform.ARM64 - - -def build_origin_payload(site: dict) -> str: - return ( - "{" - f"\\\"site_id\\\":\\\"{site['site_id']}\\\"," - f"\\\"region\\\":\\\"{site['region']}\\\"," - "\\\"role\\\":\\\"origin\\\"," - f"\\\"asn\\\":{site['asn']}," - f"\\\"edge_ip\\\":\\\"{site['edge_ip']}\\\"," - f"\\\"origin_ip\\\":\\\"{site['origin_ip']}\\\"," - f"\\\"service_domain\\\":\\\"{SERVICE_DOMAIN}\\\"" - "}" - ) - - -def build_origin_nginx_conf(site: dict) -> str: - payload = build_origin_payload(site) - return textwrap.dedent( - f"""\ - server {{ - listen 8080 default_server; - server_name _; - - location / {{ - default_type application/json; - return 200 '{payload}'; - }} - }} - """ - ) + return Platform.AMD64 if name == 'amd' else Platform.ARM64 -def build_edge_nginx_conf(site: dict) -> str: - upstream_name = f"{site['site_id']}_origin" - return textwrap.dedent( - f"""\ - upstream {upstream_name} {{ - server {site["origin_ip"]}:8080; - keepalive 16; - }} - - server {{ - listen 8080 default_server; - server_name {SERVICE_DOMAIN} _; - - location / {{ - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-CDN-Site {site["site_id"]}; - proxy_set_header X-CDN-Region {site["region"]}; - proxy_pass http://{upstream_name}; - }} - }} - """ - ) +def add_edge_sites(base, ebgp: Ebgp): + for site in EDGE_SITES: + site_as = base.createAutonomousSystem(site['asn']) + site_as.createNetwork(site['net_name'], site['prefix']) + site_as.createHost(site['node_name']).joinNetwork(site['net_name'], address=site['ip']) + site_as.createRouter(site['router_name']).joinNetwork(site['net_name']).joinNetwork(f'ix{site["ix"]}') + ebgp.addPrivatePeerings(site['ix'], site['upstreams'], [site['asn']], PeerRelationship.Provider) -def install_origin_service(host, site: dict): - host.addSoftware("nginx-light") - host.setFile("/etc/nginx/sites-available/default", build_origin_nginx_conf(site)) - host.appendStartCommand("service nginx start") +def add_origin_site(base, ebgp: Ebgp): + origin_as = base.createAutonomousSystem(ORIGIN_SITE['asn']) + origin_as.createNetwork(ORIGIN_SITE['net_name'], ORIGIN_SITE['prefix']) + origin_as.createHost(ORIGIN_SITE['node_name']).joinNetwork(ORIGIN_SITE['net_name'], address=ORIGIN_SITE['ip']) + origin_as.createRouter(ORIGIN_SITE['router_name']).joinNetwork(ORIGIN_SITE['net_name']).joinNetwork(f'ix{ORIGIN_SITE["ix"]}') + ebgp.addPrivatePeerings(ORIGIN_SITE['ix'], ORIGIN_SITE['upstreams'], [ORIGIN_SITE['asn']], PeerRelationship.Provider) -def install_edge_service(host, site: dict): - host.addSoftware("nginx-light") - host.setFile("/etc/nginx/sites-available/default", build_edge_nginx_conf(site)) - host.appendStartCommand("service nginx start") +def add_dns_host(base): + dns_as = base.getAutonomousSystem(150) + dns_as.createHost(DNS_NODE_NAME).joinNetwork('net0', address=DNS_IP) + base.setNameServers([DNS_IP]) -def build_dns_zone(answer_ip: str, dns_ip: str) -> str: - return textwrap.dedent( - f"""\ - $TTL 60 - $ORIGIN {SERVICE_ZONE_FQDN} - @ IN SOA ns1.{SERVICE_ZONE_FQDN} admin.{SERVICE_ZONE_FQDN} 1 300 300 300 60 - @ IN NS ns1.{SERVICE_ZONE_FQDN} - @ IN A {answer_ip} - ns1 IN A {dns_ip} - www IN A {answer_ip} - """ - ) +def create_dns_layer() -> DomainNameService: + dns = DomainNameService() + dns.install(DNS_VNODE).addZone(SERVICE_ZONE).setMaster().setInclude(SERVICE_ZONE, DNS_INCLUDE_PATH) + return dns -def build_bind_patch_script(dns_ip: str) -> str: - subnets_by_region = {"east": [], "west": [], "central": []} - for asn, region in CLIENT_REGION_BY_ASN.items(): - subnets_by_region[region].append(f"10.{asn}.0.0/24;") - - named_conf_local = textwrap.dedent( - f"""\ - acl east_clients {{ {' '.join(subnets_by_region['east'])} }}; - acl west_clients {{ {' '.join(subnets_by_region['west'])} }}; - acl central_clients {{ {' '.join(subnets_by_region['central'])} }}; - - view "east" {{ - match-clients {{ east_clients; }}; - recursion no; - zone "{SERVICE_ZONE}" {{ type master; file "/etc/bind/zones/{SERVICE_ZONE}.east"; }}; - }}; - - view "west" {{ - match-clients {{ west_clients; }}; - recursion no; - zone "{SERVICE_ZONE}" {{ type master; file "/etc/bind/zones/{SERVICE_ZONE}.west"; }}; - }}; - - view "central" {{ - match-clients {{ central_clients; }}; - recursion no; - zone "{SERVICE_ZONE}" {{ type master; file "/etc/bind/zones/{SERVICE_ZONE}.central"; }}; - }}; - - view "default" {{ - match-clients {{ any; }}; - recursion no; - zone "{SERVICE_ZONE}" {{ type master; file "/etc/bind/zones/{SERVICE_ZONE}.default"; }}; - }}; - """ - ) +def create_cdn_layer() -> CDNService: + cdn = CDNService() - return ( - "#!/bin/sh\n" - "set -eu\n\n" - "mkdir -p /etc/bind/zones\n\n" - "cat > /etc/bind/named.conf <<'EOF_NAMED'\n" - 'include "/etc/bind/named.conf.options";\n' - 'include "/etc/bind/named.conf.local";\n' - "EOF_NAMED\n\n" - "cat > /etc/bind/named.conf.options <<'EOF_OPTIONS'\n" - "options {\n" - ' directory "/var/cache/bind";\n' - " recursion no;\n" - " dnssec-validation no;\n" - " empty-zones-enable no;\n" - " allow-query { any; };\n" - " listen-on { any; };\n" - " listen-on-v6 { any; };\n" - "};\n" - "EOF_OPTIONS\n\n" - "cat > /etc/bind/named.conf.local <<'EOF_LOCAL'\n" - f"{named_conf_local}" - "EOF_LOCAL\n\n" - f"cat > /etc/bind/zones/{SERVICE_ZONE}.east <<'EOF_EAST'\n" - f"{build_dns_zone(EDGE_IP_BY_REGION['east'], dns_ip)}" - "EOF_EAST\n\n" - f"cat > /etc/bind/zones/{SERVICE_ZONE}.west <<'EOF_WEST'\n" - f"{build_dns_zone(EDGE_IP_BY_REGION['west'], dns_ip)}" - "EOF_WEST\n\n" - f"cat > /etc/bind/zones/{SERVICE_ZONE}.central <<'EOF_CENTRAL'\n" - f"{build_dns_zone(EDGE_IP_BY_REGION['central'], dns_ip)}" - "EOF_CENTRAL\n\n" - f"cat > /etc/bind/zones/{SERVICE_ZONE}.default <<'EOF_DEFAULT'\n" - f"{build_dns_zone(EDGE_IP_BY_REGION['central'], dns_ip)}" - "EOF_DEFAULT\n\n" - "chown -R bind:bind /etc/bind/zones\n" - "service named restart || service named start\n" + cdn.createOrigin(ORIGIN_VNODE).setPort(8080).setServerName(SERVICE_DOMAIN).setIndexContent( + '

SEED CDN origin

' ) + for site in EDGE_SITES: + cdn.createEdge(site['vnode']).setRegion(site['region']).addOrigin(ORIGIN_VNODE) -def install_dns_service(host, dns_ip: str): - host.addSoftware("bind9") - host.setFile("/root/patch_bind_cdn.sh", build_bind_patch_script(dns_ip)) - host.appendStartCommand("chmod +x /root/patch_bind_cdn.sh") - host.appendStartCommand("/root/patch_bind_cdn.sh") + cdn.setDomain( + SERVICE_DOMAIN, + dnsVnode=DNS_VNODE, + edges=[site['vnode'] for site in EDGE_SITES], + mode='region', + zone=SERVICE_ZONE, + ) + cdn.setIncludeContent(SERVICE_DOMAIN, DNS_INCLUDE_PATH) + for site in EDGE_SITES: + cdn.mapRegion(SERVICE_DOMAIN, site['region'], [site['vnode']]) -def add_cdn_sites(base, ebgp: Ebgp): - for site in CDN_SITES: - site_as = base.createAutonomousSystem(site["asn"]) - net_name = f'{site["site_id"]}_net' - edge_name = f'{site["site_id"]}_edge' - origin_name = f'{site["site_id"]}_origin' - router_name = f'{site["site_id"]}_br' + region_members = {} + for asn, region in CLIENT_REGION_BY_ASN.items(): + region_members.setdefault(region, []).append(asn) - site_as.createNetwork(net_name, site["prefix"]) - site_as.createHost(edge_name).joinNetwork(net_name, address=site["edge_ip"]) - site_as.createHost(origin_name).joinNetwork(net_name, address=site["origin_ip"]) - site_as.createRouter(router_name).joinNetwork(net_name).joinNetwork(f'ix{site["ix"]}') + for region, asns in region_members.items(): + cdn.setRegionMembers(region, asns) - install_edge_service(site_as.getHost(edge_name), site) - install_origin_service(site_as.getHost(origin_name), site) - ebgp.addPrivatePeerings(site["ix"], site["upstreams"], [site["asn"]], PeerRelationship.Provider) + return cdn -def add_dns_servers(base): - for dns in DNS_SERVERS: - dns_as = base.getAutonomousSystem(dns["asn"]) - dns_as.createHost(dns["node_name"]).joinNetwork("net0", address=dns["ip"]) - install_dns_service(dns_as.getHost(dns["node_name"]), dns["ip"]) +def add_bindings(emu: Emulator): + emu.addBinding(Binding(DNS_VNODE, filter=Filter(asn=150, nodeName=DNS_NODE_NAME), action=Action.FIRST)) + emu.addBinding( + Binding( + ORIGIN_VNODE, + filter=Filter(asn=ORIGIN_SITE['asn'], nodeName=ORIGIN_SITE['node_name']), + action=Action.FIRST, + ) + ) - for asn, region in CLIENT_REGION_BY_ASN.items(): - base.getAutonomousSystem(asn).setNameServers(DNS_ORDER_BY_REGION[region]) + for site in EDGE_SITES: + emu.addBinding( + Binding(site['vnode'], filter=Filter(asn=site['asn'], nodeName=site['node_name']), action=Action.FIRST) + ) def build_emulator() -> Emulator: - base_bin = SCRIPT_DIR / "base_internet.bin" + base_bin = SCRIPT_DIR / 'base_internet.bin' mini_internet.run(dumpfile=str(base_bin), hosts_per_as=1) emu = Emulator() emu.load(str(base_bin)) - base = emu.getLayer("Base") - ebgp = emu.getLayer("Ebgp") + base = emu.getLayer('Base') + ebgp = emu.getLayer('Ebgp') + + add_edge_sites(base, ebgp) + add_origin_site(base, ebgp) + add_dns_host(base) + + dns = create_dns_layer() + cdn = create_cdn_layer() + + emu.addLayer(dns) + emu.addLayer(cdn) + add_bindings(emu) - add_cdn_sites(base, ebgp) - add_dns_servers(base) return emu def run(): args = parse_args() - output_dir = SCRIPT_DIR / "output" + output_dir = SCRIPT_DIR / 'output' output_dir.parent.mkdir(parents=True, exist_ok=True) emu = build_emulator() @@ -317,9 +219,8 @@ def run(): override=True, ) - print(f"Generated Docker output in: {output_dir}") - print("Docker was not started automatically.") + print(f'Generated Docker output in: {OUTPUT_RELATIVE_PATH}') -if __name__ == "__main__": +if __name__ == '__main__': run() diff --git a/examples/internet/B31_mini_internet_mpls/README.md b/examples/internet/B31_mini_internet_mpls/README.md new file mode 100644 index 000000000..e1e602a8e --- /dev/null +++ b/examples/internet/B31_mini_internet_mpls/README.md @@ -0,0 +1,99 @@ +# Mini Internet With MPLS Transit AS + +This example is based on `examples/internet/B00_mini_internet`. It keeps the +same Internet exchanges, transit ASes, stub ASes, route-server peerings, private +peerings, and customized AS154 host, but changes one large transit AS: + +```text +AS2 uses MPLS. +AS3, AS4, AS11, AS12, and all stub ASes keep the normal B00 routing behavior. +``` + +AS2 is a good demonstration target because it is a tier-1 transit AS connected +to `IX100`, `IX101`, `IX102`, and `IX105`. Its IX presence and peerings are +copied from B00. Its internal links are changed to pass through non-edge core +routers, so the MPLS layer has an internal provider backbone to configure. + +## Host System Support + +MPLS requires support from the Linux kernel on the emulator host. Before running +the Docker runtime, load the MPLS kernel module as root: + +```sh +modprobe mpls_router +``` + +Depending on the host distribution, `mpls_iptunnel` and `mpls_gso` may also be +needed. + +## Implementation + +The key difference from B00 is the `Mpls` layer: + +```python +mpls = Mpls() +mpls.enableOn(2) +``` + +The emulator then adds `mpls` along with the usual routing layers: + +```python +emu.addLayer(base) +emu.addLayer(Routing()) +emu.addLayer(ebgp) +emu.addLayer(mpls) +emu.addLayer(Ibgp()) +emu.addLayer(Ospf()) +``` + +The MPLS layer masks AS2 from the regular `Ibgp` and `Ospf` layers and installs +MPLS/LDP/OSPF configuration on AS2's border and core routers. Other ASes +continue to use the normal B00 behavior. + +## Standard Arguments + +The example accepts both the legacy platform argument and the newer named +arguments: + +```sh +python examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py amd +python examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py --platform amd --output examples/internet/B31_mini_internet_mpls/output +python examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py --dumpfile examples/internet/B31_mini_internet_mpls/base_internet_mpls.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--hosts-per-as N`: number of hosts created in each stub AS. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +This example includes an `example.yaml` manifest for `seedemu.testing`. Run +these commands from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/internet/B31_mini_internet_mpls/example.yaml +python seedemu/testing/cli.py compile examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +python seedemu/testing/cli.py build examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +python seedemu/testing/cli.py up examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +python seedemu/testing/cli.py probe examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +python seedemu/testing/cli.py test examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +python seedemu/testing/cli.py down examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B31_mini_internet_mpls/example.yaml --artifact-dir ci-artifacts/b31 +``` + +The readiness stage checks representative AS2 MPLS border/core routers, +unchanged non-MPLS transit routers, and stub hosts. The probe stage checks that +reachability still works across the mini Internet. The custom `test_runtime.py` +program additionally checks that AS2 routers have MPLS/LDP configuration on the +expected internal links and that an AS3 router does not have MPLS configuration. diff --git a/examples/internet/B31_mini_internet_mpls/example.yaml b/examples/internet/B31_mini_internet_mpls/example.yaml new file mode 100644 index 000000000..bcbd18439 --- /dev/null +++ b/examples/internet/B31_mini_internet_mpls/example.yaml @@ -0,0 +1,85 @@ +id: internet-b31-mini-internet-mpls +name: Mini Internet With MPLS Transit AS +description: B00-style mini Internet where AS2 uses MPLS while the other ASes keep normal routing. +runner: internet +script: mini_internet_mpls.py +platform: amd +features: + - ipv4-default + - mini-internet + - mpls + - ebgp-route-server + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: representative B31 services are running + type: compose-ps + services: + - brdnode_2_r100 + - brdnode_2_r101 + - brdnode_2_r102 + - brdnode_2_r105 + - rnode_2_core_100_101 + - rnode_2_core_101_102 + - rnode_2_core_100_105 + - brdnode_3_r103 + - brdnode_4_r104 + - brdnode_11_r102 + - brdnode_12_r101 + - brdnode_150_router0 + - brdnode_152_router0 + - brdnode_160_router0 + - brdnode_171_router0 + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_160_host_0 + - hnode_171_host_0 + retries: 40 + interval: 3 + +probes: + - name: AS150 reaches AS152 through MPLS-enabled AS2 + type: ping + service: hnode_150_host_0 + target: 10.152.0.71 + count: 3 + retries: 40 + interval: 5 + + - name: AS150 reaches AS160 while non-MPLS AS3 remains unchanged + type: ping + service: hnode_150_host_0 + target: 10.160.0.71 + count: 3 + retries: 40 + interval: 5 + + - name: AS171 reaches AS154 customized host + type: ping + service: hnode_171_host_0 + target: 10.154.0.129 + count: 3 + retries: 40 + interval: 5 + +test_programs: + - name: B31 MPLS runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py b/examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py new file mode 100644 index 000000000..c90ecf2e1 --- /dev/null +++ b/examples/internet/B31_mini_internet_mpls/mini_internet_mpls.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Emulator +from seedemu.layers import Base, Ebgp, Ibgp, Mpls, Ospf, PeerRelationship, Routing +from seedemu.utilities import Makers + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B31 mini Internet MPLS example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def make_as2_mpls_transit(base: Base): + """Create AS2 with the same IX presence as B00 and MPLS-ready core links.""" + + as2 = base.createAutonomousSystem(2) + border_routers = {} + for ix in [100, 101, 102, 105]: + border_routers[ix] = as2.createRouter("r{}".format(ix)) + border_routers[ix].joinNetwork("ix{}".format(ix)) + + for left, right in [(100, 101), (101, 102), (100, 105)]: + core = as2.createRouter("core_{}_{}".format(left, right)) + left_net = "net_{}_core_{}_{}".format(left, left, right) + right_net = "net_core_{}_{}_{}".format(left, right, right) + as2.createNetwork(left_net) + as2.createNetwork(right_net) + border_routers[left].joinNetwork(left_net) + core.joinNetwork(left_net) + core.joinNetwork(right_net) + border_routers[right].joinNetwork(right_net) + + +def build_emulator(hosts_per_as=2) -> Emulator: + emu = Emulator() + ebgp = Ebgp() + base = Base() + mpls = Mpls() + + ############################################################################### + # Create internet exchanges + ix100 = base.createInternetExchange(100) + ix101 = base.createInternetExchange(101) + ix102 = base.createInternetExchange(102) + ix103 = base.createInternetExchange(103) + ix104 = base.createInternetExchange(104) + ix105 = base.createInternetExchange(105) + + # Customize names (for visualization purpose) + ix100.getPeeringLan().setDisplayName("NYC-100") + ix101.getPeeringLan().setDisplayName("San Jose-101") + ix102.getPeeringLan().setDisplayName("Chicago-102") + ix103.getPeeringLan().setDisplayName("Miami-103") + ix104.getPeeringLan().setDisplayName("Boston-104") + ix105.getPeeringLan().setDisplayName("Huston-105") + + ############################################################################### + # Create Transit Autonomous Systems + + ## Tier 1 ASes + make_as2_mpls_transit(base) + + Makers.makeTransitAs( + base, + 3, + [100, 103, 104, 105], + [(100, 103), (100, 105), (103, 105), (103, 104)], + ) + + Makers.makeTransitAs( + base, + 4, + [100, 102, 104], + [(100, 104), (102, 104)], + ) + + ## Tier 2 ASes + Makers.makeTransitAs(base, 11, [102, 105], [(102, 105)]) + Makers.makeTransitAs(base, 12, [101, 104], [(101, 104)]) + + # AS2 keeps the same IX presence and peerings as B00, but its internal + # routing is configured by the MPLS layer instead of regular OSPF/iBGP. + mpls.enableOn(2) + + ############################################################################### + # Create single-homed stub ASes. + Makers.makeStubAsWithHosts(emu, base, 150, 100, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 151, 100, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 152, 101, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 153, 101, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 154, 102, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 160, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 161, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 162, 103, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 163, 104, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 164, 104, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 170, 105, hosts_per_as) + Makers.makeStubAsWithHosts(emu, base, 171, 105, hosts_per_as) + + # An example to show how to add a host with customized IP address. + as154 = base.getAutonomousSystem(154) + new_host = as154.createHost("host_new").joinNetwork("net0", address="10.154.0.129") + from seedemu.core import OptionMode, OptionRegistry + + o = OptionRegistry().sysctl_netipv4_conf_rp_filter( + {"all": False, "default": False, "net0": False}, + mode=OptionMode.RUN_TIME, + ) + new_host.setOption(o) + + o = OptionRegistry().sysctl_netipv4_udp_rmem_min(5000, mode=OptionMode.RUN_TIME) + new_host.setOption(o) + + ############################################################################### + # Peering via route servers. + ebgp.addRsPeers(100, [2, 3, 4]) + ebgp.addRsPeers(102, [2, 4]) + ebgp.addRsPeers(104, [3, 4]) + ebgp.addRsPeers(105, [2, 3]) + + # Private peerings for transit service. + ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) + ebgp.addPrivatePeerings(100, [3], [150], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(101, [12], [152, 153], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) + ebgp.addPrivatePeerings(102, [11], [154], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(103, [3], [160, 161, 162], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(104, [3, 4], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [12], [164], PeerRelationship.Provider) + + ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) + ebgp.addPrivatePeerings(105, [11], [171], PeerRelationship.Provider) + + ############################################################################### + # Add layers to the emulator + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(mpls) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + return emu + + +def run( + dumpfile=None, + hosts_per_as=2, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator(hosts_per_as=hosts_per_as) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + docker = Docker(platform=platform) + emu.compile(docker, output or "./output", override=override) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(output_dir), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B31_mini_internet_mpls/test_runtime.py b/examples/internet/B31_mini_internet_mpls/test_runtime.py new file mode 100644 index 000000000..2f08f9d6c --- /dev/null +++ b/examples/internet/B31_mini_internet_mpls/test_runtime.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + host150 = test.require_service(150, "host_0") + host152 = test.require_service(152, "host_0") + host171 = test.require_service(171, "host_0") + host154_new = test.require_service(154, "host_new") + r100 = test.require_service(2, "r100") + r101 = test.require_service(2, "r101") + r102 = test.require_service(2, "r102") + r105 = test.require_service(2, "r105") + core_100_101 = test.require_service(2, "core_100_101") + as3_r103 = test.require_service(3, "r103") + + if host150 and host152: + test.exec_check("AS150 reaches AS152 through AS2", host150, "ping -c 3 {} >/dev/null".format(host152.address)) + if host171 and host154_new: + test.exec_check("AS171 reaches AS154 customized host", host171, "ping -c 3 {} >/dev/null".format(host154_new.address)) + + if r100: + test.exec_check( + "AS2 r100 has MPLS/LDP enabled on internal links", + r100, + "grep -q '^net_100_core_100_101$' /mpls_ifaces.txt && grep -q '^net_100_core_100_105$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r101: + test.exec_check( + "AS2 r101 has MPLS/LDP enabled on internal links", + r101, + "grep -q '^net_core_100_101_101$' /mpls_ifaces.txt && grep -q '^net_101_core_101_102$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r102: + test.exec_check( + "AS2 r102 has MPLS/LDP enabled on internal links", + r102, + "grep -q '^net_core_101_102_102$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if r105: + test.exec_check( + "AS2 r105 has MPLS/LDP enabled on internal links", + r105, + "grep -q '^net_core_100_105_105$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if core_100_101: + test.exec_check( + "AS2 core router participates in MPLS/LDP only on internal links", + core_100_101, + "grep -q '^net_100_core_100_101$' /mpls_ifaces.txt && grep -q '^net_core_100_101_101$' /mpls_ifaces.txt && grep -q 'mpls ldp' /etc/frr/frr.conf", + ) + if as3_r103: + test.exec_check("AS3 remains a non-MPLS transit AS", as3_r103, "test ! -e /mpls_ifaces.txt") + + test.write_summary("b31-mpls-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B32_mini_internet_exabgp/README.md b/examples/internet/B32_mini_internet_exabgp/README.md new file mode 100644 index 000000000..6a5ea0312 --- /dev/null +++ b/examples/internet/B32_mini_internet_exabgp/README.md @@ -0,0 +1,113 @@ +# Mini Internet With ExaBGP Speaker + +This example is based on `examples/internet/B00_mini_internet`. It keeps the +same mini-Internet topology and adds one external BGP control-plane speaker: + +```text +AS180/exabgp joins IX100 at 10.100.0.180. +AS180 peers with AS2/r100 at 10.100.0.2. +AS180 announces 198.51.100.0/24. +``` + +The purpose is to show how the `ExaBgpService` from +`examples/basic/A13_exabgp` can be attached to a larger Internet emulator. AS2 +continues to use the normal mini-Internet routing stack; ExaBGP is an external +BGP speaker that injects selected routes into AS2. + +## Implementation + +The example first builds the B00 mini Internet: + +```python +emu = mini_internet.build_emulator(hosts_per_as=hosts_per_as) +``` + +It then adds AS180 on IX100 and installs the ExaBGP service: + +```python +as180 = base.createAutonomousSystem(180) +as180.createHost("exabgp").joinNetwork("ix100", address="10.100.0.180") + +exabgp_speaker = exabgp.install("as180_exabgp").setLocalAsn(180) +# Prefer IX-based peering: the service resolves AS2's IX100 border router automatically. +exabgp_speaker.addPeer(ix=100, peer_asn=2, router_relationship="customer") +# Use router-based peering when multiple AS2 routers share IX100 and one must be selected explicitly. +# exabgp_speaker.addPeerByRouter("r100", router_asn=2, router_relationship="customer") +exabgp_speaker.addAnnouncement("198.51.100.0/24") +``` + +The active `addPeer()` call is the preferred form for the mini Internet because +it describes the BGP relationship at the IX level: AS180 peers with AS2 at +IX100. The service resolves the router automatically; in this topology, that +router is `AS2/r100`. The commented `addPeerByRouter()` form is the escape hatch +for topologies where the target AS has multiple routers on the same IX. + +## Manual BGP Updates + +The ExaBGP service creates a manual control FIFO inside the ExaBGP container: + +```text +/run/exabgp/manual.in +``` + +After the emulator is running, send live BGP updates with: + +```sh +sh ./exabgpctl.sh announce 203.0.113.0/24 self +sh ./exabgpctl.sh withdraw 203.0.113.0/24 self +``` + +You can also send raw ExaBGP commands: + +```sh +sh ./exabgpctl.sh command "announce route 203.0.113.0/24 next-hop self" +``` + +To inspect ExaBGP logs: + +```sh +sh ./exabgpctl.sh log +``` + +## Standard Arguments + +```sh +python examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py amd +python examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py --platform amd --output examples/internet/B32_mini_internet_exabgp/output +python examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py --dumpfile examples/internet/B32_mini_internet_exabgp/base_internet_exabgp.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--hosts-per-as N`: number of hosts created in each stub AS. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## TestRunner Lifecycle + +Run these commands from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/internet/B32_mini_internet_exabgp/example.yaml +python seedemu/testing/cli.py compile examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +python seedemu/testing/cli.py build examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +python seedemu/testing/cli.py up examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +python seedemu/testing/cli.py probe examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +python seedemu/testing/cli.py test examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +python seedemu/testing/cli.py down examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/internet/B32_mini_internet_exabgp/example.yaml --artifact-dir ci-artifacts/b32 +``` + +The readiness stage checks representative B00 nodes plus the ExaBGP speaker. +The probe stage checks ExaBGP-to-AS2 reachability and one representative B00 +path. The custom runtime test checks the generated BIRD peering, ExaBGP config, +manual-control FIFO, and AS2's route table for the announced prefix. diff --git a/examples/internet/B32_mini_internet_exabgp/exabgpctl.sh b/examples/internet/B32_mini_internet_exabgp/exabgpctl.sh new file mode 100755 index 000000000..104b36ebe --- /dev/null +++ b/examples/internet/B32_mini_internet_exabgp/exabgpctl.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +COMPOSE_FILE=${COMPOSE_FILE:-"$SCRIPT_DIR/output/docker-compose.yml"} +SERVICE=${EXABGP_SERVICE:-hnode_180_exabgp} + +usage() { + cat <<'EOF' +Usage: + exabgpctl.sh announce PREFIX [NEXT_HOP] + exabgpctl.sh withdraw PREFIX [NEXT_HOP] + exabgpctl.sh command "RAW EXABGP COMMAND" + exabgpctl.sh log + +Examples: + ./exabgpctl.sh announce 203.0.113.0/24 self + ./exabgpctl.sh withdraw 203.0.113.0/24 self + ./exabgpctl.sh command "announce route 203.0.113.0/24 next-hop self" + +Environment: + COMPOSE_FILE Path to docker-compose.yml. Defaults to output/docker-compose.yml. + EXABGP_SERVICE Compose service name. Defaults to hnode_180_exabgp. +EOF +} + +require_compose() { + if [ ! -f "$COMPOSE_FILE" ]; then + echo "compose file not found: $COMPOSE_FILE" >&2 + exit 1 + fi +} + +send_command() { + command_text=$1 + require_compose + printf 'Sending to %s: %s\n' "$SERVICE" "$command_text" + docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE" sh -lc \ + 'test -p /run/exabgp/manual.in && printf "%s\n" "$1" > /run/exabgp/manual.in' \ + sh "$command_text" +} + +case "${1:-}" in + announce|withdraw) + action=$1 + prefix=${2:-} + next_hop=${3:-self} + if [ -z "$prefix" ]; then + usage + exit 1 + fi + send_command "$action route $prefix next-hop $next_hop" + ;; + command) + if [ -z "${2:-}" ]; then + usage + exit 1 + fi + send_command "$2" + ;; + log) + require_compose + docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE" sh -lc \ + 'tail -n 80 /var/log/exabgp/exabgp.log' + ;; + ""|-h|--help|help) + usage + ;; + *) + usage + exit 1 + ;; +esac diff --git a/examples/internet/B32_mini_internet_exabgp/example.yaml b/examples/internet/B32_mini_internet_exabgp/example.yaml new file mode 100644 index 000000000..4c8aa1489 --- /dev/null +++ b/examples/internet/B32_mini_internet_exabgp/example.yaml @@ -0,0 +1,59 @@ +id: internet-b32-mini-internet-exabgp +name: Mini Internet With ExaBGP Speaker +description: B00 mini Internet with an AS180 ExaBGP speaker attached to IX100 and peering with AS2. +runner: internet +script: mini_internet_exabgp.py +platform: amd +features: + - ipv4-default + - mini-internet + - exabgp-service + - ebgp-route-server + - ebgp-private-peering + - ibgp + - ospf + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 600 + +build: + enabled: true + timeout: 1800 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: representative B32 services are running + type: compose-ps + services: + - brdnode_2_r100 + - brdnode_2_r101 + - brdnode_3_r103 + - brdnode_4_r104 + - brdnode_150_router0 + - brdnode_152_router0 + - hnode_150_host_0 + - hnode_152_host_0 + - hnode_180_exabgp + retries: 40 + interval: 3 + +probes: + - name: ExaBGP speaker reaches AS2 router on IX100 + type: ping + service: hnode_180_exabgp + target: 10.100.0.2 + count: 3 + retries: 40 + interval: 5 + +test_programs: + - name: B32 ExaBGP runtime validation + script: test_runtime.py + timeout: 240 diff --git a/examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py b/examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py new file mode 100755 index 000000000..9d70b3600 --- /dev/null +++ b/examples/internet/B32_mini_internet_exabgp/mini_internet_exabgp.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from examples.internet.B00_mini_internet import mini_internet +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base +from seedemu.services import ExaBgpService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the B32 mini Internet with ExaBGP example.") + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--hosts-per-as", type=int, default=2) + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def add_exabgp_speaker(emu: Emulator) -> None: + base: Base = emu.getLayer("Base") + as180 = base.createAutonomousSystem(180) + as180.createHost("exabgp").joinNetwork("ix100", address="10.100.0.180") + + exabgp = ExaBgpService() + exabgp_speaker = exabgp.install("as180_exabgp").setLocalAsn(180) + # Prefer IX-based peering: the service resolves AS2's IX100 border router automatically. + exabgp_speaker.addPeer(ix=100, peer_asn=2, router_relationship="customer") + # Use router-based peering when multiple AS2 routers share IX100 and one must be selected explicitly. + # exabgp_speaker.addPeerByRouter("r100", router_asn=2, router_relationship="customer") + exabgp_speaker.addAnnouncement("198.51.100.0/24") + + emu.addBinding(Binding("as180_exabgp", filter=Filter(asn=180, nodeName="exabgp"))) + emu.addLayer(exabgp) + + +def build_emulator(hosts_per_as: int = 2) -> Emulator: + emu = mini_internet.build_emulator(hosts_per_as=hosts_per_as) + add_exabgp_speaker(emu) + return emu + + +def run( + dumpfile=None, + hosts_per_as=2, + output=None, + platform=Platform.AMD64, + override=True, + render=True, +): + emu = build_emulator(hosts_per_as=hosts_per_as) + if dumpfile is not None: + emu.dump(dumpfile) + return + + if render: + emu.render() + + output_dir = Path(output or SCRIPT_DIR / "output").resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=platform), str(output_dir), override=override) + + +def main() -> int: + args = parse_args() + run( + dumpfile=args.dumpfile, + hosts_per_as=args.hosts_per_as, + output=str(Path(args.output).resolve()), + platform=resolve_platform(args.platform), + override=args.override, + render=args.render, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/B32_mini_internet_exabgp/test_runtime.py b/examples/internet/B32_mini_internet_exabgp/test_runtime.py new file mode 100644 index 000000000..5f5a0f78f --- /dev/null +++ b/examples/internet/B32_mini_internet_exabgp/test_runtime.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + router = test.require_service(2, "r100") + speaker = test.require_service(180, "exabgp") + + if router: + test.exec_check("AS2 r100 starts BIRD", router, "pgrep -x bird >/dev/null") + test.exec_check( + "AS2 r100 peers with AS180 ExaBGP speaker", + router, + "grep -q 'neighbor 10.100.0.180 as 180' /etc/bird/bird.conf", + ) + test.exec_check( + "AS2 r100 receives AS180 announcement", + router, + "birdc show route 198.51.100.0/24 | grep -q '198.51.100.0/24'", + ) + + if speaker: + test.exec_check("ExaBGP speaker process is running", speaker, "pgrep -f 'exabgp /etc/exabgp/exabgp.conf' >/dev/null") + test.exec_check("ExaBGP manual control FIFO is available", speaker, "test -p /run/exabgp/manual.in") + test.exec_check( + "ExaBGP config peers with AS2 r100", + speaker, + "grep -q 'neighbor 10.100.0.2' /etc/exabgp/exabgp.conf", + ) + test.exec_check( + "ExaBGP config enables manual-control process", + speaker, + "grep -q 'processes \\[ manual-control \\]' /etc/exabgp/exabgp.conf", + ) + test.exec_check( + "ExaBGP config announces static IPv4 route", + speaker, + "grep -q 'route 198.51.100.0/24 next-hop self' /etc/exabgp/exabgp.conf", + ) + test.exec_check("ExaBGP speaker reaches AS2 r100", speaker, "ping -c 3 10.100.0.2 >/dev/null") + + test.write_summary("b32-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/internet/b61_k8s_compile/README.md b/examples/internet/b61_k8s_compile/README.md new file mode 100644 index 000000000..9c4095e99 --- /dev/null +++ b/examples/internet/b61_k8s_compile/README.md @@ -0,0 +1,245 @@ +# B61 Native Kubernetes Example + +This example compiles a SeedEMU mini-Internet topology to native Kubernetes +manifests and deploys it with `seedemu.k8sTools`. Kube-OVN/OVS is used as the +secondary network fabric for SeedEMU simulated networks. `k8sTools.py` keeps +this example directory small by copying setup/running resources into temporary +directories while commands run. + +## Files + +| Path | Role | +| --- | --- | +| `mini_internet_k8s.py` | Compiles the SeedEMU topology into `./output` by default. | +| `k8sTools.py` | Local CLI wrapper around `seedemu.k8sTools.K8sTools`. | +| `configKvmOvn.yaml` | Full local KVM + K3s + Kube-OVN example. | +| `configKvmOvnSimply.yaml` | Minimal local KVM template; it is the simplest form of `configKvmOvn.yaml`. | +| `configK3sOvn.yaml` | Existing physical nodes + K3s + Kube-OVN example. | +| `configMultiHostKvmOvn.yaml` | Multi-hypervisor KVM + K3s + Kube-OVN example. | +| `output/` | Generated compiler output: manifests, `images.yaml`, and Docker build contexts. | +| `configK3s.yaml` | Generated by `k8sTools.py build`; consumed by `up` and `destroy`. | +| `kubeconfig.yaml` | Generated by `k8sTools.py build`; consumed by `up`, `down`, and `kubectl`. | + +## Compile + +Run this once before deploying any of the four YAML examples: + +```bash +cd examples/internet/b61_k8s_compile +python3 ./mini_internet_k8s.py +``` + +The default output directory is `./output`. To write elsewhere: + +```bash +python3 ./mini_internet_k8s.py --output-dir /tmp/seedemu-b61-output +``` + +To compile macvlan manifests instead of the default Kube-OVN/OVS manifests: + +```bash +python3 ./mini_internet_k8s.py --cni-type macvlan +``` + +The default compiler mode writes `k8s.kube-ovn.yaml`, `images.yaml`, and one +Docker build context per compiled SeedEMU node. `--cni-type macvlan` writes +`k8s.yaml` with macvlan NetworkAttachmentDefinitions. `k8sTools.py up` also +accepts `images.txt` metadata when present. The default namespace is `seedemu`. + +## Common Deploy Commands + +Every setup mode follows the same command shape: + +```bash +python3 ./k8sTools.py build \ + --input \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml + +python3 ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml +python3 ./k8sTools.py clean -f ./output -k kubeconfig.yaml +python3 ./k8sTools.py destroy -d configK3s.yaml +``` + +`build` creates or prepares infrastructure and writes `configK3s.yaml`, +`kubeconfig.yaml`, and `inventory.yaml`. `up` builds/pushes workload images and +applies the SeedEMU namespace. `clean` removes only the workload namespace. +`destroy` removes the K3s/OVN setup and any KVM infrastructure recorded in +`configK3s.yaml`. + +Add `--keep-temp` to any `k8sTools.py` command when you need to inspect its +temporary setup/running directory after a failure. + +## YAML Choices + +### `configKvmOvnSimply.yaml` + +Use this when you want the shortest local KVM configuration. + +```bash +python3 ./k8sTools.py build \ + --input configKvmOvnSimply.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml +``` + +This file is a minimal template for `kind: kvmOvn`. It only specifies the +cluster name, master/worker resources, worker count, and SSH user/key. The +k8sTools layer fills the normal KVM defaults such as VM names, IP/MAC planning, +registry port, Kube-OVN fabric, output paths, and K3s defaults. Treat it as the +starting point for a new local KVM experiment. + +### `configKvmOvn.yaml` + +Use this for a more explicit local KVM deployment. + +```bash +python3 ./k8sTools.py build \ + --input configKvmOvn.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml +``` + +Compared with `configKvmOvnSimply.yaml`, this file pins VM names, IP ranges, +MAC ranges, registry port, and `fabric.type: ovn`. It also disables the legacy +single base-image path with `kvm.legacyBaseImagePath: ""`. This is the safer +choice when the same libvirt host may already contain other SeedEMU/K3s VMs +and you want predictable naming. + +### `configK3sOvn.yaml` + +Use this when Kubernetes should be built on existing physical machines. + +```bash +python3 ./k8sTools.py build \ + --input configK3sOvn.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml +``` + +Each `nodes[]` entry describes a real machine: `name`, `role`, management `ip`, +optional `connection`, and `ssh.user/key`. A node with `connection: local` runs +setup commands locally; other nodes are reached by SSH. Edit the IPs, users, +and keys before running this on another lab. + +This path can install or reinstall K3s on the listed machines. Confirm SSH +key-based login and `sudo -n true` on every node first. + +### `configMultiHostKvmOvn.yaml` + +Use this when multiple physical machines should act as KVM hypervisors for one +K3s cluster. + +```bash +python3 ./k8sTools.py build \ + --input configMultiHostKvmOvn.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml +``` + +The `hypervisors[]` section describes each physical KVM host. Each hypervisor +gets its own routed libvirt subnet through `routedSubnet`, and `vmCount` +controls how many K3s VMs are created there. `vmSsh` is the SSH account/key +injected into those guest VMs. `master.placement` selects which hypervisor owns +the K3s master. `multiHostKvm.routingTunnel` can create VXLAN next hops between +hypervisors when the underlay does not provide a direct L2 route. + +This mode modifies every listed hypervisor: it may create libvirt networks, +routes, optional VXLAN interfaces, VM disks, cloud-init data, and K3s nodes. + +## Prerequisites + +All modes require Python 3.10+, Docker/buildx, `kubectl`, `ansible-playbook`, +`ssh`, `scp`, and `curl`. + +The KVM modes also require libvirt/KVM tools such as `virsh`, `virt-install`, +`qemu-img`, and a working libvirt network. The physical-node and multi-host +flows require SSH key authentication and non-interactive sudo on all remote +machines. + +Generate a dedicated SSH key if needed: + +```bash +ssh-keygen -t ed25519 -f ~/.ssh/seedemu_k8s -C "seedemu-k8s" +chmod 600 ~/.ssh/seedemu_k8s +ssh-keygen -y -f ~/.ssh/seedemu_k8s > ~/.ssh/seedemu_k8s.pub +``` + +Then update the selected YAML: + +```yaml +ssh: + user: ubuntu + key: ~/.ssh/seedemu_k8s +``` + +For physical machines, install the public key and verify sudo: + +```bash +ssh-copy-id -i ~/.ssh/seedemu_k8s.pub seed@192.0.2.11 +ssh -i ~/.ssh/seedemu_k8s seed@192.0.2.11 'sudo -n true' +``` + +## Runtime Notes + +- `k8sTools.py up` infers the logical compiler image prefix from + `output/images.yaml` or `output/images.txt`; use `--image-registry-prefix` + only for custom outputs. +- During `up`, generated Dockerfile `FROM` images such as + `handsonsecurity/seedemu-base:2.0` and `handsonsecurity/seedemu-router:2.0` + are prepared on the local host first and then loaded into the registry/master + Docker daemon before remote buildx runs. +- `k8sTools.py clean` keeps the cluster infrastructure and only removes the + deployed SeedEMU workload. +- `k8sTools.py destroy` uses metadata embedded in generated `configK3s.yaml`; + do not delete that file before cleanup. +- Real `build`, `up`, `down`, and `destroy` commands modify local or remote + infrastructure. The compile step alone is non-destructive. + +## Quick Kubernetes Debug Commands + +Run these commands from this directory. Use the local stable kubeconfig +explicitly so a stale `KUBECONFIG` from another experiment is not used by +mistake: + +```bash +export B61_NS=seedemu +export B61_KUBECONFIG=./examples/internet/b61_k8s_compile/kubeconfig.yaml +``` + +Check namespace, nodes, and pod status: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" get ns +kubectl --kubeconfig "${B61_KUBECONFIG}" get namespace "${B61_NS}" +kubectl --kubeconfig "${B61_KUBECONFIG}" get nodes -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get deploy -o wide +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" get events --sort-by='.lastTimestamp' +``` + +Inspect or enter one pod/deployment. Replace the object name with a real name +from `kubectl get pods` or `kubectl get deploy`: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" describe pod +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" logs --tail=80 +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it -- bash +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec -it deploy/ -- sh +kubectl --kubeconfig "${B61_KUBECONFIG}" -n "${B61_NS}" exec deploy/ -- birdc show protocols +``` +as1862brd-r13-1.13.7.70-7564dbb49c-9wctj + +Large post-BIRD runs can intermittently return `TLS handshake timeout`, +`Client.Timeout exceeded`, or `http2: client connection lost` while VM CPUs are +saturated. First retry the same command with an explicit request timeout: + +```bash +kubectl --kubeconfig "${B61_KUBECONFIG}" --request-timeout=120s -n "${B61_NS}" get pods -l seedemu.io/role=brd -o wide +``` diff --git a/examples/internet/b61_k8s_compile/SKILL.md b/examples/internet/b61_k8s_compile/SKILL.md new file mode 100644 index 000000000..b6b1839a2 --- /dev/null +++ b/examples/internet/b61_k8s_compile/SKILL.md @@ -0,0 +1,115 @@ +--- +name: seedemu-k8s-native-workflow +description: Use when working with a SeedEMU repository that contains seedemu/k8sTools or examples/internet/b61_k8s_compile, especially to understand, generate, modify, or validate native Kubernetes/K3s deployment workflows for SeedEMU emulations using k8sTools, KVM, physical nodes, multi-host KVM, Multus, Kube-OVN, or OVS. +--- + +# SeedEMU k8sTools Native Kubernetes Workflow + +Use this skill when the task involves building or modifying a SeedEMU +simulation that will run on Kubernetes/K3s through `seedemu.k8sTools`. + +## First Steps + +1. Find the repository root without assuming an absolute path. + - Start from the current working directory. + - Walk upward until a directory contains both `setup.py` and `seedemu/`. + - Prefer a root that also contains `seedemu/k8sTools/README.md`. + - Do not hardcode paths from another user's machine. + +2. Read the local documentation before making architectural claims. + - `seedemu/k8sTools/README.md` + - `seedemu/k8sTools/resources/setup/README.md` + - `examples/internet/b61_k8s_compile/README.md` + +3. Inspect the active example files. + - `examples/internet/b61_k8s_compile/mini_internet_k8s.py` + - `examples/internet/b61_k8s_compile/k8sTools.py` + - `examples/internet/b61_k8s_compile/configKvmOvn.yaml` + - `examples/internet/b61_k8s_compile/configK3sOvn.yaml` + - `examples/internet/b61_k8s_compile/configMultiHostKvmOvn.yaml` + +Avoid loading generated `output/`, `configK3s.yaml`, or `kubeconfig.yaml` +unless the task specifically concerns runtime artifacts. + +## Architecture Model + +Think of the workflow as three separate contracts: + +1. Compile contract: + - `KubernetesCompiler` turns a rendered SeedEMU emulator into + Kubernetes manifests and Docker build contexts. + - The B61 example writes compiler output to `./output` by default. + - Important files are `k8s.kube-ovn.yaml` or `k8s.yaml`, `images.yaml`, + per-node Docker build contexts, and sometimes `base_images/`. + - The compile stage does not require a live cluster inventory. + +2. Setup contract: + - `K8sTools.build()` reads one user input YAML with explicit `kind`. + - Supported kinds are `kvmOvn`, `physicalOvn`, and `multiHostKvmOvn`. + - It copies packaged `resources/setup/` into a temporary directory, runs + the setup entrypoints there, and writes user-visible `configK3s.yaml` and + `kubeconfig.yaml`. + - KVM destroy state is embedded into generated `configK3s.yaml`. + +3. Running contract: + - `K8sTools.up()` copies packaged `resources/running/` into a temporary + directory, runs preflight, builds/pushes images, renders + `kustomization.yaml`, applies the workload, and waits for readiness. + - `K8sTools.down()` removes workload resources and the namespace. + - `K8sTools.destroy()` removes K3s/OVN and optional KVM infrastructure + recorded in `configK3s.yaml`. + +## Preferred Commands + +From `examples/internet/b61_k8s_compile`: + +```bash +python3 ./mini_internet_k8s.py +python3 ./k8sTools.py build \ + --input configKvmOvn.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml +python3 ./k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml +python3 ./k8sTools.py down -f ./output -k kubeconfig.yaml +python3 ./k8sTools.py destroy -d configK3s.yaml +``` + +Use `--input configK3sOvn.yaml` for existing physical machines, or +`--input configMultiHostKvmOvn.yaml` for KVM VMs distributed across multiple +physical hypervisors. + +All `k8sTools.py` commands accept `--keep-temp` when the temporary copied +setup/running resources need to be inspected after a failed run. + +## Path Rules + +- Never write a developer machine's absolute path into docs, skills, or source + examples. +- Use repository-relative paths in documentation. +- In Python examples, discover the repository root by walking upward from + `Path(__file__).resolve()`. +- Treat `output/`, `configK3s.yaml`, `configK3sTemplate.yaml`, `kubeconfig.yaml`, + image caches, cloud-init files, and KVM state files as generated artifacts. +- If a YAML field points to an SSH key, prefer `~/.ssh/` rather than an + absolute `/home//...` path. + +## Validation + +Use non-destructive checks first: + +```bash +python3 -m py_compile \ + seedemu/k8sTools/*.py \ + seedemu/k8sTools/resources/setup/*.py \ + seedemu/k8sTools/resources/setup/kvm/*.py \ + seedemu/k8sTools/resources/setup/multiHostKvm/*.py \ + seedemu/k8sTools/resources/setup/ovn/*.py \ + seedemu/k8sTools/resources/setup/vxlan/*.py \ + seedemu/k8sTools/resources/running/*.py +for f in $(rg --files seedemu/k8sTools/resources -g '*.sh'); do bash -n "$f"; done +python3 examples/internet/b61_k8s_compile/k8sTools.py --help +``` + +Real `build`, `up`, `down`, and `destroy` commands create or modify KVM VMs, +K3s clusters, registries, images, Kubernetes namespaces, or physical nodes. +State those side effects before running them. diff --git a/examples/internet/b61_k8s_compile/configK3sOvn.yaml b/examples/internet/b61_k8s_compile/configK3sOvn.yaml new file mode 100644 index 000000000..cb508a872 --- /dev/null +++ b/examples/internet/b61_k8s_compile/configK3sOvn.yaml @@ -0,0 +1,23 @@ +kind: physicalOvn +clusterName: seedemu-b61-physical + +nodes: + - name: amd + role: master + ip: 10.202.236.88 + connection: local + ssh: + user: lxl + key: ~/.ssh/id_ed25519 + - name: idc + role: worker + ip: 10.202.191.39 + ssh: + user: seed + key: ~/.ssh/id_ed25519 + +fabric: + type: ovn + +registry: + port: 5000 diff --git a/examples/internet/b61_k8s_compile/configKvmOvn.yaml b/examples/internet/b61_k8s_compile/configKvmOvn.yaml new file mode 100644 index 000000000..de074e52c --- /dev/null +++ b/examples/internet/b61_k8s_compile/configKvmOvn.yaml @@ -0,0 +1,36 @@ +kind: kvmOvn +clusterName: seedemu-b61-kvm + +defaults: + masterName: seedemu-b61-kvm-master + workerNamePrefix: seedemu-b61-kvm-worker + ipPrefix: 192.168.122 + masterIpStart: 200 + workerIpStart: 201 + macPrefix: 52:54:00:61:10 + masterMacStart: 200 + workerMacStart: 201 + +master: + vcpus: 12 + memoryMb: 10240 + diskGb: 80 + +workers: + count: 2 + vcpus: 6 + memoryMb: 10240 + diskGb: 80 + +kvm: + legacyBaseImagePath: "" + +ssh: + user: ubuntu + key: ~/.ssh/id_ed25519 + +fabric: + type: ovn + +registry: + port: 5000 diff --git a/examples/internet/b61_k8s_compile/configKvmOvnSimply.yaml b/examples/internet/b61_k8s_compile/configKvmOvnSimply.yaml new file mode 100644 index 000000000..5cac0ee35 --- /dev/null +++ b/examples/internet/b61_k8s_compile/configKvmOvnSimply.yaml @@ -0,0 +1,17 @@ +kind: kvmOvn +clusterName: seedemu-k3s + +master: + vcpus: 12 + memoryMb: 10240 + diskGb: 80 + +workers: + count: 2 + vcpus: 6 + memoryMb: 10240 + diskGb: 80 + +ssh: + user: ubuntu + key: ~/.ssh/id_ed25519 diff --git a/examples/internet/b61_k8s_compile/configMultiHostKvmOvn.yaml b/examples/internet/b61_k8s_compile/configMultiHostKvmOvn.yaml new file mode 100644 index 000000000..e70d5e015 --- /dev/null +++ b/examples/internet/b61_k8s_compile/configMultiHostKvmOvn.yaml @@ -0,0 +1,59 @@ +kind: multiHostKvmOvn +clusterName: seedemu-k3s + +hypervisors: + - name: amd + ip: 10.202.236.88 + connection: local + ssh: + user: lxl + key: ~/.ssh/id_ed25519 + routedSubnet: + cidr: 10.80.1.0/24 + gateway: 10.80.1.1 + networkName: seedemu-amd + bridgeName: virbr-seed1 + vmIpStart: 10 + vmCount: 3 + + - name: idc + ip: 10.202.191.39 + ssh: + user: seed + key: ~/.ssh/id_ed25519 + routedSubnet: + cidr: 10.80.2.0/24 + gateway: 10.80.2.1 + networkName: seedemu-idc + bridgeName: virbr-seed2 + vmIpStart: 10 + vmCount: 3 + +vmSsh: + user: ubuntu + key: ~/.ssh/id_ed25519 + +multiHostKvm: + routingTunnel: + type: vxlan + namePrefix: vxkvm + cidr: 10.255.80.0/24 + vniBase: 4280 + dstPort: 4790 + +master: + placement: amd + vcpus: 16 + memoryMb: 32768 + diskGb: 120 + +workers: + vcpus: 4 + memoryMb: 16384 + diskGb: 80 + +fabric: + type: ovn + +registry: + port: 5000 diff --git a/examples/internet/b61_k8s_compile/k8sTools.py b/examples/internet/b61_k8s_compile/k8sTools.py new file mode 100644 index 000000000..c7aff8137 --- /dev/null +++ b/examples/internet/b61_k8s_compile/k8sTools.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Run the b61 SeedEMU Kubernetes workflow through seedemu.k8sTools. + +This file is intentionally a thin example-local entrypoint. The reusable +implementation lives in seedemu.k8sTools.K8sTools. +""" +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[3] +repo_root = str(REPO_ROOT) +if repo_root in sys.path: + sys.path.remove(repo_root) +sys.path.insert(0, repo_root) +from seedemu.k8sTools import K8sTools + + +if __name__ == "__main__": + raise SystemExit(K8sTools().runCli()) diff --git a/examples/internet/b61_k8s_compile/mini_internet_k8s.py b/examples/internet/b61_k8s_compile/mini_internet_k8s.py new file mode 100644 index 000000000..243ca8868 --- /dev/null +++ b/examples/internet/b61_k8s_compile/mini_internet_k8s.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Compile a small SeedEMU Internet topology to native Kubernetes manifests. + +Outputs are written to ./output by default: +- k8s.kube-ovn.yaml: Kubernetes namespace, Kube-OVN Vpc/Subnet, + NetworkAttachmentDefinitions, Deployments. +- images.yaml: image build contexts consumed by the generated running/ stage. +- per-node Docker build contexts and optional base_images/ contexts. +Default CNI is kube-ovn. Use --cni-type macvlan for macvlan manifests. +""" + +from __future__ import annotations +import argparse +import sys +from pathlib import Path +import os + +SCRIPT_DIR = Path(__file__).resolve().parent + + +def findRepoRoot(start: Path) -> Path: + """Find the SeedEMU source tree that contains this example.""" + for candidate in (start, *start.parents): + if (candidate / "seedemu").is_dir() and (candidate / "setup.py").is_file(): + return candidate + raise RuntimeError(f"Cannot find SeedEMU repository root above {start}") + + +REPO_ROOT = findRepoRoot(SCRIPT_DIR) +sys.path = [path for path in sys.path if path != str(REPO_ROOT)] +sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.core import Emulator +from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf, PeerRelationship +from seedemu.utilities import Makers +from seedemu.compiler import Platform + +from seedemu.compiler import KubernetesCompiler + + +HOSTS_PER_AS = 2 +OUTPUT_DIR = Path("./output") + + +def parse_args() -> argparse.Namespace: + """Parse compile options for the B61 native Kubernetes example.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=OUTPUT_DIR, + help="Output directory for Kubernetes manifests and Docker build contexts.", + ) + parser.add_argument( + "--platform", + choices=("amd64", "arm64"), + default="amd64", + help="Target platform for Docker build contexts.", + ) + parser.add_argument( + "--cni-type", + choices=("kube-ovn", "ovn", "macvlan"), + default="kube-ovn", + help="Secondary CNI backend for SeedEMU links. Defaults to kube-ovn.", + ) + return parser.parse_args() + + +def build_mini_internet(hosts_per_as: int) -> Emulator: + emu = Emulator() + ebgp = Ebgp() + base = Base() + + ix100 = base.createInternetExchange(100) + ix101 = base.createInternetExchange(101) + ix102 = base.createInternetExchange(102) + ix103 = base.createInternetExchange(103) + ix104 = base.createInternetExchange(104) + ix105 = base.createInternetExchange(105) + + ix100.getPeeringLan().setDisplayName("NYC-100") + ix101.getPeeringLan().setDisplayName("San Jose-101") + ix102.getPeeringLan().setDisplayName("Chicago-102") + ix103.getPeeringLan().setDisplayName("Miami-103") + ix104.getPeeringLan().setDisplayName("Boston-104") + ix105.getPeeringLan().setDisplayName("Houston-105") + + Makers.makeTransitAs(base, 2, [100, 101, 102, 105], [(100, 101), (101, 102), (100, 105)]) + Makers.makeTransitAs(base, 3, [100, 103, 104, 105], [(100, 103), (100, 105), (103, 105), (103, 104)]) + Makers.makeTransitAs(base, 4, [100, 102, 104], [(100, 104), (102, 104)]) + Makers.makeTransitAs(base, 11, [102, 105], [(102, 105)]) + Makers.makeTransitAs(base, 12, [101, 104], [(101, 104)]) + + for asn, ix in [ + (150, 100), (151, 100), (152, 101), (153, 101), (154, 102), + (160, 103), (161, 103), (162, 103), (163, 104), (164, 104), + (170, 105), (171, 105), + ]: + Makers.makeStubAsWithHosts(emu, base, asn, ix, hosts_per_as) + + ebgp.addRsPeers(100, [2, 3, 4]) + ebgp.addRsPeers(102, [2, 4]) + ebgp.addRsPeers(104, [3, 4]) + ebgp.addRsPeers(105, [2, 3]) + + ebgp.addPrivatePeerings(100, [2], [150, 151], PeerRelationship.Provider) + ebgp.addPrivatePeerings(100, [3], [150], PeerRelationship.Provider) + ebgp.addPrivatePeerings(101, [2], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(101, [12], [152, 153], PeerRelationship.Provider) + ebgp.addPrivatePeerings(102, [2, 4], [11, 154], PeerRelationship.Provider) + ebgp.addPrivatePeerings(102, [11], [154], PeerRelationship.Provider) + ebgp.addPrivatePeerings(103, [3], [160, 161, 162], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [3, 4], [12], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [4], [163], PeerRelationship.Provider) + ebgp.addPrivatePeerings(104, [12], [164], PeerRelationship.Provider) + ebgp.addPrivatePeerings(105, [3], [11, 170], PeerRelationship.Provider) + ebgp.addPrivatePeerings(105, [11], [171], PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(Routing()) + emu.addLayer(ebgp) + emu.addLayer(Ibgp()) + emu.addLayer(Ospf()) + emu.render() + return emu + + +def main() -> int: + args = parse_args() + + os.chdir(SCRIPT_DIR) + output_dir = args.output_dir.expanduser() + if not output_dir.is_absolute(): + output_dir = (SCRIPT_DIR / output_dir).resolve() + platform = Platform.AMD64 if args.platform == "amd64" else Platform.ARM64 + + compiler = KubernetesCompiler(platform=platform, cni_type=args.cni_type) + + emu = build_mini_internet(hosts_per_as=HOSTS_PER_AS) + emu.compile(compiler, str(output_dir), override=True) + + print("=" * 72) + print("Native Kubernetes baseline compilation complete.") + print("=" * 72) + print("Config file: not used") + print(f"Output directory: {output_dir}") + print("Image registry prefix: seedemu") + print("Namespace: seedemu") + print(f"CNI type: {args.cni_type}") + print("Runtime workflow: seedemu.k8sTools") + print("Inventory required for compile: no") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sample/README.md b/examples/sample/README.md new file mode 100644 index 000000000..5a0eb51c3 --- /dev/null +++ b/examples/sample/README.md @@ -0,0 +1,145 @@ +# SEED Emulator Test Runner Sample + +This folder demonstrates the standardized testing lifecycle for a real SEED +Emulator example. The topology is intentionally close to +`examples/basic/A00_simple_as`: three autonomous systems share Internet Exchange +`IX100`, each AS has one router and one web host, and eBGP route-server peerings +provide cross-AS reachability. + +The purpose of this folder is not to introduce a new network scenario. It is a +small reference example for a lifecycle that CI, agents, and developers can +execute in the same way. + +## Files + +- `sample_example.py`: standardized SEED Emulator example entrypoint. +- `example.yaml`: test manifest consumed by the runner. +- `test_runtime.py`: custom runtime test program for checks that are easier to + express in Python than YAML. +- `output/`: generated Docker compiler output, removed by the clean command. + +## Standard Arguments + +The example accepts both the legacy platform argument and the newer named +arguments: + +```sh +python examples/sample/sample_example.py amd +python examples/sample/sample_example.py --platform amd --output examples/sample/output +python examples/sample/sample_example.py --dumpfile examples/sample/sample.bin +``` + +Supported arguments: + +- `amd|arm`: optional legacy platform argument. +- `--platform amd|arm`: named platform argument. +- `--output PATH`: output folder for Docker compiler results. +- `--dumpfile PATH`: save a serialized emulator instead of compiling Docker output. +- `--override` / `--no-override`: control whether existing output is replaced. +- `--skip-render`: compile without calling `emu.render()` first. + +## Runner + +Use the new testing runner from the repository root: + +```sh +python seedemu/testing/cli.py clean examples/sample/example.yaml +python seedemu/testing/cli.py compile examples/sample/example.yaml --artifact-dir ci-artifacts/sample +python seedemu/testing/cli.py build examples/sample/example.yaml --artifact-dir ci-artifacts/sample +python seedemu/testing/cli.py up examples/sample/example.yaml --artifact-dir ci-artifacts/sample +python seedemu/testing/cli.py probe examples/sample/example.yaml --artifact-dir ci-artifacts/sample +python seedemu/testing/cli.py test examples/sample/example.yaml --artifact-dir ci-artifacts/sample +python seedemu/testing/cli.py down examples/sample/example.yaml --artifact-dir ci-artifacts/sample +``` + +The full lifecycle can also be run with: + +```sh +python seedemu/testing/cli.py all examples/sample/example.yaml --artifact-dir ci-artifacts/sample +``` + +The manifest declares `runner: internet` because it includes Internet-style +probes such as `ping`. + +## What The Runner Checks + +The compile stage verifies: + +```text +output/docker-compose.yml +``` + +The readiness stage checks that the generated Docker Compose services for the +three web hosts and three routers are running. + +The probe stage performs declarative cross-AS reachability checks: + +- AS151 web host fetches the AS150 web service. +- AS152 web host fetches the AS151 web service. +- AS150 web host pings the AS152 web host. + +Declarative probes are useful when the success condition is simple and should be +visible directly in `example.yaml`. + +The test stage runs custom programs listed in `test_programs`. This sample uses +`test_runtime.py` to demonstrate the standard custom runtime-test design. + +Custom runtime tests should use `ComposeRuntimeTest` instead of duplicating +Docker Compose execution, YAML parsing, retry logic, artifact writing, or +environment-variable handling: + +```python +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + web150 = test.require_service(150, "web") + web151 = test.require_service(151, "web") + + if web150 and web151: + test.exec_check( + "AS151 fetches AS150 web service", + web151, + "curl -fsS http://{} >/dev/null".format(web150.address), + ) + + test.write_summary("sample-custom-runtime-test.json") + return test.exit_code() +``` + +Use `require_service()` for structural checks based on SEED Emulator Compose +labels, and use `exec_check()` for runtime checks that should retry while +services and routes converge. Keep the test program focused on the +example-specific intent: which nodes should exist and which behavior should be +verified. + +A custom test program is useful when the success condition needs dynamic service +discovery, loops, tables, richer parsing, or domain-specific logic that would +make the YAML hard to read. + +When the new `TestRunner` starts a test program, it provides these environment +variables: + +- `TEST_RUNNER_NAME`: runner type, such as `internet`. +- `TEST_RUNNER_EMULATION_ID`: stable ID from `example.yaml`. +- `TEST_RUNNER_EMULATION_DIR`: absolute path to this example folder. +- `TEST_RUNNER_MANIFEST`: absolute path to `example.yaml`. +- `TEST_RUNNER_COMPOSE_FILE`: absolute path to the generated compose file. +- `TEST_RUNNER_ARTIFACT_DIR`: artifact folder, if one was provided. + +The runner also provides the older `EXAMPLE_RUNNER_*` names for compatibility +with existing custom test programs. + +The custom test exits with `0` on success and nonzero on failure. Its stdout and +stderr are captured under the artifact directory, `ComposeRuntimeTest` writes +the example-specific JSON summary, and the runner also writes +`test-summary.json`. + +## Notes + +The service names in `example.yaml` are Docker Compose service names generated +by the SEED Emulator Docker compiler, such as `hnode_151_web` for a host and +`brdnode_151_router0` for an IX-connected border router. The container names +include IP addresses, but `docker compose exec` uses the service names. diff --git a/examples/sample/example.yaml b/examples/sample/example.yaml new file mode 100644 index 000000000..5eb06ca51 --- /dev/null +++ b/examples/sample/example.yaml @@ -0,0 +1,69 @@ +id: sample-example-runner +name: TestRunner SEED Emulator Sample +description: A small A00-style SEED Emulator topology used to demonstrate TestRunner lifecycle commands. +runner: internet +script: sample_example.py +platform: amd +features: + - example-runner + - ipv4-default + - web-service + - ebgp-route-server + +compile: + enabled: true + output: output + clean: + - output + expected: + - output/docker-compose.yml + timeout: 300 + +build: + enabled: true + timeout: 1200 + +runtime: + compose: output/docker-compose.yml + readiness: + - name: sample emulator services are running + type: compose-ps + services: + - hnode_150_web + - hnode_151_web + - hnode_152_web + - brdnode_150_router0 + - brdnode_151_router0 + - brdnode_152_router0 + retries: 30 + interval: 2 + +probes: + - name: AS151 web host reaches AS150 web service + type: exec + service: hnode_151_web + command: curl -fsS http://10.150.0.71 + expect_exit: 0 + retries: 20 + interval: 3 + + - name: AS152 web host reaches AS151 web service + type: exec + service: hnode_152_web + command: curl -fsS http://10.151.0.71 + expect_exit: 0 + retries: 20 + interval: 3 + + - name: AS150 can ping AS152 web host + type: ping + service: hnode_150_web + target: 10.152.0.71 + count: 3 + retries: 10 + interval: 3 + +test_programs: + - name: sample custom runtime validation + script: test_runtime.py + timeout: 180 diff --git a/examples/sample/sample_example.py b/examples/sample/sample_example.py new file mode 100644 index 000000000..2c932dfc8 --- /dev/null +++ b/examples/sample/sample_example.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from seedemu.compiler import Docker, Platform +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Routing +from seedemu.services import WebService + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build a small SEED Emulator topology for TestRunner." + ) + parser.add_argument("legacy_platform", nargs="?", choices=["amd", "arm"]) + parser.add_argument("--platform", choices=["amd", "arm"]) + parser.add_argument("--output", default=str(SCRIPT_DIR / "output")) + parser.add_argument("--dumpfile") + parser.add_argument("--override", dest="override", action="store_true", default=True) + parser.add_argument("--no-override", dest="override", action="store_false") + parser.add_argument("--skip-render", dest="render", action="store_false", default=True) + args = parser.parse_args() + args.platform = args.platform or args.legacy_platform or "amd" + return args + + +def resolve_platform(name: str) -> Platform: + return Platform.AMD64 if name == "amd" else Platform.ARM64 + + +def build_emulator() -> Emulator: + emu = Emulator() + + base = Base() + routing = Routing() + ebgp = Ebgp() + web = WebService() + + base.createInternetExchange(100) + + for asn in [150, 151, 152]: + current_as = base.createAutonomousSystem(asn) + current_as.createNetwork("net0") + current_as.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + current_as.createHost("web").joinNetwork("net0") + + vnode = "web{}".format(asn) + web.install(vnode) + emu.addBinding(Binding(vnode, filter=Filter(nodeName="web", asn=asn))) + ebgp.addRsPeer(100, asn) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(web) + return emu + + +def main() -> int: + args = parse_args() + emu = build_emulator() + + if args.dumpfile: + emu.dump(args.dumpfile) + print("Saved sample emulator to {}".format(args.dumpfile)) + return 0 + + if args.render: + emu.render() + + output_dir = Path(args.output).resolve() + output_dir.parent.mkdir(parents=True, exist_ok=True) + emu.compile(Docker(platform=resolve_platform(args.platform)), str(output_dir), override=args.override) + print("Generated sample SEED Emulator Docker output in {}".format(output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sample/skill.md b/examples/sample/skill.md new file mode 100644 index 000000000..a7409ab2e --- /dev/null +++ b/examples/sample/skill.md @@ -0,0 +1,8 @@ + + +# IP address convention + +- If we don't assign IP address to host/router, the emulator will automatically assign one +- Don't assign .1 address to any host or router, because this address is reserved by Docker + + diff --git a/examples/sample/test_runtime.py b/examples/sample/test_runtime.py new file mode 100644 index 000000000..729b84c39 --- /dev/null +++ b/examples/sample/test_runtime.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from seedemu.testing import ComposeRuntimeTest + + +def main() -> int: + test = ComposeRuntimeTest(__file__) + + # Discover generated services by stable SEED Emulator labels. This keeps the + # test independent from Docker Compose service-name details. + web150 = test.require_service(150, "web") + web151 = test.require_service(151, "web") + web152 = test.require_service(152, "web") + test.require_service(150, "router0") + test.require_service(151, "router0") + test.require_service(152, "router0") + + if web150 and web151 and web152: + test.exec_check("AS150 web service is ready", web150, "curl -fsS http://127.0.0.1 >/dev/null") + test.exec_check("AS151 web service is ready", web151, "curl -fsS http://127.0.0.1 >/dev/null") + test.exec_check("AS152 web service is ready", web152, "curl -fsS http://127.0.0.1 >/dev/null") + test.exec_check( + "AS151 fetches AS150 web service", + web151, + "curl -fsS http://{} >/dev/null".format(web150.address), + ) + test.exec_check( + "AS152 fetches AS151 web service", + web152, + "curl -fsS http://{} >/dev/null".format(web151.address), + ) + test.exec_check( + "AS150 reaches AS152 by ICMP", + web150, + "ping -c 3 {} >/dev/null".format(web152.address), + ) + + test.write_summary("sample-custom-runtime-test.json") + return test.exit_code() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/yesterday_once_more/Y03_mirai/demo/mirai.ipynb b/examples/yesterday_once_more/Y03_mirai/demo/mirai.ipynb index a0863b75c..371f2c1a9 100644 --- a/examples/yesterday_once_more/Y03_mirai/demo/mirai.ipynb +++ b/examples/yesterday_once_more/Y03_mirai/demo/mirai.ipynb @@ -87,7 +87,7 @@ "\n", "The simulation environment provides a web-based visualization interface for real-time monitoring of the network topology and data flow.\n", "\n", - "Please open the following address in your web browser: [http://localhost:8080/map.html](http://localhost:8080/map.html)\n", + "Please open the following address in your web browser: [http://localhost:8080/pro/map](http://localhost:8080/pro/map)\n", "\n", "Through this interactive map, you can visually observe the worm's propagation path and the traffic flow of the DDoS attack in the subsequent steps." ] @@ -136,7 +136,7 @@ "\n", "Once the C2 service is ready, the worm can be released. We will start from the C2 server itself by running the `mirai.py` script. It will act as the first infected node, scanning the network for other devices with weak Telnet credentials and implanting copies of itself.\n", "\n", - "**Preparation for Observation**: To clearly observe the propagation process, please first go to the [network map](http://localhost:8080/map.html) and enter `dst 10.170.0.100` in the filter box, then press Enter. This will highlight all traffic directed to the C2 server (IP: `10.170.0.100`).\n", + "**Preparation for Observation**: To clearly observe the propagation process, please first go to the [network map](http://localhost:8080/pro/map) and enter `dst 10.170.0.100` in the filter box, then press Enter. This will highlight all traffic directed to the C2 server (IP: `10.170.0.100`).\n", "\n", "Now, execute the code cell below to start the worm." ] @@ -162,7 +162,7 @@ "source": [ "### 3.3 Observing the Propagation Process\n", "\n", - "After executing the previous step, the worm has started to spread. Please switch to the browser window with the [network map](http://localhost:8080/map.html) to observe.\n", + "After executing the previous step, the worm has started to spread. Please switch to the browser window with the [network map](http://localhost:8080/pro/map) to observe.\n", "\n", "You will see a continuous stream of new nodes initiating connections to the C2 server, which indicates that these nodes are downloading the `mirai.py` worm script. The traffic dynamics of the entire propagation process will be visually presented before you.\n", "\n", @@ -267,7 +267,7 @@ "\n", "\n", "**Observe on the Visualization Map**\n", - "At the same time, on the [network map](http://localhost:8080/map.html), set the filter to `dst 10.170.0.99`. You will see a large volume of packets flocking from various bot nodes to the victim host, forming a dense traffic storm." + "At the same time, on the [network map](http://localhost:8080/pro/map), set the filter to `dst 10.170.0.99`. You will see a large volume of packets flocking from various bot nodes to the victim host, forming a dense traffic storm." ] }, { diff --git a/examples/yesterday_once_more/Y03_mirai/demo_with_proxmox_vm/mirai.ipynb b/examples/yesterday_once_more/Y03_mirai/demo_with_proxmox_vm/mirai.ipynb index 55bbde5c3..2fcac7e4b 100644 --- a/examples/yesterday_once_more/Y03_mirai/demo_with_proxmox_vm/mirai.ipynb +++ b/examples/yesterday_once_more/Y03_mirai/demo_with_proxmox_vm/mirai.ipynb @@ -87,7 +87,7 @@ "\n", "The simulation environment provides a web-based visualization interface for real-time monitoring of the network topology and data flow.\n", "\n", - "Please open the following address in your web browser: [http://localhost:8080/map.html](http://localhost:8080/map.html)\n", + "Please open the following address in your web browser: [http://localhost:8080/pro/map](http://localhost:8080/pro/map)\n", "\n", "Through this interactive map, you can visually observe the worm's propagation path and the traffic flow of the DDoS attack in the subsequent steps." ] @@ -136,7 +136,7 @@ "\n", "Once the C2 service is ready, the worm can be released. We will start from the C2 server itself by running the `mirai.py` script. It will act as the first infected node, scanning the network for other devices with weak Telnet credentials and implanting copies of itself.\n", "\n", - "**Preparation for Observation**: To clearly observe the propagation process, please first go to the [network map](http://localhost:8080/map.html) and enter `dst 10.170.0.100` in the filter box, then press Enter. This will highlight all traffic directed to the C2 server (IP: `10.170.0.100`).\n", + "**Preparation for Observation**: To clearly observe the propagation process, please first go to the [network map](http://localhost:8080/pro/map) and enter `dst 10.170.0.100` in the filter box, then press Enter. This will highlight all traffic directed to the C2 server (IP: `10.170.0.100`).\n", "\n", "Now, execute the code cell below to start the worm." ] @@ -162,7 +162,7 @@ "source": [ "### 3.3 Observing the Propagation Process\n", "\n", - "After executing the previous step, the worm has started to spread. Please switch to the browser window with the [network map](http://localhost:8080/map.html) to observe.\n", + "After executing the previous step, the worm has started to spread. Please switch to the browser window with the [network map](http://localhost:8080/pro/map) to observe.\n", "\n", "You will see a continuous stream of new nodes initiating connections to the C2 server, which indicates that these nodes are downloading the `mirai.py` worm script. The traffic dynamics of the entire propagation process will be visually presented before you.\n", "\n", @@ -269,7 +269,7 @@ "\n", "\n", "**Observe on the Visualization Map**\n", - "At the same time, on the [network map](http://localhost:8080/map.html), set the filter to `dst 10.170.0.99`. You will see a large volume of packets flocking from various bot nodes to the victim host, forming a dense traffic storm." + "At the same time, on the [network map](http://localhost:8080/pro/map), set the filter to `dst 10.170.0.99`. You will see a large volume of packets flocking from various bot nodes to the victim host, forming a dense traffic storm." ] }, { diff --git a/experiments/blockchain/testing/batch_docker_compose_build.sh b/experiments/blockchain/testing/batch_docker_compose_build.sh new file mode 100755 index 000000000..0962a7dcb --- /dev/null +++ b/experiments/blockchain/testing/batch_docker_compose_build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Disable Docker BuildKit to avoid multi-thread conflicts +export DOCKER_BUILDKIT=0 + +# =============================================== +# Batch docker-compose build script +# Default batch size: 50 services per batch +# Includes logging, retry on failure, and detailed output +# =============================================== + +BATCH_SIZE=50 +LOG_FILE="batch_compose_build.log" + +echo "===== Executing docker compose build in batches =====" | tee "$LOG_FILE" +echo "Batch size: $BATCH_SIZE" | tee -a "$LOG_FILE" +echo "Start time: $(date)" | tee -a "$LOG_FILE" + +# Retrieve all service names from docker-compose.yml +services=($(docker compose config --services)) +total=${#services[@]} +echo "Total number of services: $total" | tee -a "$LOG_FILE" + +if [ $total -eq 0 ]; then + echo "? No services found in docker-compose.yml. Exiting script." | tee -a "$LOG_FILE" + exit 1 +fi + +batch_count=0 + +# =============================================== +# Batch build +# =============================================== +for ((i=0; i last_block: + now = time.time() + diff = now - last_time + normal=(f"?? New block detected: {current_block} | Interval: {diff:.2f}s") + print(normal) + log_message(normal,log_file) + if diff > args.threshold: + warning = f"?? WARNING, Block interval too long ({diff:.2f}s) at block {current_block}" + print(warning) + log_message(warning, log_file) + + last_block = current_block + last_time = now + + time.sleep(args.interval) + +if __name__ == "__main__": + main() diff --git a/experiments/blockchain/testing/check_block_cpu_mem.py b/experiments/blockchain/testing/check_block_cpu_mem.py new file mode 100755 index 000000000..c0dd6814d --- /dev/null +++ b/experiments/blockchain/testing/check_block_cpu_mem.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import sys +import time +import datetime +from web3 import Web3 +import psutil +import urllib.parse + +# ------------------------- +# Check args +# ------------------------- +if len(sys.argv) < 2: + print("Usage: python3 check_block_cpu_mem.py ") + print("Example: python3 check_block_cpu_mem.py http://ip:8545") + sys.exit(1) + +rpc_url = sys.argv[1] + +# ------------------------- +# Log file initialization +# ------------------------- +safe_rpc = urllib.parse.quote_plus(rpc_url) +timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") +filename = f"system_info-{safe_rpc}-{timestamp}.log" + +with open(filename, "a") as f: + f.write("time, block_number, mem_usage(GB), cpu_usage(%)\n") + +print(f"[INFO] Logging to {filename}") +print(f"[INFO] RPC URL: {rpc_url}") + +# ------------------------- +# Connect Web3 +# ------------------------- +web3 = Web3(Web3.HTTPProvider(rpc_url)) +if not web3.isConnected(): + print(f"[ERROR] Cannot connect to RPC URL: {rpc_url}") + sys.exit(1) + +print("[INFO] Successfully connected to Geth RPC") + +# ------------------------- +# Functions +# ------------------------- +def get_cpu_usage(): + return psutil.cpu_percent(interval=10) + +def get_memory_usage_gb(): + mem = psutil.virtual_memory() + return round(mem.used / (1024 ** 3), 2) + +# ------------------------- +# Main loop +# ------------------------- +while True: + now = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + # Get block number + try: + block_num = web3.eth.block_number + except Exception as e: + print(f"[FATAL] RPC connection lost, cannot get block number: {e}") + print("[FATAL] Script will exit now.") + sys.exit(1) + # CPU + Mem + mem_usage = get_memory_usage_gb() + cpu_usage = get_cpu_usage() + + # Console print + print(f"{now}: Block={block_num} | Mem={mem_usage}GB | CPU={cpu_usage}%") + + # Write log + with open(filename, "a") as f: + f.write(f"{now}, {block_num}, {mem_usage}, {cpu_usage}\n") + diff --git a/experiments/blockchain/testing/clear_docker.sh b/experiments/blockchain/testing/clear_docker.sh new file mode 100755 index 000000000..892437f1c --- /dev/null +++ b/experiments/blockchain/testing/clear_docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash +docker container prune -f +docker network prune -f +#docker system prune -a -f --volumes diff --git a/experiments/blockchain/testing/get_balance.py b/experiments/blockchain/testing/get_balance.py new file mode 100755 index 000000000..8953b82db --- /dev/null +++ b/experiments/blockchain/testing/get_balance.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +#from SEEDBlockchain import Wallet +from eth_account import Account +from web3 import Web3 + +def get_eth_balance(address): + """Get ETH balance of an address""" + try: + balance_wei = w3.eth.get_balance(address) + balance_eth = w3.fromWei(balance_wei, 'ether') + return balance_eth + except Exception as e: + print(f"Error getting balance: {e}") + return 0 + + +# Set your Ethereum node URL +eth_node_url = 'http://10.153.0.71:8545' +w3 = Web3(Web3.HTTPProvider(eth_node_url)) + +Account.enable_unaudited_hdwallet_features() + +for i in range(1000): + mnemonic = "gentle always fun glass foster produce north tail security list example gain" + path = f"m/44'/60'/0'/0/{i}" + account = Account.from_mnemonic(mnemonic, account_path=path) + print("({} --- {})".format(account.address, get_eth_balance(account.address))) + diff --git a/experiments/blockchain/testing/get_each_pos_block.sh b/experiments/blockchain/testing/get_each_pos_block.sh new file mode 100755 index 000000000..264beb29c --- /dev/null +++ b/experiments/blockchain/testing/get_each_pos_block.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +LOGFILE="pos_block_$(date '+%Y%m%d_%H%M%S').log" + +docker ps --format '{{.Names}}' | grep Geth | while IFS= read -r eth_container; do + block=$(docker exec "$eth_container" geth attach --exec 'eth.blockNumber' 2>&1) + exit_code=$? + + timestamp=$(date '+%F %T') + + if [ $exit_code -ne 0 ]; then + line="$timestamp ERROR on $eth_container: $block" + else + line="$timestamp $eth_container Block: $block" + fi + + echo "$line" + echo "$line" >> "$LOGFILE" +done + diff --git a/experiments/blockchain/testing/get_el_peers.py b/experiments/blockchain/testing/get_el_peers.py new file mode 100755 index 000000000..4fdef0c3b --- /dev/null +++ b/experiments/blockchain/testing/get_el_peers.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import argparse +from web3 import Web3 +import json + + +def extract_ip(addr: str | None): + if not addr: + return None + return addr.rsplit(":", 1)[0] + + +def main(): + parser = argparse.ArgumentParser(description="Query peer list from an Ethereum node") + parser.add_argument( + "--node", + type=str, + default="http://10.152.0.71:8545", + help="Ethereum node RPC URL" + ) + args = parser.parse_args() + + w3 = Web3(Web3.HTTPProvider(args.node)) + peers = w3.provider.make_request("admin_peers", []) + + result = {} + + for p in peers.get("result", []): + network = p.get("network", {}) + local_ip = extract_ip(network.get("localAddress")) + remote_ip = extract_ip(network.get("remoteAddress")) + + if not local_ip or not remote_ip: + continue + + if local_ip not in result: + result[local_ip] = { + "peers": [], + "count": 0 + } + + result[local_ip]["peers"].append(remote_ip) + result[local_ip]["count"] += 1 + + # ---- JSON only output ---- + print(json.dumps(result, indent=4)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/blockchain/testing/get_node_local_validator.sh b/experiments/blockchain/testing/get_node_local_validator.sh new file mode 100644 index 000000000..ff818359e --- /dev/null +++ b/experiments/blockchain/testing/get_node_local_validator.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Configuration +CONTAINER_FILTER="POS" +DATA_DIR="/tmp/vc/local-testnet/testnet/validators" + +echo "Scanning containers with name containing: $CONTAINER_FILTER" +echo "---------------------------------------------------------------" +printf "%-30s %-15s %-15s\n" "CONTAINER NAME" "IP ADDRESS" "VAL COUNT" +echo "---------------------------------------------------------------" + +# Get all running containers matching the filter +CONTAINERS=$(docker ps --format '{{.Names}}' | grep "$CONTAINER_FILTER") + +TOTAL_VALS=0 +TOTAL_NODES=0 + +for CONTAINER in $CONTAINERS; do + # Fetch container IP address + IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER") + + # Count directories starting with 0x in the validator path + # Using 'ls' to count physical keystore folders + VAL_COUNT=$(docker exec "$CONTAINER" bash -c "ls -d $DATA_DIR/0x* 2>/dev/null | wc -l") + + # Print row + printf "%-30s %-15s %-15s\n" "$CONTAINER" "$IP" "$VAL_COUNT" + + # Aggregate statistics + TOTAL_VALS=$((TOTAL_VALS + VAL_COUNT)) + TOTAL_NODES=$((TOTAL_NODES + 1)) +done + +echo "---------------------------------------------------------------" +echo "Summary Statistics:" +echo "Total POS Nodes Found: $TOTAL_NODES" +echo "Total Validators Found: $TOTAL_VALS" + diff --git a/experiments/blockchain/testing/get_peer.py b/experiments/blockchain/testing/get_peer.py new file mode 100755 index 000000000..c2e44634b --- /dev/null +++ b/experiments/blockchain/testing/get_peer.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import argparse +from web3 import Web3 +from datetime import datetime +import json + +def main(): + # ---- Command-line arguments ---- + parser = argparse.ArgumentParser(description="Query peer list from an Ethereum node") + parser.add_argument( + "--node", + type=str, + default="http://10.152.0.71:8545", + help="Ethereum node RPC URL (default: http://10.152.0.71:8545)" + ) + args = parser.parse_args() + + node_url = args.node + print(f"Connecting to node: {node_url}\n") + + # ---- Connect to Ethereum RPC ---- + w3 = Web3(Web3.HTTPProvider(node_url)) + + # ---- Request peer list (requires admin API enabled) ---- + peers = w3.provider.make_request("admin_peers", []) + + # ---- Print raw JSON ---- + print("===== Raw Peers JSON =====") + print(json.dumps(peers, indent=4)) + + print("\n===== Extracted Addresses =====") + + # ---- Extract localAddress and remoteAddress from peers ---- + for p in peers.get("result", []): + network = p.get("network", {}) + local_addr = network.get("localAddress") + remote_addr = network.get("remoteAddress") + + print(f"local: {local_addr}, remote: {remote_addr}") + + +if __name__ == "__main__": + main() + diff --git a/experiments/blockchain/testing/log2excel.py b/experiments/blockchain/testing/log2excel.py new file mode 100755 index 000000000..cf9ecd7a3 --- /dev/null +++ b/experiments/blockchain/testing/log2excel.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import sys +from datetime import datetime +from openpyxl import Workbook + +def parse_line(line): + line = line.strip() + if not line or line.startswith("time"): + return None + + parts = [x.strip() for x in line.split(",")] + if len(parts) != 4: + return None + + time, block, mem, cpu = parts + return [time, int(block), float(mem), float(cpu)] + +def main(): + if len(sys.argv) < 2: + print("Usage: python3 log_to_excel.py input.log") + return + + input_file = sys.argv[1] + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + output_file = f"log_{timestamp}.xlsx" + + wb = Workbook() + ws = wb.active + ws.append(["time", "block_number", "mem_usage_GB", "cpu_usage_percent"]) + + with open(input_file, "r") as f: + for line in f: + row = parse_line(line) + if row: + ws.append(row) + + wb.save(output_file) + print(f"Saved: {output_file}") + +if __name__ == "__main__": + main() + diff --git a/experiments/blockchain/testing/max_net_setting.sh b/experiments/blockchain/testing/max_net_setting.sh new file mode 100755 index 000000000..45d18408e --- /dev/null +++ b/experiments/blockchain/testing/max_net_setting.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +echo "===== High-Capacity Network Parameter Configuration (for 96 cores + 256GB RAM) =====" +echo "Writing to /etc/sysctl.conf ..." + +cat <<'EOF' | sudo tee -a /etc/sysctl.conf > /dev/null + +### ===== Large ARP table capacity (1,000,000 entries) ===== ### +net.ipv4.neigh.default.gc_thresh1 = 262144 +net.ipv4.neigh.default.gc_thresh2 = 524288 +net.ipv4.neigh.default.gc_thresh3 = 1048576 + +### ===== Large socket buffer size (128MB) ===== ### +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 +net.core.rmem_default = 134217728 +net.core.wmem_default = 134217728 +net.ipv4.tcp_rmem = 4096 87380 134217728 +net.ipv4.tcp_wmem = 4096 65536 134217728 +net.core.optmem_max = 131072 + +### ===== High-concurrency queue tuning ===== ### +net.core.somaxconn = 65535 +net.core.netdev_max_backlog = 1000000 + +### ===== Large conntrack table (16 million entries) ===== ### +net.netfilter.nf_conntrack_max = 16777216 +net.netfilter.nf_conntrack_buckets = 4194304 + +### ===== IP fragment buffer expansion ===== ### +net.ipv4.ipfrag_high_thresh = 268435456 +net.ipv4.ipfrag_low_thresh = 134217728 + +EOF + +echo "===== Applying new sysctl settings =====" +sudo sysctl -p + +echo "===== Flushing ARP/Neighbor table (important) =====" +sudo ip -s -s neigh flush all + +echo "===== Configuration completed =====" +echo "Your system ARP table capacity is now increased to 1,000,000 entries, optimized for high-density container networking." + diff --git a/experiments/blockchain/testing/parameter_description.md b/experiments/blockchain/testing/parameter_description.md new file mode 100644 index 000000000..92d695696 --- /dev/null +++ b/experiments/blockchain/testing/parameter_description.md @@ -0,0 +1,210 @@ + +--- + +# 一、脚本内容逐项解释 + +## 1. kernel.pid_max = 4194303 + +这是 系统允许的最大 PID 数量。 + +Linux 每个进程都有一个 PID。默认通常是 4194304 或更低（如 32768、65536）。 + +你设置为： + +``` +4194303 （靠近 Linux 最大允许值） +``` + +➤ 作用： + 允许系统同时创建更多的进程（包括容器、线程）。 + +--- + +## 2. /etc/security/limits.conf 参数解释 + +这些是 用户级别的资源限制，由 PAM（Pluggable Authentication Modules）管理。 + +在容器、Docker、大量并发服务时非常重要。 + +--- + +### nofile（打开文件数限制） + +``` +* soft nofile 1048576 +* hard nofile 1048576 +``` + +- soft = 当前生效的限制 +- hard = 最大允许限制（soft 不能超过 hard） +- nofile = 最大文件描述符数量（含 socket、pipe） + +➤ 作用：允许每个用户打开 最多 1,048,576 个文件/socket + Docker、大量容器、大量网络连接必须调大这个值。 + +--- + +### 📌 nproc（最大进程数限制） + +``` +* soft nproc unlimited +* hard nproc unlimited +``` + +➤ 作用：允许用户创建 无限数量的线程 / 进程 + +如果 nproc 不够，Docker 会报： + +``` +Fork failed: resource temporarily unavailable +``` + +--- + +### 📌 root 用户的设置 + +``` +root soft nofile 1048576 +root hard nofile 1048576 +root soft nproc unlimited +root hard nproc unlimited +``` + +和上面一样，但专门针对 root 用户。 + +因为 Docker daemon 以 root 运行，所以 root 的限制尤其关键。 + +--- + +# ✅ 二、如何检查这些参数是否生效？ + +--- + +# 1️⃣ 检查 pid_max + +``` +cat /proc/sys/kernel/pid_max +``` + +应该看到： + +``` +4194303 +``` + +--- + +# 2️⃣ 检查当前会话文件描述符限制（nofile） + +``` +ulimit -n +ulimit -Hn +``` + +期望输出： + +``` +1048576 +``` + +⚠️ 注意： + 如果你用的是 SSH 或 login shell，需要重新登录一次才会生效。 + +--- + +# 3️⃣ 检查当前会话最大进程数（nproc） + +``` +ulimit -u +ulimit -Hu +``` + +期望输出： + +``` +unlimited +``` + +--- + +# 4️⃣ 检查系统级 FD 使用情况 + +查看总 FD 使用情况： + +``` +cat /proc/sys/fs/file-nr +``` + +会输出 3 个数字，例如： + +``` +51200 0 1048576 +``` + +解释： + +| 字段 | 含义 | +|-----|--------------| +| 第1列 | 已分配但未使用的 FD | +| 第2列 | 已分配且正在使用的 FD | +| 第3列 | 最大可用 FD（系统级） | + +最大值可以查看： + +``` +cat /proc/sys/fs/file-max +``` + +--- + +# 5️⃣ 检查每个进程打开的 FD + +例如查看 Docker 进程： + +``` +pidof dockerd | xargs -I{} ls /proc/{}/fd | wc -l +``` + +--- + +# 6️⃣ 检查 nproc 使用情况 + +``` +ps -eLf | wc -l +``` + +查看线程/进程总数。 + +--- + +# 7️⃣ 检查 PAM 是否生效 + +``` +grep pam_limits.so /etc/pam.d/* +``` + +在 Ubuntu 默认是生效的。 + +--- + +# 8️⃣ 检查 limits.conf 是否正确写入 + +``` +grep -E "nofile|nproc" /etc/security/limits.conf +``` + +--- + +# ⚠️ 是否要重启？ + +| 参数 | 是否要重启 | +|-----------------------------|--------------------------| +| pid_max | 不需要，马上生效 | +| limits.conf（nofile / nproc） | ⚠️ 需要重登 或重启所有服务 | +| Docker 守护进程需要重新启动 | systemctl restart docker | + +如果你要确保所有服务应用最新限制，建议最终重启： + +``` +reboot +``` \ No newline at end of file diff --git a/experiments/blockchain/testing/pos_container_size_check.sh b/experiments/blockchain/testing/pos_container_size_check.sh new file mode 100644 index 000000000..6119569c0 --- /dev/null +++ b/experiments/blockchain/testing/pos_container_size_check.sh @@ -0,0 +1 @@ +dockps |grep POS | sort -t'-' -k4,4n |wc -l diff --git a/experiments/blockchain/testing/proposer_monitor.py b/experiments/blockchain/testing/proposer_monitor.py new file mode 100644 index 000000000..7f378731d --- /dev/null +++ b/experiments/blockchain/testing/proposer_monitor.py @@ -0,0 +1,90 @@ +import requests +import pandas as pd +import numpy as np +import time +import os +from collections import deque +from datetime import datetime +import argparse + +# Argument parsing +parser = argparse.ArgumentParser(description="Beacon Chain Slot Monitor") +parser.add_argument("--ip", type=str, default="127.0.0.1", + help="Beacon node IP address (default: 127.0.0.1)") +parser.add_argument("--port", type=int, default=8000, + help="Beacon node port (default: 8000)") + +args = parser.parse_args() + +NODE_IP = args.ip +PORT = args.port +BASE_URL = f"http://{NODE_IP}:{PORT}" + +MAX_SLOTS = 32 + +# Buffer to store the last 32 slots of data +slot_buffer = deque(maxlen=MAX_SLOTS) + +def clear_terminal(): + os.system('cls' if os.name == 'nt' else 'clear') + +def fetch_data(): + try: + header_url = f"{BASE_URL}/eth/v1/beacon/headers/head" + header_res = requests.get(header_url, timeout=5).json() + + header_data = header_res['data']['header']['message'] + current_slot = header_data['slot'] + proposer_index = header_data['proposer_index'] + + committee_url = f"{BASE_URL}/eth/v1/beacon/states/head/committees?slot={current_slot}" + committee_res = requests.get(committee_url, timeout=5).json() + + validators = [] + for comm in committee_res.get('data', []): + validators.extend(comm.get('validators', [])) + + return { + "SLOT": int(current_slot), + "PROPOSER_INDEX": proposer_index, + "VALIDATORS_TOTAL": len(validators), + "VALIDATOR_LIST_PREVIEW": str(validators[:5]) + "..." if len(validators) > 5 else str(validators) + } + + except Exception as e: + print(f"Error fetching data: {e}") + return None + +def main(): + print(f"Connecting to {NODE_IP}:{PORT}...") + + while True: + new_data = fetch_data() + + if new_data: + if not any(d['SLOT'] == new_data['SLOT'] for d in slot_buffer): + slot_buffer.append(new_data) + + df = pd.DataFrame(list(slot_buffer)) + + cols = ["SLOT", "PROPOSER_INDEX", "VALIDATORS_TOTAL", "VALIDATOR_LIST_PREVIEW"] + df = df[cols].sort_values(by="SLOT", ascending=False) + + clear_terminal() + print(f"=== Beacon Chain Slot Monitor (Rolling 32) | {datetime.now().strftime('%H:%M:%S')} ===") + print(f"Node: {NODE_IP}:{PORT}") + print("=" * 100) + + pd.set_option('display.width', 1000) + print(df.to_string(index=False, col_space=15, justify='left')) + + print("=" * 100) + print(f"Buffer Size: {len(slot_buffer)}/32 | Refresh: 12s") + + time.sleep(12) + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nMonitor stopped.") diff --git a/experiments/blockchain/testing/readme.md b/experiments/blockchain/testing/readme.md new file mode 100644 index 000000000..d0cd09a9d --- /dev/null +++ b/experiments/blockchain/testing/readme.md @@ -0,0 +1,92 @@ +# 区块连测试说明 +## 测试环境 + +- ubuntu24.04 server +- cpu：AMD EPYC 9654 96-Core Processor， 256G Mem +- python env： conda activate seedpy310 +## 网络设置（防止 ping 报错 no buffer space available） +执行 [max_net_setting.sh](./max_net_setting.sh) +``` +./max_net_setting.sh +``` + +## 后台持续交易100亿次（1000个账户）间隔0.5S +执行 [send_rand_tx.py](./send_rand_tx.py) +``` +./send_rand_tx.py +``` +## 测试block数量，内存使用和cpu使用率(10s检测一次) +执行 [check_block_cpu_mem.py](./check_block_cpu_mem.py) +``` +Usage: python3 check_block_cpu_mem.py +Example: python3 check_blosck_cpu_mem.py http://ip:8545 +``` +## block内存测试日志转excel +将上面测试block内存的log文件转换为excel +执行 [log2excel.py](./log2excel.py) +``` +Usage: python3 log_to_excel.py input.log +``` + +## 测试block生成间隔 +执行 [block_monitor.py](./block_monitor.py) +``` +usage: block_monitor.py [-h] --node NODE [--interval INTERVAL] [--threshold THRESHOLD] +``` +**参数说明** +- --interval 为每次get_block的间隔，默认5s，建议设置为1 +- --threshold 为block间隔的警告时间。默认20S，超过20s会写入日志 +**推荐** +``` +python3 block_monitor.py --node http://10.164.0.84:8545 --threshold 20 --interval 1 +``` + + +## 定时交易检测receipt是否成功 +执行 [web3_raw_tx.py](./web3_raw_tx.py) + +``` +python3 web3_raw_tx.py +``` +程序自动每60s自动发一次交易并写入日志 + + +## docker limit设置 +执行 +``` +sudo ./set_docker_limits.sh +``` + +## 系统设置 +执行 +``` +sudo ./set_sys_limit.sh +``` + +## docker compose 分批build 和up ，默认50个，可以自行修改 +将脚本拷贝到output/文件夹 +执行分批build +``` +./batch_docker_compose_build.sh +``` +执行分批up +``` +./batch_docker_compose_up.sh +``` + +## 检测运行的Ethereum-POS容器数量 +``` +source pos_container_size_check.sh +``` +## 获取node节点的peer +执行 +``` +./get_peer.py +``` +参数node在代码里面修改 + +## 查看所有POS能否geth attach 获取block +执行 +``` +./get_each_pos_block.sh +``` \ No newline at end of file diff --git a/experiments/blockchain/testing/send_rand_tx.py b/experiments/blockchain/testing/send_rand_tx.py new file mode 100755 index 000000000..6e4e0d6f2 --- /dev/null +++ b/experiments/blockchain/testing/send_rand_tx.py @@ -0,0 +1,111 @@ +#!/bin/env python3 +# Send a raw transaction + +import random, time, json +from web3 import Web3 +from eth_account import Account + + + +# Print balance +def print_balance(w, msg, account): + print("{}: {} (account: {})".format(msg, w.getBalance(account), account)) + + +def get_eth_balance(address): + """Get ETH balance of an address""" + try: + balance_wei = w3.eth.get_balance(address) + balance_eth = w3.from_wei(balance_wei, 'ether') + return balance_eth + except Exception as e: + print(f"Error getting balance: {e}") + return 0 + + +def send_eth_transaction(from_account, to_address, amount_eth, nonce=None): + """Send ETH transaction from one account to another""" + try: + # Convert amount to wei + amount_wei = w3.toWei(amount_eth, 'ether') + + # Get current gas price + gas_price = w3.eth.gas_price + + # Standard ETH transfer gas limit + gas_limit = 21000 + + # Calculate total cost (amount + gas fees) + total_cost = amount_wei + (gas_limit * gas_price) + + # Build transaction + transaction = { + 'to': to_address, + 'value': amount_wei, + 'gas': gas_limit, + 'gasPrice': gas_price, + 'nonce': nonce if nonce is not None else w3.eth.get_transaction_count(from_account.address), + 'chainId': 1337 + } + + # Sign transaction + signed_txn = w3.eth.account.sign_transaction(transaction, from_account.key) + + # Send transaction + tx_hash = w3.eth.send_raw_transaction(signed_txn.rawTransaction) + + return tx_hash.hex() + + except Exception as e: + print(f"Error sending transaction (ignored): {e}") + return None + + +# Connect to the blockchain +eth_node_url = 'http://10.151.0.71:8545' +w3 = Web3(Web3.HTTPProvider(eth_node_url)) + +Account.enable_unaudited_hdwallet_features() + + +# Generate the accouts +accounts = [] +mnemonic = "gentle always fun glass foster produce north tail security list example gain" + + +# Make sure that these accounts are pre-funded in the emulator, +# otherwise they don't have sufficient fund to send transactions. +total = 1000 + +print(f"Generating {total} accounts ...") +for i in range(total): + path = f"m/44'/60'/0'/0/{i}" + account = Account.from_mnemonic(mnemonic, account_path=path) + #print("({}:{})".format(i, wallet.getBalance(account.address))) + accounts.append(account) + + +rounds = 10000000000 # how many rounds: each round one transaction is sent +wait_time = 0.5 # waiting time after a transaction is sent + +for x in range(rounds): + amount = random.randint(1, 10)/10 + + sender_index = random.randint(0, total-1) + recipt_index = random.randint(0, total-1) + while recipt_index == sender_index: + recipt_index = random.randint(0, total-1) + + print("----------------------------------------------------") + print("{}: Sending {} ethers from accounts[{}] to accounts[{}]".format(x, amount, sender_index, recipt_index)) + + # Select the sender and recipient + sender = accounts[sender_index] + to_address = accounts[recipt_index].address + + tx_hash = send_eth_transaction(sender, to_address, amount) + if tx_hash: + print(f"Transaction is sent! Hash: {tx_hash}") + + time.sleep(wait_time) + diff --git a/experiments/blockchain/testing/set_docker_limits.sh b/experiments/blockchain/testing/set_docker_limits.sh new file mode 100755 index 000000000..f6d8bc794 --- /dev/null +++ b/experiments/blockchain/testing/set_docker_limits.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e + +echo "===== Configure Docker systemd limits (High Concurrency) =====" + +# Create override directory +sudo mkdir -p /etc/systemd/system/docker.service.d + +# Write override config +cat </dev/null +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TasksMax=infinity +EOF + + +echo "? override.conf created with high limits" + +# Reload systemd +sudo systemctl daemon-reload +echo "? systemd daemon reloaded" + +# Restart Docker +sudo systemctl restart docker +echo "? Docker restarted" + +echo "===== Done: High concurrency limits applied =====" + diff --git a/experiments/blockchain/testing/set_sys_limit.sh b/experiments/blockchain/testing/set_sys_limit.sh new file mode 100755 index 000000000..4bd3ea5db --- /dev/null +++ b/experiments/blockchain/testing/set_sys_limit.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -e +echo "kernel.pid_max = 4194303" | sudo tee -a /etc/sysctl.conf +sudo sysctl -p + +echo "===== Applying system-wide file descriptor and process limits =====" + +LIMITS_FILE="/etc/security/limits.conf" + +append_if_missing() { + local pattern="$1" + local line="$2" + + if ! grep -qF "$pattern" "$LIMITS_FILE"; then + echo "$line" | sudo tee -a "$LIMITS_FILE" >/dev/null + echo "Added: $line" + else + echo "Exists: $line" + fi +} + +# Add limits (soft/hard nofile + nproc) +append_if_missing "* soft nofile" "* soft nofile 1048576" +append_if_missing "* hard nofile" "* hard nofile 1048576" +append_if_missing "* soft nproc" "* soft nproc unlimited" +append_if_missing "* hard nproc" "* hard nproc unlimited" + +# Add root-specific limits +append_if_missing "root soft nofile" "root soft nofile 1048576" +append_if_missing "root hard nofile" "root hard nofile 1048576" +append_if_missing "root soft noproc" "root soft nofile unlimited" +append_if_missing "root hard noproc" "root hard nofile unlimited" + + +echo "===== Done. Reboot required for full effect =====" + diff --git a/experiments/blockchain/testing/vc_monitor.py b/experiments/blockchain/testing/vc_monitor.py new file mode 100644 index 000000000..bfae485ee --- /dev/null +++ b/experiments/blockchain/testing/vc_monitor.py @@ -0,0 +1,84 @@ +import requests +import pandas as pd +import time +import os +import argparse # Added for argument parsing +from datetime import datetime + +def clear_terminal(): + # Clear terminal screen based on OS + os.system('cls' if os.name == 'nt' else 'clear') + +def fetch_validator_data(ip): + # Standard Beacon API endpoint + url = f"http://{ip}:8000/eth/v1/beacon/states/head/validators" + try: + response = requests.get(url, timeout=15) + response.raise_for_status() + return response.json().get('data', []) + except Exception as e: + print(f"API Request Error: {e}") + return None + +def run_monitor(target_ip): + # Set pandas display options for wider spacing and full content + pd.set_option('display.max_colwidth', None) + pd.set_option('display.width', 1000) + pd.set_option('display.colheader_justify', 'left') + + print(f"Initializing monitor for node: {target_ip}...") + + while True: + raw_validators = fetch_validator_data(target_ip) + + if raw_validators: + extracted_rows = [] + + for item in raw_validators: + extracted_rows.append({ + "INDEX": item['index'], + "BALANCE (GWEI)": item['balance'], # Raw Gwei string, no rounding + "STATUS": item['status'], + "PUBKEY": item['validator']['pubkey'] # Full pubkey + }) + + # Create and sort DataFrame + df = pd.DataFrame(extracted_rows) + df['INDEX'] = df['INDEX'].astype(int) + df = df.sort_values(by='INDEX') + + # Refresh output + clear_terminal() + current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + print(f"=== Validator Monitor | Last Update: {current_time} ===") + print(f"Node IP: {target_ip}") + print("=" * 160) + + # Use to_string with justify and col_space for large gaps + print(df.to_string(index=False, col_space=20, justify='left')) + + print("=" * 160) + print("Refreshing every 12 seconds... (Press Ctrl+C to stop)") + + else: + print(f"[{datetime.now().strftime('%H:%M:%S')}] No data received from {target_ip}. Retrying in 12s...") + + time.sleep(12) + +if __name__ == "__main__": + # Setup Argument Parser + parser = argparse.ArgumentParser(description="Beacon Chain Validator Monitor") + parser.add_argument( + "--ip", + type=str, + default="10.153.0.96", + help="The IP address of the Ethereum node (default: 10.153.0.96)" + ) + + args = parser.parse_args() + + try: + run_monitor(args.ip) + except KeyboardInterrupt: + print("\nMonitor stopped by user.") diff --git a/experiments/blockchain/testing/web3_raw_tx.py b/experiments/blockchain/testing/web3_raw_tx.py new file mode 100755 index 000000000..2ea017998 --- /dev/null +++ b/experiments/blockchain/testing/web3_raw_tx.py @@ -0,0 +1,141 @@ +#!/bin/env python3 +# Send ETH transaction, print status, and write detailed log with timestamped filename + +import random +import time +import argparse +from web3 import Web3 +from eth_account import Account +from datetime import datetime +import traceback +# -------------------------- +# Parse command-line arguments +# -------------------------- +# -------------------------- +# Parse command-line arguments +# -------------------------- +parser = argparse.ArgumentParser(description="ETH transaction sender") + +parser.add_argument( + "--ip", + type=str, + default="10.151.0.80", + help="Ethereum node IP address" +) + +parser.add_argument( + "--port", + type=int, + default=8545, + help="Ethereum node RPC port" +) + +args = parser.parse_args() + +# Build RPC URL +NODE_URL = f"http://{args.ip}:{args.port}" + + +# Connect to blockchain +w3 = Web3(Web3.HTTPProvider(NODE_URL)) + +Account.enable_unaudited_hdwallet_features() + +# Generate accounts +accounts = [] +mnemonic = "gentle always fun glass foster produce north tail security list example gain" +total = 1000 + +print(f"Generating {total} accounts ...") +for i in range(total): + path = f"m/44'/60'/0'/0/{i}" + account = Account.from_mnemonic(mnemonic, account_path=path) + accounts.append(account) + +rounds = 10000000000 +wait_time = 60 # seconds between transactions + +# Log filename with timestamp +log_file = f"transaction_log_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log" + +def log_transaction(amount, from_addr, to_addr, tx_hash, status): + """Append detailed transaction info to log file""" + timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + with open(log_file, "a") as f: + f.write(f"{timestamp} | Amount: {amount} ETH | From: {from_addr} | To: {to_addr} | Hash: {tx_hash} | Status: {status}\n") + +def send_eth_transaction(from_account, to_address, amount_eth): + try: + # 1. 检查发送者余额 (v5 使用 fromWei) + balance_wei = w3.eth.get_balance(from_account.address) + balance_eth = w3.fromWei(balance_wei, 'ether') + + amount_wei = w3.toWei(amount_eth, 'ether') + gas_price = w3.eth.gas_price + gas_limit = 21000 + + if balance_wei < (amount_wei + (gas_price * gas_limit)): + print(f"Skipping: Insufficient funds. Balance: {balance_eth} ETH") + return None + + nonce = w3.eth.get_transaction_count(from_account.address) + + tx = { + 'to': to_address, + 'value': amount_wei, + 'gas': gas_limit, + 'gasPrice': gas_price, + 'nonce': nonce, + 'chainId': 1337 # 使用动态获取的 ID + } + + # v5 使用 sign_transaction 和 rawTransaction + signed_txn = w3.eth.account.sign_transaction(tx, from_account.key) + tx_hash = w3.eth.send_raw_transaction(signed_txn.rawTransaction) + print(f"Transaction sent! Hash: {tx_hash.hex()}") + + print(f"Waiting for receipt (Max 50s)...") + # 如果这里报错，traceback 会抓住它 + receipt = w3.eth.wait_for_transaction_receipt( + tx_hash, + timeout=50, + poll_latency=1 + ) + + status = "Success" if receipt.status == 1 else "Fail" + print(f"Transaction status: {status}\n") + + log_transaction(amount_eth, from_account.address, to_address, tx_hash.hex(), status) + return tx_hash.hex() + + except Exception as e: + print("\n" + "!"*60) + print("ERROR DETECTED:") + # 打印详细的报错行号和错误类型 + traceback.print_exc() + print("!"*60 + "\n") + + # 记录到日志 + log_transaction(amount_eth, from_account.address, to_address, "ERROR", str(e)) + return None + + +for x in range(rounds): + amount = random.randint(1, 10) / 10 + + sender_index = random.randint(0, total - 1) + recipt_index = random.randint(0, total - 1) + while recipt_index == sender_index: + recipt_index = random.randint(0, total - 1) + + sender = accounts[sender_index] + to_address = accounts[recipt_index].address + + print("----------------------------------------------------") + print(f"{x}: Sending {amount} ETH from accounts[{sender_index}] to accounts[{recipt_index}]") + + send_eth_transaction(sender, to_address, amount) + + print(f"Sleeping {wait_time} seconds...\n") + time.sleep(wait_time) + diff --git a/requirements.txt b/requirements.txt index 1705dcb59..d63f4d27e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,8 @@ lru-dict==1.3.0 multiaddr==0.0.9 multidict==6.1.0 netaddr==1.3.0 +networkx==3.4.2 +numpy==1.26.4 parsimonious==0.8.1 propcache==0.3.0 protobuf==3.19.5 diff --git a/seedemu/compiler/Docker.py b/seedemu/compiler/Docker.py index cf270b4b0..59d7ce462 100644 --- a/seedemu/compiler/Docker.py +++ b/seedemu/compiler/Docker.py @@ -18,7 +18,9 @@ # The Etherview is updated on 2025/11, we name the new version 2.0 #SEEDEMU_ETHER_VIEW_IMAGE='handsonsecurity/seedemu-multiarch-etherview:buildx-latest' -SEEDEMU_ETHER_VIEW_IMAGE='handsonsecurity/seedemu-multiarch-etherview:2.0' +# SEEDEMU_ETHER_VIEW_IMAGE='handsonsecurity/seedemu-multiarch-etherview:2.0' +SEEDEMU_ETH_EXPLORER_BACKEND_IMAGE='handsonsecurity/seedemu-ethexplorer-backend:1.0' +SEEDEMU_ETH_EXPLORER_WEB_IMAGE='handsonsecurity/seedemu-ethexplorer-web:1.0' DockerCompilerFileTemplates: Dict[str, str] = {} @@ -101,7 +103,6 @@ ''' DockerCompilerFileTemplates['compose'] = """\ -version: "3.4" services: {dummies} {services} @@ -214,14 +215,160 @@ {labelList} """ -DockerCompilerFileTemplates['seedemu_ether_view'] = """\ - seedemu-ether-client: - image: {clientImage} - container_name: seedemu_ether_view - volumes: - - /var/run/docker.sock:/var/run/docker.sock +DockerCompilerFileTemplates['seedemu_eth_explorer'] = """\ + clickhouse: + image: clickhouse/clickhouse-server:latest + container_name: clickhouse + restart: unless-stopped + environment: + - CLICKHOUSE_DB=beaconchain + - CLICKHOUSE_USER=beacon + - CLICKHOUSE_PASSWORD=pass + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] + interval: 30s + timeout: 10s + retries: 3 + networks: + - beacon-network + + postgres: + image: postgres:15.2-alpine + container_name: postgres + restart: unless-stopped + environment: + POSTGRES_PASSWORD: pass + POSTGRES_DB: db + POSTGRES_USER: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 30s + timeout: 10s + retries: 5 + networks: + - beacon-network + + alloy: + image: postgres:15.2-alpine + container_name: alloy + restart: unless-stopped + environment: + POSTGRES_PASSWORD: pass + POSTGRES_DB: db + POSTGRES_USER: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 30s + timeout: 10s + retries: 5 + networks: + - beacon-network + + redis: + image: redis:7-alpine + container_name: redis + restart: unless-stopped + command: redis-server --appendonly yes + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 3 + networks: + - beacon-network + + littlebigtable: + image: gobitfly/little_bigtable:latest + container_name: littlebigtable + restart: unless-stopped + depends_on: + - postgres + networks: + - beacon-network + + rawbigtable: + image: gobitfly/little_bigtable:latest + container_name: rawbigtable + restart: unless-stopped + depends_on: + - alloy + networks: + - beacon-network + + seedemu-ethexplorer-backend: + image: {clientBackendImage} + container_name: seedemu_ethexplorer_backend + cap_add: + - ALL + pid: host + privileged: true + healthcheck: + test: + [ + "CMD-SHELL", + "PGPASSWORD=pass psql -h postgres -U postgres -d db -tAc \\"SELECT 1 FROM information_schema.tables WHERE table_name='blocks';\\" | grep -q 1" + ] + interval: 5s + timeout: 3s + retries: 60 + start_period: 10s + depends_on: + clickhouse: + condition: service_healthy + postgres: + condition: service_healthy + alloy: + condition: service_healthy + redis: + condition: service_healthy + littlebigtable: + condition: service_started + rawbigtable: + condition: service_started + environment: + - EL_HOST={el_host} + - CL_HOST={cl_host} + networks: + - beacon-network + - {beaconNetwork} + - {gEthNetwork} + + seedemu-ethexplorer-web: + image: {clientWebImage} + container_name: seedemu_ethexplorer_web + cap_add: + - ALL + pid: host + privileged: true + depends_on: + clickhouse: + condition: service_healthy + postgres: + condition: service_healthy + alloy: + condition: service_healthy + redis: + condition: service_healthy + littlebigtable: + condition: service_started + rawbigtable: + condition: service_started + seedemu-ethexplorer-backend: + condition: service_healthy + environment: + - EL_HOST={el_host} + - CL_HOST={cl_host} ports: - {clientPort}:5000/tcp + networks: + - beacon-network + - {beaconNetwork} + - {gEthNetwork} +""" + +DockerCompilerFileTemplates['seedemu_eth_explorer_net'] = """\ + beacon-network: + driver: bridge """ DockerCompilerFileTemplates['zshrc_pre'] = """\ @@ -1317,8 +1464,10 @@ def _doCompile(self, emulator: Emulator): self._groupSoftware(emulator) - for ((scope, type, name), obj) in registry.getAll().items(): + el_host = cl_host = '' + beaconNetwork = gEthNetwork = '' + for ((scope, type, name), obj) in registry.getAll().items(): if type == 'net': self._log('creating network: {}/{}...'.format(scope, name)) self.__networks += self._compileNet(obj) @@ -1336,6 +1485,19 @@ def _doCompile(self, emulator: Emulator): self._log('compiling host node {} for as{}...'.format(name, scope)) self.__services += self._compileNode(obj) + if el_host and cl_host: + continue + + ip = str(obj.getInterfaces()[0].getAddress()) + name = obj.getDisplayName() if obj.getDisplayName() is not None else obj.getName() + net = self._getRealNetName(obj.getInterfaces()[0].getNet()) + if "-Beacon-" in name and cl_host == "": + cl_host = ip + gEthNetwork = net + elif "-Geth-" in name and el_host == "": + el_host = ip + beaconNetwork = net + if type == 'rs': self._log('compiling rs node for {}...'.format(name)) self.__services += self._compileNode(obj) @@ -1351,24 +1513,26 @@ def _doCompile(self, emulator: Emulator): # Attach the Map container to the default network self.attachInternetMap(port_forwarding="{}:8080/tcp".format(self.__internet_map_port)) - # Add the Ether View contaienr to docker's default network if self.__ether_view_enabled: - self._log('enabling seedemu-ether-view...') - - self.__services += DockerCompilerFileTemplates['seedemu_ether_view'].format( - clientImage = SEEDEMU_ETHER_VIEW_IMAGE, - clientPort = self.__ether_view_port + self._log('enabling seedemu-eth-explorer...') + + self.__services += DockerCompilerFileTemplates['seedemu_eth_explorer'].format( + clientWebImage=SEEDEMU_ETH_EXPLORER_WEB_IMAGE, + clientBackendImage=SEEDEMU_ETH_EXPLORER_BACKEND_IMAGE, + cl_host=cl_host, + el_host=el_host, + beaconNetwork=beaconNetwork, + gEthNetwork=gEthNetwork, + clientPort=self.__ether_view_port ) self.__services += '\n' - + self.__networks += DockerCompilerFileTemplates['seedemu_eth_explorer_net'] # Add custom entries (typically added through Docker::attachCustomContainer APIs) self.__services += self.__custom_services - local_images = '' - for (image, _) in self.__images.values(): if image.getName() not in self._used_images or not image.isLocal(): continue local_images += DockerCompilerFileTemplates['local_image'].format( @@ -1380,10 +1544,10 @@ def _doCompile(self, emulator: Emulator): self._log('creating docker-compose.yml...'.format(scope, name)) print(DockerCompilerFileTemplates['compose'].format( - services = self.__services, - networks = self.__networks, - volumes = toplevelvolumes, - dummies = local_images + self._makeDummies() + services=self.__services, + networks=self.__networks, + volumes=toplevelvolumes, + dummies=local_images + self._makeDummies() ), file=open('docker-compose.yml', 'w')) self.generateEnvFile(Scope(ScopeTier.Global),'') diff --git a/seedemu/compiler/DockerImageConstant.py b/seedemu/compiler/DockerImageConstant.py index 68a54452c..a785a74d2 100644 --- a/seedemu/compiler/DockerImageConstant.py +++ b/seedemu/compiler/DockerImageConstant.py @@ -2,16 +2,16 @@ from seedemu.core import BaseSystem from enum import Enum -UBUNTU_IMAGE = DockerImage(name='ubuntu:20.04', +UBUNTU_IMAGE = DockerImage(name='ubuntu:24.04', software=[], subset=None) -BASE_IMAGE = DockerImage(name='handsonsecurity/seedemu-multiarch-base:buildx-latest', +BASE_IMAGE = DockerImage(name='handsonsecurity/seedemu-base:2.0', software=['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', - 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat'], + 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat-openbsd'], subset=UBUNTU_IMAGE) -ROUTER_IMAGE = DockerImage(name='handsonsecurity/seedemu-multiarch-router:buildx-latest', +ROUTER_IMAGE = DockerImage(name='handsonsecurity/seedemu-router:2.0', software=['bird2'], subset=BASE_IMAGE) @@ -23,12 +23,16 @@ software=['software-properties-common', 'python3', 'python3-pip'], subset=BASE_IMAGE) -ETHEREUM_IMAGE_POS = DockerImage(name='handsonsecurity/seedemu-ethereum:pos', +ETHEREUM_IMAGE_POS = DockerImage(name='handsonsecurity/seedemu-ethereum:pos2.0', software=['software-properties-common', 'python3', 'python3-pip'], subset=BASE_IMAGE) MONERO_IMAGE = DockerImage(name='handsonsecurity/seedemu-monero:latest', software=[], subset=BASE_IMAGE) +# Agave (Solana) is packaged as a multiarch image. The image contains the +# pinned upstream binaries for amd64 and arm64 from docker_images/seedemu-solana. +SOLANA_IMAGE = DockerImage(name='handsonsecurity/seedemu-solana:1.0', software=[], local=False, subset=BASE_IMAGE) + OP_STACK_IMAGE = DockerImage(name='huagluck/seedemu-op-stack', software=[], subset=BASE_IMAGE) SC_DEPLOYER_IMAGE = DockerImage(name='huagluck/seedemu-sc-deployer', software=[], subset=BASE_IMAGE) @@ -41,12 +45,12 @@ software=[], subset=None) -BASE_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-multiarch-base:buildx-latest', +BASE_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-base:2.0', software=['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', - 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat'], + 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat-openbsd'], subset=UBUNTU_IMAGE_ARM64) -ROUTER_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-multiarch-router:buildx-latest', +ROUTER_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-router:2.0', software=['bird2'], subset=BASE_IMAGE_ARM64) @@ -58,12 +62,15 @@ software=['software-properties-common', 'python3', 'python3-pip'], subset=BASE_IMAGE_ARM64) -ETHEREUM_IMAGE_ARM64_POS = DockerImage(name='handsonsecurity/seedemu-ethereum-arm64:pos', +ETHEREUM_IMAGE_ARM64_POS = DockerImage(name='handsonsecurity/seedemu-ethereum:pos2.0', software=['software-properties-common', 'python3', 'python3-pip'], subset=BASE_IMAGE_ARM64) MONERO_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-monero:latest', software=[], subset=BASE_IMAGE_ARM64) +# The same multiarch Solana image is used for arm64. +SOLANA_IMAGE_ARM64 = DockerImage(name='handsonsecurity/seedemu-solana:1.0', software=[], local=False, subset=BASE_IMAGE_ARM64) + OP_STACK_IMAGE_ARM64 = DockerImage(name='huagluck/seedemu-op-stack', software=[], subset=BASE_IMAGE_ARM64) SC_DEPLOYER_IMAGE_ARM64 = DockerImage(name='huagluck/seedemu-sc-deployer', software=[], subset=BASE_IMAGE_ARM64) @@ -80,6 +87,7 @@ BaseSystem.SEEDEMU_ETHEREUM_LEGACY: ETHEREUM_IMAGE_LEGACY, BaseSystem.SEEDEMU_ETHEREUM_POS: ETHEREUM_IMAGE_POS, BaseSystem.SEEDEMU_MONERO: MONERO_IMAGE, + BaseSystem.SEEDEMU_SOLANA: SOLANA_IMAGE, BaseSystem.SEEDEMU_OP_STACK: OP_STACK_IMAGE, BaseSystem.SEEDEMU_SC_DEPLOYER: SC_DEPLOYER_IMAGE, BaseSystem.SEEDEMU_CHAINLINK: CHAINLINK_IMAGE @@ -93,6 +101,7 @@ BaseSystem.SEEDEMU_ETHEREUM_LEGACY: ETHEREUM_IMAGE_ARM64_LEGACY, BaseSystem.SEEDEMU_ETHEREUM_POS: ETHEREUM_IMAGE_ARM64_POS, BaseSystem.SEEDEMU_MONERO: MONERO_IMAGE_ARM64, + BaseSystem.SEEDEMU_SOLANA: SOLANA_IMAGE_ARM64, BaseSystem.SEEDEMU_OP_STACK: OP_STACK_IMAGE_ARM64, BaseSystem.SEEDEMU_SC_DEPLOYER: SC_DEPLOYER_IMAGE_ARM64, BaseSystem.SEEDEMU_CHAINLINK: CHAINLINK_IMAGE_ARM64 diff --git a/seedemu/compiler/__init__.py b/seedemu/compiler/__init__.py index 99c20fab5..552e5d449 100644 --- a/seedemu/compiler/__init__.py +++ b/seedemu/compiler/__init__.py @@ -4,4 +4,5 @@ # Disable the compilers options from .DistributedDocker import DistributedDocker from .Graphviz import Graphviz -from .GcpDistributedDocker import GcpDistributedDocker \ No newline at end of file +from .GcpDistributedDocker import GcpDistributedDocker +from .kubernetes import KubernetesCompiler, NativeKubernetesCompiler, SchedulingStrategy diff --git a/seedemu/compiler/kubernetes.py b/seedemu/compiler/kubernetes.py new file mode 100644 index 000000000..46b13749a --- /dev/null +++ b/seedemu/compiler/kubernetes.py @@ -0,0 +1,591 @@ +from __future__ import annotations + +import json +import os +import ipaddress +import re +import shutil +from hashlib import md5, sha1 +from os import chdir, mkdir +from pathlib import Path +from typing import Any, Dict, List + +import yaml + +from seedemu.compiler.Docker import Docker +from seedemu.core import Network, Node + + +INTERNET_MAP_META_PREFIX = "org.seedsecuritylabs.seedemu.meta." +# Border routers are already represented through the router node path in SEED's +# registry. Compiling "brdnode" again creates duplicate Deployment names. +SEEDEMU_NODE_TYPES = ["rnode", "csnode", "hnode", "rs", "snode"] + + +class SchedulingStrategy: + """Compatibility constants for older Kubernetes compiler callers.""" + + NONE = "none" + AUTO = "auto" + BY_AS = "by_as" + BY_AS_HARD = "by_as_hard" + BY_ROLE = "by_role" + CUSTOM = "custom" + + +class KubernetesCompiler(Docker): + """Minimal Kubernetes compiler for a k8s-native baseline. + + Design goals: + - public compiler name is KubernetesCompiler + - kube-ovn is the default secondary CNI, macvlan is the only alternative + - no inventory dependency during compile + - no nodeSelector / affinity / topology spreading + - no node-aware preload planning + - generated orchestration artifacts are limited to one manifest file and + images.yaml; build/deploy orchestration lives in seedemu.k8sTools. + """ + + __namespace: str + __cni_type: str + __cni_master_interface: str + __image_pull_policy: str + __image_registry_prefix: str + __manifests: List[Dict[str, Any]] + __image_entries: List[Dict[str, str]] + + def __init__( + self, + image_registry_prefix: str = "seedemu", + registry_prefix: str | None = None, + namespace: str = "seedemu", + use_multus: bool = True, + cni_type: str = "kube-ovn", + local_link_cni_type: str | None = None, + cni_master_interface: str = "ens2", + image_pull_policy: str = "Always", + scheduling_strategy: str = SchedulingStrategy.NONE, + node_labels: Dict[str, Dict[str, str]] | None = None, + default_resources: Dict[str, Dict[str, str]] | None = None, + generate_services: bool = False, + service_type: str = "ClusterIP", + **kwargs: Any, + ) -> None: + kwargs["selfManagedNetwork"] = True + super().__init__(**kwargs) + if not use_multus: + raise ValueError("KubernetesCompiler requires Multus network attachments.") + if registry_prefix is not None: + image_registry_prefix = registry_prefix + self.__image_registry_prefix = image_registry_prefix.strip().strip("/") + self.__namespace = namespace.strip() or "seedemu" + self.__cni_type = self._normalizeCniType(cni_type) + if local_link_cni_type is not None and str(local_link_cni_type).strip(): + self._normalizeCniType(str(local_link_cni_type)) + self.__cni_master_interface = cni_master_interface.strip() or "eth0" + self.__image_pull_policy = image_pull_policy.strip() or "Always" + self.__manifests = [] + self.__image_entries = [] + + def getName(self) -> str: + return "Kubernetes" + + def getManifestName(self) -> str: + """Return the manifest filename generated by this compiler.""" + if self._usesKubeOvn(): + return "k8s.kube-ovn.yaml" + return "k8s.yaml" + + @staticmethod + def _normalizeCniType(cni_type: str | None) -> str: + """Normalize and validate the secondary CNI mode.""" + value = (cni_type or "kube-ovn").strip().lower().replace("_", "-") + if value in {"kube-ovn", "ovn"}: + return "kube-ovn" + if value == "macvlan": + return "macvlan" + raise ValueError( + f"KubernetesCompiler supports only cni_type \"kube-ovn\" or " + f"cni_type \"macvlan\"; got {cni_type!r}." + ) + + @staticmethod + def _safeBridgeName(name: str) -> str: + return f"br-{md5(name.encode()).hexdigest()[:12]}" + + def _usesKubeOvn(self) -> bool: + """Return true when the compiler should emit Kube-OVN resources.""" + return self.__cni_type == "kube-ovn" + + def _doCompile(self, emulator) -> None: + registry = emulator.getRegistry() + self._groupSoftware(emulator) + + self.__manifests.append( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": self.__namespace}, + } + ) + + for ((_, obj_type, _), obj) in registry.getAll().items(): + if obj_type == "net": + self.__manifests.append(self._compileNetK8s(obj)) + + for ((_, obj_type, _), obj) in registry.getAll().items(): + if obj_type in SEEDEMU_NODE_TYPES: + self.__manifests.append(self._compileNodeK8s(obj)) + + manifests = ( + self._renderKubeOvnManifests(self.__manifests) + if self._usesKubeOvn() + else self.__manifests + ) + with open(self.getManifestName(), "w", encoding="utf-8") as handle: + yaml.safe_dump_all(manifests, handle, sort_keys=False) + + self._stage_base_image_contexts() + with open("images.yaml", "w", encoding="utf-8") as handle: + yaml.safe_dump({"images": self.__image_entries}, handle, sort_keys=False) + + def _compileNetK8s(self, net: Network) -> Dict[str, Any]: + name = self._getRealNetName(net).replace("_", "-").lower() + + if self._usesKubeOvn(): + config = { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": f"{name}.{self.__namespace}.ovn", + } + elif self.__cni_type == "macvlan": + config = { + "cniVersion": "0.3.1", + "type": "macvlan", + "master": self.__cni_master_interface, + "mode": "bridge", + "ipam": {"type": "static"}, + } + else: + raise ValueError(f"unsupported Kubernetes CNI type: {self.__cni_type}") + + return { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": name, + "namespace": self.__namespace, + "annotations": self._getInternetMapNetMeta(net), + }, + "spec": {"config": json.dumps(config)}, + } + + def _renderKubeOvnManifests(self, manifests: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert compiler manifests to Kube-OVN secondary-network resources. + + Args: + manifests: Namespace, NAD, and workload manifests generated by the + compiler before fabric-specific conversion. + + Returns: + Manifest list containing a Kube-OVN Vpc, one Subnet per SeedEMU + network, kube-ovn NADs, and workloads with provider IP annotations. + """ + namespace_name = self._findManifestNamespace(manifests) + vpc_name = self._kubeOvnResourceName("vpc", namespace_name) + rendered: List[Dict[str, Any]] = [] + vpc_written = False + + for manifest in manifests: + if manifest.get("kind") == "Namespace" and not vpc_written: + rendered.append(manifest) + rendered.append(self._kubeOvnVpc(namespace_name, vpc_name)) + vpc_written = True + continue + if manifest.get("kind") == "NetworkAttachmentDefinition": + rendered.extend(self._convertNetworkAttachmentToKubeOvn(manifest, namespace_name, vpc_name)) + continue + self._convertWorkloadAnnotationsToKubeOvn(manifest, namespace_name) + rendered.append(manifest) + + if not vpc_written: + rendered.insert(0, self._kubeOvnVpc(namespace_name, vpc_name)) + return rendered + + def _findManifestNamespace(self, manifests: List[Dict[str, Any]]) -> str: + """Return the namespace used by workload manifests.""" + for manifest in manifests: + if manifest.get("kind") == "Namespace": + name = (manifest.get("metadata") or {}).get("name") + if name: + return str(name) + for manifest in manifests: + metadata = manifest.get("metadata") or {} + name = metadata.get("namespace") + if name: + return str(name) + return self.__namespace + + def _kubeOvnResourceName(self, prefix: str, value: str) -> str: + """Return a DNS-safe Kube-OVN cluster-scoped resource name.""" + safe = "".join(ch if ch.isalnum() or ch == "-" else "-" for ch in value.lower()).strip("-") + digest = sha1(value.encode("utf-8")).hexdigest()[:8] + return f"{prefix}-{digest}-{safe}"[:63].rstrip("-") + + def _kubeOvnVpc(self, namespace_name: str, vpc_name: str) -> Dict[str, Any]: + """Return a Kube-OVN Vpc that isolates one SeedEMU namespace.""" + return { + "apiVersion": "kubeovn.io/v1", + "kind": "Vpc", + "metadata": {"name": vpc_name}, + "spec": {"namespaces": [namespace_name]}, + } + + def _convertNetworkAttachmentToKubeOvn( + self, + manifest: Dict[str, Any], + default_namespace: str, + vpc_name: str, + ) -> List[Dict[str, Any]]: + """Return [Subnet, NAD] for one SeedEMU network.""" + metadata = manifest.get("metadata") or {} + nad_name = str(metadata.get("name") or "") + namespace_name = str(metadata.get("namespace") or default_namespace) + if not nad_name: + return [manifest] + annotations = metadata.get("annotations") or {} + prefix = annotations.get(self._metaKey("prefix")) + if not prefix: + raise ValueError(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") + + provider = f"{nad_name}.{namespace_name}.ovn" + subnet_name = self._kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") + subnet = { + "apiVersion": "kubeovn.io/v1", + "kind": "Subnet", + "metadata": {"name": subnet_name}, + "spec": { + "protocol": "IPv4", + "provider": provider, + "vpc": vpc_name, + "cidrBlock": str(prefix), + "gateway": self._firstUsableIp(str(prefix)), + "gatewayType": "distributed", + "natOutgoing": False, + "private": False, + }, + } + + converted = dict(manifest) + converted["spec"] = { + "config": json.dumps( + { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + }, + separators=(",", ":"), + ) + } + return [subnet, converted] + + def _convertWorkloadAnnotationsToKubeOvn( + self, + manifest: Dict[str, Any], + default_namespace: str, + ) -> None: + """Rewrite Multus static IP annotations for Kube-OVN attached NICs.""" + annotations = self._getPodTemplateAnnotations(manifest) + if not annotations: + return + raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") + if not isinstance(raw_networks, str): + return + try: + networks = json.loads(raw_networks) + except json.JSONDecodeError: + return + if not isinstance(networks, list): + return + + changed = False + for item in networks: + if not isinstance(item, dict): + continue + ips = item.pop("ips", None) + if not ips: + continue + nad_name, nad_namespace = self._parseNetworkSelection(item, default_namespace) + if not nad_name: + continue + ip_values = [str(ip_value).strip().split("/", 1)[0] for ip_value in ips if str(ip_value).strip()] + if not ip_values: + continue + annotations[f"{nad_name}.{nad_namespace}.ovn.kubernetes.io/ip_address"] = ",".join(ip_values) + changed = True + + if changed: + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) + + def _getPodTemplateAnnotations(self, manifest: Dict[str, Any]) -> Dict[str, str] | None: + """Return pod-level annotations from a workload or Pod manifest.""" + kind = manifest.get("kind") + if kind == "Pod": + metadata = manifest.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + if kind in {"Deployment", "DaemonSet", "StatefulSet", "Job"}: + template = manifest.setdefault("spec", {}).setdefault("template", {}) + metadata = template.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + return None + + def _parseNetworkSelection(self, item: Dict[str, Any], default_namespace: str) -> tuple[str, str]: + """Return (nad_name, namespace) from one Multus network selection item.""" + raw_name = str(item.get("name") or "") + namespace = str(item.get("namespace") or default_namespace) + if "/" in raw_name: + namespace, raw_name = raw_name.split("/", 1) + return raw_name, namespace + + def _firstUsableIp(self, cidr: str) -> str: + """Return the first usable IPv4 address in a CIDR block.""" + network = ipaddress.ip_network(cidr, strict=False) + if network.version != 4: + raise ValueError(f"Kube-OVN manifest generation currently supports IPv4 only: {cidr}") + hosts = network.hosts() + try: + return str(next(hosts)) + except StopIteration: + return str(network.network_address) + + def _compileNodeK8s(self, node: Node) -> Dict[str, Any]: + real_nodename = self._getRealNodeName(node) + if not os.path.exists(real_nodename): + mkdir(real_nodename) + + cwd = os.getcwd() + chdir(real_nodename) + dockerfile = self._computeDockerfile(node) + with open("Dockerfile", "w", encoding="utf-8") as handle: + handle.write(dockerfile) + self._patch_interface_setup_context(Path("Dockerfile")) + chdir(cwd) + + full_image_name = f"{self.__image_registry_prefix}/{real_nodename}:latest" + self.__image_entries.append( + { + "name": full_image_name, + "context": f"./{real_nodename}", + } + ) + + node_name = self._getComposeNodeName(node).replace("_", "-").lower() + asn = str(node.getAsn()) + role = self._nodeRoleToString(node.getRole()) + + annotations = self._getInternetMapNodeMeta(node) + net_specs = [] + for iface in node.getInterfaces(): + net = iface.getNet() + net_name = self._getRealNetName(net).replace("_", "-").lower() + prefix_len = net.getPrefix().prefixlen + net_specs.append({"name": net_name, "ips": [f"{iface.getAddress()}/{prefix_len}"]}) + if net_specs: + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(net_specs) + + envs = [{"name": "CONTAINER_NAME", "value": node_name}] + for opt, _scope in node.getScopedRuntimeOptions(): + envs.append({"name": opt.name.upper(), "value": str(opt.value)}) + + labels = { + "app": node_name, + "seedemu.io/asn": asn, + "seedemu.io/role": role, + "seedemu.io/name": node.getName(), + "seedemu.io/workload": "seedemu", + } + + return { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": node_name, + "namespace": self.__namespace, + "labels": labels, + }, + "spec": { + "replicas": 1, + "selector": {"matchLabels": {"app": node_name}}, + "template": { + "metadata": {"labels": labels, "annotations": annotations}, + "spec": { + "containers": [ + { + "name": "main", + "image": full_image_name, + "imagePullPolicy": self.__image_pull_policy, + "securityContext": { + "privileged": True, + "capabilities": {"add": ["ALL"]}, + }, + "command": ["/start.sh"], + "env": envs, + "volumeMounts": [], + } + ] + }, + }, + }, + } + + def _metaKey(self, key: str) -> str: + return f"{INTERNET_MAP_META_PREFIX}{key}" + + def _nodeInternetMapRole(self, node: Node) -> str: + _scope, obj_type, _name = node.getRegistryInfo() + if obj_type == "hnode": + return "Host" + if obj_type == "rnode": + return "Router" + if obj_type == "brdnode": + return "BorderRouter" + if obj_type == "csnode": + return "SCION Control Service" + if obj_type == "snode": + return "Emulator Service Worker" + if obj_type == "rs": + return "Route Server" + return obj_type + + def _getInternetMapNodeMeta(self, node: Node) -> Dict[str, str]: + _scope, _obj_type, name = node.getRegistryInfo() + meta = { + self._metaKey("asn"): str(node.getAsn()), + self._metaKey("nodename"): str(name), + self._metaKey("role"): self._nodeInternetMapRole(node), + } + + if node.getDisplayName() is not None: + meta[self._metaKey("displayname")] = str(node.getDisplayName()) + if node.getDescription() is not None: + meta[self._metaKey("description")] = str(node.getDescription()) + if len(node.getClasses()) > 0: + meta[self._metaKey("class")] = json.dumps(node.getClasses()) + + for key, value in node.getLabel().items(): + meta[self._metaKey(key)] = str(value) + + for index, iface in enumerate(node.getInterfaces()): + net = iface.getNet() + meta[self._metaKey(f"net.{index}.name")] = str(net.getName()) + meta[self._metaKey(f"net.{index}.address")] = ( + f"{iface.getAddress()}/{net.getPrefix().prefixlen}" + ) + + return meta + + def _getInternetMapNetMeta(self, net: Network) -> Dict[str, str]: + scope, _obj_type, name = net.getRegistryInfo() + meta = { + self._metaKey("type"): "global" if scope == "ix" else "local", + self._metaKey("scope"): str(scope), + self._metaKey("name"): str(name), + self._metaKey("prefix"): str(net.getPrefix()), + } + + if net.getDisplayName() is not None: + meta[self._metaKey("displayname")] = str(net.getDisplayName()) + if net.getDescription() is not None: + meta[self._metaKey("description")] = str(net.getDescription()) + + return meta + + def _patch_interface_setup_context(self, dockerfile_path: Path) -> None: + dockerfile = dockerfile_path.read_text(encoding="utf-8") + match = re.search(r"^COPY\s+(\S+)\s+/interface_setup\s*$", dockerfile, re.MULTILINE) + if not match: + return + + script_path = dockerfile_path.parent / match.group(1) + if not script_path.exists(): + return + + script_path.write_text( + """#!/bin/bash +set -euo pipefail + +cidr_to_net() { + ipcalc -n "$1" | sed -E -n 's/^Network: +([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\/[0-9]{1,2}) +.*/\\1/p' +} + +tmpfile="$(mktemp)" +trap 'rm -f "$tmpfile"' EXIT + +ip -j addr | jq -cr '.[]' | while read -r iface; do + ifname="$(jq -cr '.ifname' <<< "$iface")" + jq -cr '.addr_info[]?' <<< "$iface" | while read -r iaddr; do + addr="$(jq -cr '"\\(.local)/\\(.prefixlen)"' <<< "$iaddr")" + net="$(cidr_to_net "$addr")" + [ -z "$net" ] && continue + line="$(grep -F "$net" ifinfo.txt || true)" + [ -z "$line" ] && continue + new_ifname="$(cut -d: -f1 <<< "$line")" + latency="$(cut -d: -f3 <<< "$line")" + bw="$(cut -d: -f4 <<< "$line")" + loss="$(cut -d: -f5 <<< "$line")" + [ -z "$new_ifname" ] && continue + [ "$bw" = 0 ] && bw=1000000000000 + printf '%s|%s|%s|%s|%s\\n' "$ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "$tmpfile" + done +done + +i=0 +while IFS='|' read -r ifname new_ifname latency bw loss; do + [ -z "$ifname" ] && continue + tmp_ifname="seedtmp${i}" + if [ "$ifname" != "$new_ifname" ]; then + ip link set "$ifname" down + ip link set "$ifname" name "$tmp_ifname" + else + tmp_ifname="$ifname" + fi + printf '%s|%s|%s|%s|%s\\n' "$tmp_ifname" "$new_ifname" "$latency" "$bw" "$loss" >> "${tmpfile}.stage2" + i=$((i + 1)) +done < "$tmpfile" + +while IFS='|' read -r tmp_ifname new_ifname latency bw loss; do + [ -z "$tmp_ifname" ] && continue + if [ "$tmp_ifname" != "$new_ifname" ]; then + ip link set "$tmp_ifname" name "$new_ifname" + fi + ip link set "$new_ifname" up + [ -z "$loss" ] && loss=0 + tc qdisc add dev "$new_ifname" root handle 1:0 tbf rate "${bw}bit" buffer 1000000 limit 1000 + tc qdisc add dev "$new_ifname" parent 1:0 handle 10: netem delay "${latency}ms" loss "${loss}%" +done < "${tmpfile}.stage2" + +rm -f "${tmpfile}.stage2" +""", + encoding="utf-8", + ) + + def _stage_base_image_contexts(self) -> None: + used_images = sorted(getattr(self, "_used_images", set())) + if os.path.isdir("base_images"): + shutil.rmtree("base_images") + if not used_images: + return + + for image in used_images: + digest = md5(image.encode("utf-8")).hexdigest() + dst_root = os.path.join("base_images", digest) + os.makedirs(dst_root, exist_ok=True) + dockerfile = os.path.join(dst_root, "Dockerfile") + with open(dockerfile, "w", encoding="utf-8") as handle: + handle.write(f"FROM {image}\n") + +NativeKubernetesCompiler = KubernetesCompiler diff --git a/seedemu/core/AutonomousSystem.py b/seedemu/core/AutonomousSystem.py index b6c3e6ebf..62563b087 100644 --- a/seedemu/core/AutonomousSystem.py +++ b/seedemu/core/AutonomousSystem.py @@ -4,18 +4,49 @@ from .Network import Network from .AddressAssignmentConstraint import AddressAssignmentConstraint from .enums import NetworkType, NodeRole -from .Node import Node, Router +from .Node import Node, Router, ROUTER_BGP_ROLE_EDGE from .Scope import ScopeTier, Scope from .Emulator import Emulator from .Configurable import Configurable from .Customizable import Customizable from .Node import promote_to_real_world_router from ipaddress import IPv4Network -from typing import Dict, List +from typing import Dict, List, Optional, Set, Tuple import requests RIS_PREFIXLIST_URL = 'https://stat.ripe.net/data/announced-prefixes/data.json' +IBGP_MODE_FULL_MESH = "full-mesh" +IBGP_MODE_ROUTE_REFLECTOR = "route-reflector" +IBGP_MODE_DISABLED = "disabled" + +IBGP_MODES = { + IBGP_MODE_FULL_MESH, + IBGP_MODE_ROUTE_REFLECTOR, + IBGP_MODE_DISABLED, +} + +BGP_SCOPE_ALL_ROUTERS = "all-routers" +BGP_SCOPE_EDGE_ONLY = "edge-only" +BGP_SCOPES = {BGP_SCOPE_ALL_ROUTERS, BGP_SCOPE_EDGE_ONLY} + +CORE_FORWARDING_PLAIN_IP = "plain-ip" +CORE_FORWARDING_MPLS = "mpls" +CORE_FORWARDING_SR = "sr" +CORE_FORWARDING_TUNNEL = "tunnel" +CORE_FORWARDING_REDISTRIBUTE = "redistribute" +CORE_FORWARDING_MODES = { + CORE_FORWARDING_PLAIN_IP, + CORE_FORWARDING_MPLS, + CORE_FORWARDING_SR, + CORE_FORWARDING_TUNNEL, + CORE_FORWARDING_REDISTRIBUTE, +} + +OSPF_MODE_LEGACY = "legacy" +OSPF_MODE_ROUTER_TRANSIT_ONLY = "router-transit-only" +OSPF_MODES = {OSPF_MODE_LEGACY, OSPF_MODE_ROUTER_TRANSIT_ONLY} + class AutonomousSystem(Printable, Graphable, Configurable, Customizable): """! @brief AutonomousSystem class. @@ -29,6 +60,13 @@ class AutonomousSystem(Printable, Graphable, Configurable, Customizable): __hosts: Dict[str, Node] __nets: Dict[str, Network] __name_servers: List[str] + __clusters: Dict[str, Tuple[Set[str], Set[str]]] + __ibgp_mode: str + __ibgp_mode_explicit: bool + __bgp_scope: str + __core_forwarding: str + __ospf_mode: str + __ospf_mode_explicit: bool def __init__(self, asn: int, subnetTemplate: str = "10.{}.0.0/16"): """! @@ -44,7 +82,351 @@ def __init__(self, asn: int, subnetTemplate: str = "10.{}.0.0/16"): self.__asn = asn self.__subnets = None if asn > 255 else list(IPv4Network(subnetTemplate.format(asn)).subnets(new_prefix = 24)) self.__name_servers = [] + self.__clusters = {} + self.__ibgp_mode = IBGP_MODE_FULL_MESH + self.__ibgp_mode_explicit = False + self.__bgp_scope = BGP_SCOPE_ALL_ROUTERS + self.__core_forwarding = CORE_FORWARDING_PLAIN_IP + self.__ospf_mode = OSPF_MODE_LEGACY + self.__ospf_mode_explicit = False + + def setIbgpMode(self, mode: str) -> AutonomousSystem: + """! + @brief Set the AS-level iBGP route propagation mode. + + The default is full-mesh when unset. This setting records AS + intent; the Ibgp layer still renders concrete BGP sessions. + + @param mode full-mesh, route-reflector, or disabled. + + @returns self, for chaining API calls. + """ + value = self._normalizeIbgpMode(mode) + assert value in IBGP_MODES, "unsupported iBGP mode: {}. valid values: {}".format( + mode, sorted(IBGP_MODES) + ) + self.__ibgp_mode = value + self.__ibgp_mode_explicit = True + return self + + def _normalizeIbgpMode(self, mode: str) -> str: + value = str(mode or IBGP_MODE_FULL_MESH).strip().lower() + return value + + def getIbgpMode(self) -> str: + """! + @brief Get the AS-level iBGP route propagation mode. + """ + return self.__ibgp_mode + + def hasIbgpMode(self) -> bool: + """! + @brief Return whether an iBGP mode was explicitly set on this AS. + """ + return self.__ibgp_mode_explicit + def setBgpScope(self, scope: str) -> AutonomousSystem: + """! + @brief Set which routers participate in BGP/iBGP inside this AS. + + @param scope all-routers or edge-only. + + @returns self, for chaining API calls. + """ + value = str(scope or BGP_SCOPE_ALL_ROUTERS).strip().lower() + assert value in BGP_SCOPES, "unsupported BGP scope: {}. valid values: {}".format( + scope, sorted(BGP_SCOPES) + ) + self.__bgp_scope = value + return self + + def getBgpScope(self) -> str: + """! + @brief Get the AS-level BGP participation scope. + """ + return self.__bgp_scope + + def setCoreForwarding(self, mode: str) -> AutonomousSystem: + """! + @brief Set the forwarding mechanism expected for a BGP-free core. + + @param mode plain-ip, mpls, sr, tunnel, or redistribute. + + @returns self, for chaining API calls. + """ + value = str(mode or CORE_FORWARDING_PLAIN_IP).strip().lower() + assert value in CORE_FORWARDING_MODES, "unsupported core forwarding mode: {}. valid values: {}".format( + mode, sorted(CORE_FORWARDING_MODES) + ) + self.__core_forwarding = value + return self + + def getCoreForwarding(self) -> str: + """! + @brief Get the AS-level core forwarding mode. + """ + return self.__core_forwarding + + def setIbgpDesign( + self, + mode: str = IBGP_MODE_FULL_MESH, + scope: str = BGP_SCOPE_ALL_ROUTERS, + core_forwarding: str = CORE_FORWARDING_PLAIN_IP, + ) -> AutonomousSystem: + """! + @brief Set iBGP mode, BGP scope, and core forwarding together. + """ + self.setIbgpMode(mode) + self.setBgpScope(scope) + self.setCoreForwarding(core_forwarding) + return self + + def setOspfMode(self, mode: str) -> AutonomousSystem: + """! + @brief Set the AS-level OSPF interface classification mode. + + The default is legacy when unset. Ospf records interface intent and + Routing renders backend-specific BIRD/FRR configuration. + + @param mode legacy or router-transit-only. + + @returns self, for chaining API calls. + """ + value = str(mode or OSPF_MODE_LEGACY).strip().lower() + assert value in OSPF_MODES, "unsupported OSPF mode: {}".format(mode) + self.__ospf_mode = value + self.__ospf_mode_explicit = True + return self + + def getOspfMode(self) -> str: + """! + @brief Get the AS-level OSPF interface classification mode. + """ + return self.__ospf_mode + + def hasOspfMode(self) -> bool: + """! + @brief Return whether an OSPF mode was explicitly set on this AS. + """ + return self.__ospf_mode_explicit + + def createBgpCluster(self, address: str) -> AutonomousSystem: + """! + @brief Register an iBGP Route Reflector cluster ID for this AS. + + The cluster starts with empty RR/client membership. Calling this method + with an existing cluster ID is idempotent. + + @param address cluster ID rendered into BIRD's rr cluster id field. + + @returns self, for chaining API calls. + """ + self.__ensureRouteReflectorModeAllowed("createBgpCluster") + if not self.__ibgp_mode_explicit: + self.__ibgp_mode = IBGP_MODE_ROUTE_REFLECTOR + self.__ibgp_mode_explicit = True + if address not in self.__clusters: + self.__clusters[address] = (set(), set()) + + return self + + def __ensureRouteReflectorModeAllowed(self, api_name: str) -> None: + if self.__ibgp_mode_explicit and self.__ibgp_mode != IBGP_MODE_ROUTE_REFLECTOR: + raise AssertionError( + "AS{} cannot call {} while ibgp_mode is {}".format( + self.__asn, api_name, self.__ibgp_mode + ) + ) + + def __hasRouteReflectorRouterConfig(self) -> bool: + for router in self.__routers.values(): + if router.getBgpClusterId() is not None or router.isRouteReflector(): + return True + return False + + def __hasRouteReflectorConfig(self) -> bool: + return len(self.__clusters) > 0 or self.__hasRouteReflectorRouterConfig() + + def __defaultBgpClusterId(self) -> str: + if self.__asn <= 255: + base = "10.{}.0".format(self.__asn) + else: + base = "10.{}.{}".format((self.__asn // 256) % 256, self.__asn % 256) + + for suffix in range(1, 255): + candidate = "{}.{}".format(base, suffix) + if candidate not in self.__clusters: + return candidate + + raise AssertionError("AS{} cannot allocate a default BGP cluster ID".format(self.__asn)) + + def __getIbgpParticipantRouters(self) -> List[Router]: + routers = list(self.__routers.values()) + if self.__bgp_scope == BGP_SCOPE_ALL_ROUTERS: + return routers + + participants = [ + router for router in routers + if ( + (hasattr(router, "getBgpRole") and router.getBgpRole() == ROUTER_BGP_ROLE_EDGE) + or router.isBorderRouter() + ) + ] + assert len(participants) > 0, ( + "AS{} has bgp_scope=edge-only but no edge routers were found; " + "mark routers with setBgpRole('edge') or connect them to IXes".format(self.__asn) + ) + return participants + + def getIbgpParticipants(self) -> Set[str]: + """! + @brief Get router names participating in the AS iBGP design. + """ + return {router.getName() for router in self.__getIbgpParticipantRouters()} + + def _validate_cluster_integrity(self, data: Dict[str, Tuple[Set[str], Set[str]]]): + """! + @brief Validate Route Reflector cluster membership. + + @param data mapping from cluster ID to RR names and client names. + """ + for cid, (rr_set, client_set) in data.items(): + assert len(rr_set) > 0, ( + "[Topology Error] AS{} cluster '{}' is invalid: missing Route " + "Reflector.".format(self.__asn, cid) + ) + assert len(client_set) > 0, ( + "[Topology Error] AS{} cluster '{}' is invalid: missing clients. " + "Route Reflector(s) {} have no clients to serve.".format( + self.__asn, cid, sorted(rr_set) + ) + ) + + def completeIbgpSetup(self) -> AutonomousSystem: + """! + @brief Complete and validate the AS-level iBGP design. + + The AS owns iBGP intent. The Ibgp layer only renders the completed + design into router configuration. + """ + mode = self.getIbgpMode() + has_rr_config = self.__hasRouteReflectorConfig() + + if mode == IBGP_MODE_FULL_MESH: + if self.hasIbgpMode() and has_rr_config: + raise AssertionError( + "AS{} has route-reflector cluster/router configuration but ibgp_mode is full-mesh".format( + self.__asn + ) + ) + if not self.hasIbgpMode() and has_rr_config: + self.__ibgp_mode = IBGP_MODE_ROUTE_REFLECTOR + mode = IBGP_MODE_ROUTE_REFLECTOR + + if mode == IBGP_MODE_DISABLED: + return self + + participants = self.__getIbgpParticipantRouters() + if mode == IBGP_MODE_FULL_MESH: + return self + + assert mode == IBGP_MODE_ROUTE_REFLECTOR, "unsupported iBGP mode: {}".format(mode) + + explicit_clusters = len(self.__clusters) > 0 + advanced_multi_cluster = len(self.__clusters) > 1 + + if not explicit_clusters: + self.createBgpCluster(self.__defaultBgpClusterId()) + + if advanced_multi_cluster: + self.__completeAdvancedRouteReflectorSetup(participants) + else: + self.__completeDefaultRouteReflectorSetup(participants) + + return self + + def __completeDefaultRouteReflectorSetup(self, participants: List[Router]) -> None: + assert len(participants) > 0, "AS{} route-reflector mode requires at least one router".format(self.__asn) + cluster_id = sorted(self.__clusters.keys())[0] + + for router in participants: + if router.getBgpClusterId() is None: + router.joinBgpCluster(cluster_id) + assert router.getBgpClusterId() == cluster_id, ( + "AS{} router {} joined unknown or non-default cluster {}".format( + self.__asn, router.getName(), router.getBgpClusterId() + ) + ) + + rrs = [router for router in participants if router.isRouteReflector()] + if not rrs: + sorted(participants, key=lambda router: router.getName())[0].makeRouteReflector() + + self._aggregateBgpClusters(validate=True) + + def __completeAdvancedRouteReflectorSetup(self, participants: List[Router]) -> None: + cluster_ids = set(self.__clusters.keys()) + participant_names = {router.getName() for router in participants} + + for router in participants: + cluster_id = router.getBgpClusterId() + assert cluster_id is not None, ( + "Router {} in AS{} is missing a cluster ID".format(router.getName(), self.__asn) + ) + assert cluster_id in cluster_ids, ( + "Router {} in AS{} joined unknown cluster {}".format( + router.getName(), self.__asn, cluster_id + ) + ) + + data = self._aggregateBgpClusters(validate=False) + data = { + cid: (rrs & participant_names, clients & participant_names) + for cid, (rrs, clients) in data.items() + } + self._validate_cluster_integrity(data) + self.__clusters = data + + def _aggregateBgpClusters(self, validate: bool = True) -> Dict[str, Tuple[Set[str], Set[str]]]: + """! + @brief Build Route Reflector cluster membership from AS/router state. + + Explicitly registered clusters provide the allowed cluster IDs. Routers + that joined a cluster are assigned to that cluster; routers without an + explicit cluster are assigned to the implicit default cluster. + + @returns mapping from cluster ID to RR names and client names. + """ + merged_data = { + cid: (set(rrs), set(clients)) + for cid, (rrs, clients) in self.__clusters.items() + } + + for router in self.__getIbgpParticipantRouters(): + r_cid = router.getBgpClusterId() + is_rr = router.isRouteReflector() + r_name = router.getName() + + if r_cid is not None: + assert r_cid in merged_data, "Cluster ID {} does not exist in AS{}.".format( + r_cid, self.__asn + ) + target_cid = r_cid + else: + assert len(merged_data) == 1, ( + "Router {} in AS{} is missing a cluster ID".format(r_name, self.__asn) + ) + target_cid = next(iter(merged_data.keys())) + + if is_rr: + merged_data[target_cid][0].add(r_name) + else: + merged_data[target_cid][1].add(r_name) + + if validate: + self._validate_cluster_integrity(merged_data) + self.__clusters = merged_data + return self.__clusters def setNameServers(self, servers: List[str]) -> AutonomousSystem: @@ -209,15 +591,16 @@ def getNetworks(self) -> List[str]: """ return list(self.__nets.keys()) - def createRouter(self, name: str) -> Node: + def createRouter(self, name: str, routingBackend: str = "bird") -> Node: """! @brief Create a router node. @param name name of the new node. + @param routingBackend routing daemon backend, bird or frr. Default to bird. @returns Node. """ assert name not in self.__routers, 'Router with name {} already exists.'.format(name) - self.__routers[name] = Router(name, NodeRole.Router, self.__asn) + self.__routers[name] = Router(name, NodeRole.Router, self.__asn, routingBackend=routingBackend) return self.__routers[name] @@ -384,4 +767,4 @@ def createOpenVpnRouter(self, name: str) -> Node: assert name not in self.__routers, 'Router with name {} already exists.'.format(name) self.__routers[name] = Router(name, NodeRole.OpenVpnRouter, self.__asn) - return self.__routers[name] \ No newline at end of file + return self.__routers[name] diff --git a/seedemu/core/BaseSystem.py b/seedemu/core/BaseSystem.py index f41e8e58d..e4578cac8 100644 --- a/seedemu/core/BaseSystem.py +++ b/seedemu/core/BaseSystem.py @@ -17,6 +17,7 @@ class BaseSystem(Enum): SEEDEMU_ETHEREUM_LEGACY = 'seedemu-ethereum-legacy' SEEDEMU_ETHEREUM_POS = 'seedemu-ethereum-pos' SEEDEMU_MONERO = 'seedemu-monero' + SEEDEMU_SOLANA = 'seedemu-solana' SEEDEMU_OP_STACK = 'seedemu-op-stack' SEEDEMU_SC_DEPLOYER = 'seedemu-sc-deployer' SEEDEMU_CHAINLINK = 'seedemu-chainlink' @@ -32,6 +33,7 @@ class BaseSystem(Enum): SEEDEMU_ETHEREUM_LEGACY: [UBUNTU_20_04, SEEDEMU_BASE], SEEDEMU_ETHEREUM_POS: [UBUNTU_20_04, SEEDEMU_BASE], SEEDEMU_MONERO: [UBUNTU_20_04, SEEDEMU_BASE], + SEEDEMU_SOLANA: [UBUNTU_20_04, SEEDEMU_BASE], SEEDEMU_OP_STACK: [UBUNTU_20_04, SEEDEMU_BASE], SEEDEMU_SC_DEPLOYER: [UBUNTU_20_04, SEEDEMU_BASE], SEEDEMU_CHAINLINK: [UBUNTU_20_04, SEEDEMU_BASE], diff --git a/seedemu/core/Node.py b/seedemu/core/Node.py index 108d12f80..763968b71 100644 --- a/seedemu/core/Node.py +++ b/seedemu/core/Node.py @@ -18,7 +18,11 @@ from random import choice from .BaseSystem import BaseSystem -DEFAULT_SOFTWARE: List[str] = ['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat'] +DEFAULT_SOFTWARE: List[str] = ['zsh', 'curl', 'nano', 'vim-nox', 'mtr-tiny', 'iproute2', 'iputils-ping', 'tcpdump', 'termshark', 'dnsutils', 'jq', 'ipcalc', 'netcat-openbsd'] + +ROUTER_BGP_ROLE_EDGE = "edge" +ROUTER_BGP_ROLE_CORE = "core" +ROUTER_BGP_ROLES = {ROUTER_BGP_ROLE_EDGE, ROUTER_BGP_ROLE_CORE} class File(Printable): """! @@ -1134,14 +1138,118 @@ class Router(Node): __loopback_address: str __is_border_router: bool - + __is_bgp_rr: bool + __bgp_cluster_id: Optional[str] + __routing_backend: str + __bgp_role: Optional[str] + __disabled_control_planes: Set[str] __extensions: Dict[str, RouterExtension] - def __init__(self, name: str, role: NodeRole, asn: int, scope: str = None): + def __init__(self, name: str, role: NodeRole, asn: int, scope: str = None, routingBackend: str = "bird"): self.__is_border_router = False self.__loopback_address = None + self.__is_bgp_rr = False + self.__bgp_cluster_id = None + self.__routing_backend = "bird" + self.__bgp_role = None + self.__disabled_control_planes = set() self.__extensions = {} super().__init__( name,role,asn,scope) + self.setRoutingBackend(routingBackend) + + def setBgpRole(self, role: str) -> Router: + """! + @brief Set this router's BGP participation role. + + This does not change the structural NodeRole. It is an AS-level routing + hint used by iBGP designs such as edge-only BGP with a BGP-free core. + + @param role edge or core. + + @returns self, for chaining API calls. + """ + value = str(role or "").strip().lower() + assert value in ROUTER_BGP_ROLES, "unsupported BGP role: {}. valid values: {}".format( + role, sorted(ROUTER_BGP_ROLES) + ) + self.__bgp_role = value + self.setLabel("seedemu_bgp_role", value) + return self + + def getBgpRole(self) -> Optional[str]: + """! + @brief Get this router's optional BGP participation role. + + @returns role name, or None if no role was set. + """ + return self.__bgp_role + + def disableControlPlane(self, protocol: str) -> Router: + """! + @brief Disable a protocol-layer participation hint on this router. + + The first supported flag is ibgp. It lets new opt-in iBGP modes exclude + a specific router without changing AS-wide legacy defaults. + + @param protocol protocol participation flag to disable. + + @returns self, for chaining API calls. + """ + value = str(protocol or "").strip().lower() + assert value in {"ibgp"}, "unsupported router control-plane disable flag: {}".format(protocol) + self.__disabled_control_planes.add(value) + self.setLabel("seedemu_control_plane_disabled_{}".format(value), "true") + return self + + def isControlPlaneDisabled(self, protocol: str) -> bool: + """! + @brief Check whether a protocol-layer participation hint is disabled. + """ + return str(protocol or "").strip().lower() in self.__disabled_control_planes + + def getDisabledControlPlanes(self) -> Set[str]: + """! + @brief Get disabled protocol-layer participation hints. + """ + return set(self.__disabled_control_planes) + + def makeRouteReflector(self, is_rr: bool = True) -> Router: + """! + @brief Mark this router as an iBGP Route Reflector. + + @param is_rr whether this router should act as a Route Reflector. + + @returns self, for chaining API calls. + """ + self.__is_bgp_rr = is_rr + return self + + def joinBgpCluster(self, cluster_id: str) -> Router: + """! + @brief Assign this router to an iBGP Route Reflector cluster. + + @param cluster_id cluster ID previously registered on the AS. + + @returns self, for chaining API calls. + """ + self.__bgp_cluster_id = cluster_id + return self + + def getBgpClusterId(self) -> Optional[str]: + """! + @brief Get the configured Route Reflector cluster ID. + + @returns cluster ID, or None if the router uses the default cluster. + """ + return self.__bgp_cluster_id + + def isRouteReflector(self) -> bool: + """! + @brief Check whether this router is configured as a Route Reflector. + + @returns True if the router acts as an iBGP Route Reflector. + """ + return self.__is_bgp_rr def hasExtension(self, name: str) -> bool: return name in self.__extensions @@ -1170,6 +1278,30 @@ def setBorderRouter(self, is_border_router=True): def isBorderRouter(self): return self.__is_border_router + def setRoutingBackend(self, backend: str) -> Router: + """! + @brief Set the full routing daemon backend for this router. + + @param backend routing backend. Supported values are bird and frr. + ExaBGP is installed as a service speaker, not as a router backend. + + @returns self, for chaining API calls. + """ + value = str(backend or "bird").strip().lower() or "bird" + assert value in {"bird", "frr"}, "unsupported routing backend: {}".format(backend) + self.__routing_backend = value + self.setLabel("seedemu_routing_backend", value) + self.setLabel("seedemu_bgp_backend", value) + return self + + def getRoutingBackend(self) -> str: + """! + @brief Get the full routing daemon backend for this router. + + @returns backend name. + """ + return self.__routing_backend + def setLoopbackAddress(self, address: str): """! @brief Set loopback address. @@ -1472,4 +1604,3 @@ def promote_to_scion_router(node: Node): extn.initScionRouter() node.installExtension(extn) return node - diff --git a/seedemu/k8sTools/README.md b/seedemu/k8sTools/README.md new file mode 100644 index 000000000..7012b7a01 --- /dev/null +++ b/seedemu/k8sTools/README.md @@ -0,0 +1,80 @@ +# seedemu.k8sTools + +`seedemu.k8sTools` is the simplified Kubernetes workflow entrypoint for +SeedEMU examples. It keeps the user-facing example directory small: commands +use temporary setup/running resources internally and do not persist `setup/`, +`running/`, or `.k8sTools/` directories. + +## Commands + +Build infrastructure: + +```bash +python k8sTools.py build \ + --input configKvmOvn.yaml \ + --config-k3s configK3s.yaml \ + --kubeconfig kubeconfig.yaml \ + --inventory inventory.yaml +``` + +Deploy workload: + +```bash +python k8sTools.py up -f ./output -k kubeconfig.yaml -d configK3s.yaml +``` + +`up` infers the logical compiler image prefix from `images.yaml` or +`images.txt`. Pass +`--image-registry-prefix ` only when the compile output intentionally +uses a non-standard or mixed prefix. + +Clean workload only: + +```bash +python k8sTools.py clean -f ./output -k kubeconfig.yaml +``` + +`down` remains accepted as a compatibility alias for `clean`. + +Destroy infrastructure: + +```bash +python k8sTools.py destroy -d configK3s.yaml +``` + +## Config Kinds + +- `kind: kvmOvn` creates local KVM VMs, builds K3s, and installs Kube-OVN. +- `kind: physicalOvn` uses existing physical nodes, builds K3s, and installs Kube-OVN. +- `kind: multiHostKvmOvn` creates KVM VMs across multiple hypervisors, builds + K3s, and installs Kube-OVN. + +## Internal Architecture + +`build` copies bundled `resources/setup/` into a temporary directory and invokes +Python entrypoints there: + +- `kvmOvn`: `kvm/prepareHostAssets.py`, `kvm/createKvmVms.py`, + `kvm/tuneVmLimits.py`, `applyK3sCluster.py`, then Kube-OVN installation. +- `physicalOvn`: `preparePhysicalNodes.py`, `applyK3sCluster.py`, then + Kube-OVN installation. +- `multiHostKvmOvn`: `multiHostKvm/prepareKvmHypervisors.py`, + `multiHostKvm/createMultiHostKvmVms.py`, `kvm/tuneVmLimits.py`, + `applyK3sCluster.py`, then Kube-OVN installation. + +`up` copies bundled `resources/running/` into a temporary directory and invokes +`manageRunningStage.py preflight`, `build`, and `up`. The running stage renders +`kustomization.yaml`, rewrites OVN manifests when needed, reads image metadata +from `images.yaml` or `images.txt`, builds/pushes images to the registry +resolved from `configK3s.yaml`, applies the manifest, and waits for readiness. +Before a remote registry-host build, it scans generated Dockerfiles for external +`FROM` images, ensures those images exist on the local Docker daemon, and loads +them into the registry host Docker daemon so buildx does not need to resolve +compiler base images through Docker Hub from the master node. + +No persistent setup/running working directory is created by default. Persistent +outputs are limited to the user-requested `configK3s.yaml`, `kubeconfig.yaml`, +optional `inventory.yaml`, and the compiler output directory. + +All commands accept `--keep-temp` to leave the copied setup/running resources +on disk for debugging. diff --git a/seedemu/k8sTools/__init__.py b/seedemu/k8sTools/__init__.py new file mode 100644 index 000000000..077e6b7f7 --- /dev/null +++ b/seedemu/k8sTools/__init__.py @@ -0,0 +1,3 @@ +from .k8sTools import K8sTools + +__all__ = ["K8sTools"] diff --git a/seedemu/k8sTools/__main__.py b/seedemu/k8sTools/__main__.py new file mode 100644 index 000000000..0d3af83c7 --- /dev/null +++ b/seedemu/k8sTools/__main__.py @@ -0,0 +1,7 @@ +"""Allow ``python -m seedemu.k8sTools`` to run the K8sTools CLI.""" + +from .k8sTools import K8sTools + + +if __name__ == "__main__": + raise SystemExit(K8sTools().runCli()) diff --git a/seedemu/k8sTools/config.py b/seedemu/k8sTools/config.py new file mode 100644 index 000000000..7dec13fe0 --- /dev/null +++ b/seedemu/k8sTools/config.py @@ -0,0 +1,501 @@ +from __future__ import annotations + +import copy +import getpass +import hashlib +import os +from pathlib import Path +from typing import Any + +import yaml + + +SUPPORTED_KINDS = {"kvmOvn", "physicalOvn", "multiHostKvmOvn"} + + +def loadYaml(path: str | Path) -> dict[str, Any]: + """Load a YAML mapping from path. + + Args: + path: YAML file path supplied by the caller. + + Returns: + Parsed YAML mapping. Empty files are treated as an empty mapping. + """ + with Path(path).expanduser().open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise ValueError(f"Invalid YAML root in {path}: expected a mapping") + return data + + +def writeYaml(path: str | Path, data: dict[str, Any]) -> None: + """Write a YAML mapping to path, creating parent directories as needed. + + Args: + path: Destination YAML file path. + data: Mapping to serialize. + """ + output = Path(path).expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8") as handle: + yaml.safe_dump(data, handle, default_flow_style=False, sort_keys=False) + + +def detectKind(config: dict[str, Any], source: str | Path) -> str: + """Return the explicit k8sTools workflow kind from a user config. + + Args: + config: Parsed YAML mapping. + source: Config path used only for a readable error message. + + The new b61 flow intentionally requires `kind` in YAML so command behavior + never depends on file names or ambient environment. + """ + kind = str(config.get("kind") or "").strip() + if kind not in SUPPORTED_KINDS: + supported = ", ".join(sorted(SUPPORTED_KINDS)) + raise ValueError(f"{source} must set kind to one of: {supported}") + return kind + + +def resolvePath(path: str | Path, base_dir: str | Path | None = None) -> Path: + """Resolve a user path relative to base_dir or the current directory. + + Args: + path: User supplied path. + base_dir: Optional base directory for relative paths. + """ + raw = Path(path).expanduser() + if raw.is_absolute(): + return raw.resolve() + base = Path(base_dir).expanduser().resolve() if base_dir is not None else Path.cwd().resolve() + return (base / raw).resolve() + + +def setOutputPaths(config: dict[str, Any], *, kubeconfig: str | Path, tmp_dir: str | Path, inventory: str | Path | None = None) -> None: + """Set K3s-stage output paths in a config mapping. + + Args: + config: Config mapping that will be consumed by setup scripts. + kubeconfig: Final kubeconfig path requested by the user. + tmp_dir: Temporary directory used by setup scripts. + inventory: Optional generated Ansible inventory path. + """ + outputs = _mapping(config, "outputs") + outputs["kubeconfig"] = str(Path(kubeconfig).expanduser().resolve()) + outputs["tmpDir"] = str(Path(tmp_dir).expanduser().resolve()) + outputs["inventory"] = str(Path(inventory).expanduser().resolve()) if inventory else str(Path(tmp_dir).expanduser().resolve() / "cluster.inventory.yaml") + + +def addK8sToolsMetadata( + config: dict[str, Any], + *, + kind: str, + source_config: str | Path, + kubeconfig: str | Path, + destroy_type: str, + destroy_state: dict[str, Any] | None = None, + destroy_config: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Attach k8sTools metadata needed by later up/down/destroy commands. + + Args: + config: Final configK3s.yaml mapping. + kind: Workflow kind such as kvmOvn or multiHostKvmOvn. + source_config: User-provided input YAML path. + kubeconfig: Final kubeconfig path. + destroy_type: Destroy workflow selector. + destroy_state: Optional state mapping for KVM cleanup. + destroy_config: Optional original setup config needed by destroy. + """ + config["kind"] = kind + config["sourceConfig"] = str(Path(source_config).expanduser().resolve()) + outputs = _mapping(config, "outputs") + outputs["kubeconfig"] = str(Path(kubeconfig).expanduser().resolve()) + tools = _mapping(config, "k8sTools") + destroy = { + "type": destroy_type, + } + if destroy_state is not None: + destroy["state"] = destroy_state + if destroy_config is not None: + destroy["config"] = destroy_config + tools["destroy"] = destroy + return config + + +def getRegistryPrefix(config: dict[str, Any]) -> str: + """Return registry host:port from configK3s.yaml. + + Args: + config: Parsed configK3s.yaml mapping. + """ + registry = config.get("registry") if isinstance(config.get("registry"), dict) else {} + host = registry.get("host") + port = registry.get("port") or 5000 + if not host: + for node in config.get("nodes") or []: + if isinstance(node, dict) and str(node.get("role") or "").lower() == "master": + host = node.get("ip") + break + if not host: + raise ValueError("configK3s.yaml must contain registry.host or one role=master node with ip") + return f"{host}:{port}" + + +def makeKvmConfig( + *, + config: str | Path | None = None, + setup_dir: str | Path | None = None, + cluster_name: str = "seedemu-k3s", + ssh_user: str = "ubuntu", + ssh_key: str = "~/.ssh/id_ed25519", + registry_port: int = 5000, + disk_dir: str | Path | None = None, + base_image_path: str | Path | None = None, + cloud_init_dir: str | Path | None = None, + tmp_dir: str | Path | None = None, + kubeconfig_path: str | Path | None = None, + inventory_path: str | Path | None = None, + k3s_config_path: str | Path | None = None, + master: bool = True, + workers: bool = True, + master_vcpus: int = 12, + master_memory_mb: int = 10240, + master_disk_gb: int = 80, + worker_count: int = 2, + worker_vcpus: int = 6, + worker_memory_mb: int = 10240, + worker_disk_gb: int = 80, +) -> dict[str, Any]: + """Create the KVM-stage YAML config consumed by setup scripts. + + Args: + config: Optional user YAML. Existing fields have priority. + setup_dir: Generated setup directory; used to derive output paths. + cluster_name: Default K3s cluster name. + ssh_user: Default SSH user for generated cloud-init users. + ssh_key: Default SSH private key path used for VM access. + registry_port: Default registry port on the K3s master. + disk_dir: Optional KVM qcow2 disk directory override. + base_image_path: Optional Ubuntu cloud image path override. + cloud_init_dir: Optional cloud-init artifact directory override. + tmp_dir: Optional setup temporary directory override. + kubeconfig_path: Optional generated kubeconfig output path. + inventory_path: Optional generated cluster inventory output path. + k3s_config_path: Optional generated configK3s.yaml output path. + master: Whether to request a generated master VM. Current KVM scripts + require one master. + workers: Whether to request generated worker VMs. + master_vcpus: Default master vCPU count. + master_memory_mb: Default master memory size in MiB. + master_disk_gb: Default master disk size in GiB. + worker_count: Default number of generated worker VMs. + worker_vcpus: Default worker vCPU count. + worker_memory_mb: Default worker memory size in MiB. + worker_disk_gb: Default worker disk size in GiB. + """ + if not master: + raise ValueError("master=False is not supported by current KVM scripts") + + setup_path = Path(setup_dir).expanduser().resolve() if setup_dir is not None else None + data = copy.deepcopy(loadYaml(config)) if config is not None else {} + effective_cluster_name = str(data.get("clusterName") or data.get("cluster_name") or cluster_name) + data["clusterName"] = effective_cluster_name + data.pop("cluster_name", None) + + master_cfg = _mapping(data, "master") + workers_cfg = _mapping(data, "workers") + _normalizeLegacyResourceKeys(master_cfg) + _normalizeLegacyResourceKeys(workers_cfg) + _fillMissing(master_cfg, {"vcpus": master_vcpus, "memoryMb": master_memory_mb, "diskGb": master_disk_gb}) + _fillMissing(workers_cfg, {"count": worker_count, "vcpus": worker_vcpus, "memoryMb": worker_memory_mb, "diskGb": worker_disk_gb}) + + if not workers: + workers_cfg["count"] = 0 + + ssh_cfg = _mapping(data, "ssh") + _fillMissing(ssh_cfg, {"user": ssh_user, "key": ssh_key}) + + registry_cfg = _mapping(data, "registry") + _fillMissing(registry_cfg, {"port": registry_port}) + + if setup_path is not None: + kvm_data_dir = _defaultKvmDataDir(setup_path) + kvm_defaults = { + "storageDir": _portablePath(setup_path, setup_path), + "diskDir": str(Path(disk_dir).expanduser().resolve()) if disk_dir is not None else str(kvm_data_dir / "disks"), + "cloudInitDir": ( + str(Path(cloud_init_dir).expanduser().resolve()) + if cloud_init_dir is not None + else _portablePath(setup_path / "cloud-init", setup_path) + ), + "baseImagePath": str(Path(base_image_path).expanduser().resolve()) if base_image_path is not None else str(kvm_data_dir / "base" / "jammy-server-cloudimg-amd64.img"), + } + output_defaults = { + "tmpDir": str(Path(tmp_dir).expanduser().resolve()) if tmp_dir is not None else _portablePath(setup_path / "tmp", setup_path), + "kubeconfig": ( + str(Path(kubeconfig_path).expanduser().resolve()) + if kubeconfig_path is not None + else _portablePath(setup_path / f"{effective_cluster_name}.kubeconfig.yaml", setup_path) + ), + "inventory": ( + str(Path(inventory_path).expanduser().resolve()) + if inventory_path is not None + else _portablePath(setup_path / f"{effective_cluster_name}.inventory.yaml", setup_path) + ), + "k3sConfig": ( + str(Path(k3s_config_path).expanduser().resolve()) + if k3s_config_path is not None + else _portablePath(setup_path / "configK3s.yaml", setup_path) + ), + "kvmState": _portablePath(setup_path / "kvmState.yaml", setup_path), + } + kvm_cfg = _mapping(data, "kvm") + outputs_cfg = _mapping(data, "outputs") + _normalizeLegacyKvmKeys(kvm_cfg) + _normalizeLegacyOutputKeys(outputs_cfg) + outputs_cfg.pop("runningConfig", None) + outputs_cfg.pop("running_config", None) + _fillMissing(kvm_cfg, kvm_defaults) + _fillMissing(outputs_cfg, output_defaults) + else: + _normalizeLegacyKvmKeys(_mapping(data, "kvm")) + outputs_cfg = _mapping(data, "outputs") + _normalizeLegacyOutputKeys(outputs_cfg) + outputs_cfg.pop("runningConfig", None) + outputs_cfg.pop("running_config", None) + return data + + +def makeMultiHostKvmConfig( + *, + config: str | Path | None = None, + setup_dir: str | Path | None = None, + cluster_name: str = "seedemu-k3s", + registry_port: int = 5000, +) -> dict[str, Any]: + """Create multi-hypervisor KVM config consumed by setup scripts. + + Args: + config: User YAML describing hypervisors, routed subnets, VM resources, + and optional K3s/fabric settings. + setup_dir: Generated setup directory; used to derive output paths. + cluster_name: Default cluster name when config omits clusterName. + registry_port: Default registry port on the generated master VM. + + The multi-host flow requires an explicit config file because physical + hypervisor IPs, SSH keys and routed subnets cannot be guessed safely. + """ + if config is None: + raise ValueError("writeMultiHostKvmInstallScripts(config=...) requires a multi-host kvm.yaml") + + setup_path = Path(setup_dir).expanduser().resolve() if setup_dir is not None else None + data = copy.deepcopy(loadYaml(config)) + effective_cluster_name = str(data.get("clusterName") or data.get("cluster_name") or cluster_name) + data["clusterName"] = effective_cluster_name + data.pop("cluster_name", None) + + if not isinstance(data.get("hypervisors"), list) or not data["hypervisors"]: + raise ValueError("multi-host kvm.yaml requires a non-empty hypervisors list") + + _normalizeLegacyResourceKeys(_mapping(data, "master")) + _normalizeLegacyResourceKeys(_mapping(data, "workers")) + + registry_cfg = _mapping(data, "registry") + _fillMissing(registry_cfg, {"port": registry_port}) + + if setup_path is not None: + outputs_cfg = _mapping(data, "outputs") + _normalizeLegacyOutputKeys(outputs_cfg) + _fillMissing( + outputs_cfg, + { + "tmpDir": _portablePath(setup_path / "tmp", setup_path), + "k3sConfig": _portablePath(setup_path / "configK3s.yaml", setup_path), + "multiHostKvmState": _portablePath(setup_path / "multiHostKvmState.yaml", setup_path), + "kubeconfig": _portablePath(setup_path / f"{effective_cluster_name}.kubeconfig.yaml", setup_path), + "inventory": _portablePath(setup_path / f"{effective_cluster_name}.inventory.yaml", setup_path), + }, + ) + return data + + +def makeRunningConfig( + *, + setup_dir: str | Path, + running_dir: str | Path, + output_dir: str | Path | None = None, + image_registry_prefix: str = "seedemu", + rollout_timeout_seconds: int = 1800, +) -> dict[str, Any]: + """Create running-stage YAML config consumed by manageRunningStage.py. + + Args: + setup_dir: Setup directory containing configK3s.yaml. + running_dir: Running scripts directory. + output_dir: SeedEMU compile output directory. Defaults to ../output + relative to the generated root. + image_registry_prefix: Logical image prefix used in k8s.yaml. + rollout_timeout_seconds: Rollout/wait timeout used by the up/wait stage. + """ + setup_path = Path(setup_dir).expanduser().resolve() + running_path = Path(running_dir).expanduser().resolve() + if output_dir is None: + output_path = (running_path.parent / "output").resolve() + else: + output_path = Path(output_dir).expanduser().resolve() + return { + "setupConfig": _portablePath(setup_path / "configK3s.yaml", running_path), + "outputDir": _portablePath(output_path, running_path), + "imageRegistryPrefix": image_registry_prefix, + "rolloutTimeoutSeconds": rollout_timeout_seconds, + } + + +def makeK3sConfig( + *, + config: str | Path | None = None, + setup_dir: str | Path | None = None, + cluster_name: str = "seedemu-k3s", + ssh_user: str = "ubuntu", + ssh_key: str = "~/.ssh/id_ed25519", + registry_port: int = 5000, +) -> dict[str, Any]: + """Create configK3s.yaml for existing or newly created VMs. + + Args: + config: Optional YAML containing at least a nodes list. A minimal node + item can be {"ip": "192.168.122.10"}; name/role are inferred later. + setup_dir: Generated setup directory; used to derive output paths. + cluster_name: Default K3s cluster name. + ssh_user: Default SSH user for all nodes. + ssh_key: Default SSH private key path. + registry_port: Default registry port on the master. + """ + setup_path = Path(setup_dir).expanduser().resolve() if setup_dir is not None else None + data = copy.deepcopy(loadYaml(config)) if config is not None else {} + effective_cluster_name = str(data.get("clusterName") or data.get("cluster_name") or cluster_name) + data["clusterName"] = effective_cluster_name + data.pop("cluster_name", None) + + if "nodes" not in data: + data["nodes"] = [] + if not isinstance(data["nodes"], list): + raise ValueError("configK3s.yaml field 'nodes' must be a list") + + default_ssh = data.get("ssh") if isinstance(data.get("ssh"), dict) else {} + default_user = default_ssh.get("user") or ssh_user + default_key = default_ssh.get("key") or ssh_key + for node in data["nodes"]: + if not isinstance(node, dict): + raise ValueError(f"configK3s.yaml node item must be a mapping: {node}") + ssh_cfg = _mapping(node, "ssh") + _fillMissing(ssh_cfg, {"user": default_user, "key": default_key}) + # Passwords are only useful for a one-time manual SSH key bootstrap. + # Generated K3s configs should not replicate them because the build + # scripts require key-based SSH and sudo -n. + ssh_cfg.pop("password", None) + ssh_cfg.pop("passwd", None) + data.pop("ssh", None) + if "registry" in data: + registry_cfg = _mapping(data, "registry") + _fillMissing(registry_cfg, {"port": registry_port}) + if registry_cfg == {"port": registry_port}: + data.pop("registry", None) + data.pop("outputs", None) + return data + + +def _fillMissing(target: dict[str, Any], defaults: dict[str, Any]) -> None: + for key, value in defaults.items(): + if key not in target or target[key] is None: + target[key] = value + + +def _mapping(data: dict[str, Any], key: str) -> dict[str, Any]: + value = data.get(key) + if value is None: + value = {} + data[key] = value + if not isinstance(value, dict): + raise ValueError(f"Invalid config: '{key}' must be a mapping") + return value + + +def _defaultKvmDataDir(setup_path: Path) -> Path: + # KVM disks are placed under /data so libvirt does not depend on access to + # a user's home directory. + root = Path(f"/data/{getpass.getuser()}/k8sTools").expanduser() + base = setup_path.parent + digest = hashlib.sha1(str(base).encode("utf-8")).hexdigest()[:8] + return root / f"{base.name}-{digest}" + + +def _portablePath(path: Path, base_dir: Path) -> str: + """Return a readable path for generated YAML. + + Args: + path: Absolute path to write. + base_dir: Directory that will contain the generated YAML. + + Paths inside the same nearby source tree are written relative to the YAML + file's directory. Unrelated paths, such as a repo output consumed from a + temporary directory, stay absolute so copied generated directories continue + to resolve correctly. + """ + path = path.expanduser().resolve() + base_dir = base_dir.expanduser().resolve() + common = Path(os.path.commonpath([str(path), str(base_dir)])) + if common == Path(path.anchor): + return str(path) + relative = Path(os.path.relpath(path, start=base_dir)) + if len(relative.parts) <= 8: + return relative.as_posix() + return str(path) + + +def _normalizeLegacyResourceKeys(data: dict[str, Any]) -> None: + aliases = {"memory_mb": "memoryMb", "disk_gb": "diskGb", "name_prefix": "namePrefix"} + for old, new in aliases.items(): + if old in data and new not in data: + data[new] = data.pop(old) + + +def _normalizeLegacyKvmKeys(data: dict[str, Any]) -> None: + aliases = { + "storage_dir": "storageDir", + "disk_dir": "diskDir", + "cloud_init_dir": "cloudInitDir", + "base_image_path": "baseImagePath", + "legacy_base_image_path": "legacyBaseImagePath", + "base_image_url": "baseImageUrl", + "ubuntu_series": "ubuntuSeries", + "boot_timeout_seconds": "bootTimeoutSeconds", + "allow_existing": "allowExisting", + } + for old, new in aliases.items(): + if old in data and new not in data: + data[new] = data.pop(old) + + +def _normalizeLegacyOutputKeys(data: dict[str, Any]) -> None: + aliases = { + "tmp_dir": "tmpDir", + "kvm_state": "kvmState", + "k3s_config": "k3sConfig", + "multi_host_kvm_state": "multiHostKvmState", + } + for old, new in aliases.items(): + if old in data and new not in data: + data[new] = data.pop(old) + + +# Backward-compatible aliases for callers that still use the first prototype. +load_yaml = loadYaml +write_yaml = writeYaml +make_kvm_config = makeKvmConfig +make_multi_host_kvm_config = makeMultiHostKvmConfig +make_k3s_config = makeK3sConfig +make_running_config = makeRunningConfig diff --git a/seedemu/k8sTools/k8sTools.py b/seedemu/k8sTools/k8sTools.py new file mode 100644 index 000000000..8a2d64014 --- /dev/null +++ b/seedemu/k8sTools/k8sTools.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +from .runner import buildCluster, cleanWorkload, deployWorkload, destroyCluster + + +class K8sTools: + """User-facing CLI/API for SeedEMU Kubernetes workflows. + + The first-phase implementation intentionally keeps user directories clean: + setup/running resources are expanded into temporary directories during each + command and removed afterwards unless --keep-temp is used. Persistent user + outputs are limited to the requested configK3s.yaml, kubeconfig.yaml, and + compiler output directory. + """ + + def build( + self, + inputConfig: str | Path, + configK3s: str | Path, + kubeconfig: str | Path, + *, + inventory: str | Path | None = None, + keepTemp: bool = False, + ) -> None: + """Create infrastructure and write K3s access files. + + Args: + inputConfig: YAML with kind=kvmOvn, physicalOvn, or multiHostKvmOvn. + configK3s: Output configK3s.yaml path. + kubeconfig: Output kubeconfig path. + inventory: Optional output inventory.yaml path. + keepTemp: Keep temporary setup resources for debugging. + """ + buildCluster(inputConfig, configK3s, kubeconfig, inventory=inventory, keep_temp=keepTemp) + + def up( + self, + outputDir: str | Path, + kubeconfig: str | Path, + configK3s: str | Path, + *, + imageRegistryPrefix: str | None = None, + keepTemp: bool = False, + ) -> None: + """Build/push images, deploy workload, and wait for readiness. + + Args: + outputDir: Compile output directory. + kubeconfig: Kubeconfig path. + configK3s: configK3s.yaml with registry and network backend data. + imageRegistryPrefix: Logical compiler image prefix. If omitted, + it is inferred from images.yaml. + keepTemp: Keep temporary running resources for debugging. + """ + deployWorkload( + outputDir, + kubeconfig, + configK3s, + image_registry_prefix=imageRegistryPrefix, + keep_temp=keepTemp, + ) + + def down(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool = False) -> None: + """Delete workload resources while keeping cluster infrastructure. + + Args: + outputDir: Compile output directory. + kubeconfig: Kubeconfig path. + keepTemp: Keep temporary cleanup resources for debugging. + """ + cleanWorkload(outputDir, kubeconfig, keep_temp=keepTemp) + + def clean(self, outputDir: str | Path, kubeconfig: str | Path, *, keepTemp: bool = False) -> None: + """Delete workload resources while keeping cluster infrastructure. + + Args: + outputDir: Compile output directory. + kubeconfig: Kubeconfig path. + keepTemp: Keep temporary cleanup resources for debugging. + """ + self.down(outputDir, kubeconfig, keepTemp=keepTemp) + + def destroy(self, configK3s: str | Path, *, keepTemp: bool = False) -> None: + """Destroy K3s/OVN and optional KVM infrastructure. + + Args: + configK3s: configK3s.yaml produced by build(). + keepTemp: Keep temporary destroy resources for debugging. + """ + destroyCluster(configK3s, keep_temp=keepTemp) + + def runCli(self, argv: list[str] | None = None) -> int: + """Parse and execute the command-line interface. + + Args: + argv: Optional argv override for tests. + """ + parser = argparse.ArgumentParser(prog="k8sTools.py") + sub = parser.add_subparsers(dest="command", required=True) + + build = sub.add_parser("build", help="create KVM/physical K3s + OVN infrastructure") + build.add_argument("--input", required=True, help="input YAML with explicit kind") + build.add_argument("--config-k3s", required=True, help="output configK3s.yaml") + build.add_argument("--kubeconfig", required=True, help="output kubeconfig.yaml") + build.add_argument("--inventory", help="optional output inventory.yaml with node resources") + build.add_argument("--keep-temp", action="store_true", help="keep temporary setup resources") + + up = sub.add_parser("up", help="build images and deploy workload") + up.add_argument("-f", "--folder", required=True, help="compile output directory") + up.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") + up.add_argument("-d", "--config-k3s", required=True, help="configK3s.yaml") + up.add_argument( + "--image-registry-prefix", + help="logical compiler image prefix; inferred from images.yaml when omitted", + ) + up.add_argument("--keep-temp", action="store_true", help="keep temporary running resources") + + down = sub.add_parser("down", help="delete workload resources only") + down.add_argument("-f", "--folder", required=True, help="compile output directory") + down.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") + down.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + + clean = sub.add_parser("clean", help="delete workload resources only") + clean.add_argument("-f", "--folder", required=True, help="compile output directory") + clean.add_argument("-k", "--kubeconfig", required=True, help="kubeconfig.yaml") + clean.add_argument("--keep-temp", action="store_true", help="keep temporary cleanup resources") + + destroy = sub.add_parser("destroy", help="destroy cluster and optional KVM resources") + destroy.add_argument("-d", "--config-k3s", required=True, help="configK3s.yaml") + destroy.add_argument("--keep-temp", action="store_true", help="keep temporary destroy resources") + + args = parser.parse_args(argv) + if args.command == "build": + self.build(args.input, args.config_k3s, args.kubeconfig, inventory=args.inventory, keepTemp=args.keep_temp) + elif args.command == "up": + self.up( + args.folder, + args.kubeconfig, + args.config_k3s, + imageRegistryPrefix=args.image_registry_prefix, + keepTemp=args.keep_temp, + ) + elif args.command == "down": + self.down(args.folder, args.kubeconfig, keepTemp=args.keep_temp) + elif args.command == "clean": + self.clean(args.folder, args.kubeconfig, keepTemp=args.keep_temp) + elif args.command == "destroy": + self.destroy(args.config_k3s, keepTemp=args.keep_temp) + else: + parser.error(f"unsupported command: {args.command}") + return 0 diff --git a/seedemu/k8sTools/resources/__init__.py b/seedemu/k8sTools/resources/__init__.py new file mode 100644 index 000000000..36df2e420 --- /dev/null +++ b/seedemu/k8sTools/resources/__init__.py @@ -0,0 +1 @@ +"""Packaged setup and running resources for seedemu.k8sTools.""" diff --git a/seedemu/k8sTools/resources/running/_embeddedShell.py b/seedemu/k8sTools/resources/running/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/running/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/running/buildRegistryImages.py b/seedemu/k8sTools/resources/running/buildRegistryImages.py new file mode 100755 index 000000000..e4de2feeb --- /dev/null +++ b/seedemu/k8sTools/resources/running/buildRegistryImages.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Python entrypoint for buildRegistryImages with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Build SeedEMU workload images with BuildKit/buildx and push them to the +# registry selected by command-line arguments or configRunning.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +OUTPUT_DIR="" +REGISTRY_PREFIX="" +IMAGE_REGISTRY_PREFIX="" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${OUTPUT_DIR}" ]; then + OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +fi +if [ -z "${REGISTRY_PREFIX}" ]; then + REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +fi +if [ -z "${IMAGE_REGISTRY_PREFIX}" ]; then + IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +fi + +IMAGES_YAML="${OUTPUT_DIR}/images.yaml" +[ -s "${IMAGES_YAML}" ] || IMAGES_YAML="${OUTPUT_DIR}/images.txt" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +cd "${OUTPUT_DIR}" + +buildImageWithBuildx() { + # Args: + # $1: full destination image reference + # $2: Docker build context directory + local image="$1" + local context="$2" + local log_file + log_file="$(mktemp)" + echo "+ DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t ${image} ${context}" + if DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" 2>&1 | tee "${log_file}"; then + rm -f "${log_file}" + return 0 + fi + + if grep -Eq 'parent snapshot .* does not exist|failed to prepare extraction snapshot' "${log_file}"; then + echo "[k8s_build] buildx export/cache failure while building ${image}; pruning buildx cache and retrying once" >&2 + docker buildx prune -af >/dev/null 2>&1 || true + rm -f "${log_file}" + DOCKER_BUILDKIT=1 docker buildx build --load --provenance=false -t "${image}" "${context}" + return $? + fi + + rm -f "${log_file}" + return 1 +} + +firstFromImage() { + # Args: + # $1: Dockerfile path. + # Prints the first FROM image reference, handling optional --platform. + awk ' + toupper($1) == "FROM" { + if ($2 ~ /^--platform=/) { + print $3 + } else { + print $2 + } + exit + } + ' "$1" +} + +isCompilerBaseTag() { + # Args: + # $1: image reference to check. + # Returns true for Docker compiler hash tags used in generated FROM lines. + [[ "$1" =~ ^[0-9a-f]{32}(:latest)?$ ]] +} + +ensureCompilerBaseImages() { + # Build every hash-tagged base image required by generated node Dockerfiles. + # A missing base_images//Dockerfile means the compiler output is + # incomplete; without this check Docker would try pulling library/. + local tmpfile + local dockerfile + local from_image + local base_tag + local context + local missing=0 + + tmpfile="$(mktemp)" + while IFS= read -r dockerfile; do + from_image="$(firstFromImage "${dockerfile}")" + if isCompilerBaseTag "${from_image}"; then + printf '%s\n' "${from_image%:latest}" >> "${tmpfile}" + fi + done < <(find . -path './base_images/*' -prune -o -name Dockerfile -print | sort) + + while IFS= read -r base_tag; do + [ -n "${base_tag}" ] || continue + context="base_images/${base_tag}" + if [ -f "${context}/Dockerfile" ]; then + buildImageWithBuildx "${base_tag}" "${context}" + elif docker image inspect "${base_tag}" >/dev/null 2>&1; then + continue + else + echo "Missing compiler base image context: ${context}/Dockerfile" >&2 + echo "Re-run compile with the current KubernetesCompiler so base_images/ is generated." >&2 + missing=1 + fi + done < <(sort -u "${tmpfile}") + + rm -f "${tmpfile}" + return "${missing}" +} + +ensureCompilerBaseImages + +python3 "${HELPER}" mapped-images \ + --images-yaml "${IMAGES_YAML}" \ + --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" \ + --registry-prefix "${REGISTRY_PREFIX}" | +while IFS=$'\t' read -r image context; do + [ -n "${image}" ] || continue + [ -n "${context}" ] || { echo "Missing context for ${image}" >&2; exit 1; } + buildImageWithBuildx "${image}" "${context}" + echo "+ docker push ${image}" + docker push "${image}" +done +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/running/manageK8sManifest.py b/seedemu/k8sTools/resources/running/manageK8sManifest.py new file mode 100755 index 000000000..157181521 --- /dev/null +++ b/seedemu/k8sTools/resources/running/manageK8sManifest.py @@ -0,0 +1,1014 @@ +#!/usr/bin/env python3 +"""Resolve running-stage configuration and render deploy helper artifacts. + +Inputs: +- configRunning.yaml, which points to configK3s.yaml and compile output. +- configK3s.yaml, whose master node provides the default registry/SSH target. +- k8s.kube-ovn.yaml or k8s.yaml plus images.yaml from compile output. + +Outputs: +- scalar values consumed by manageRunningStage.py, +- kustomization.yaml image mappings, +- manifest-derived namespace and deployment names. +""" +from __future__ import annotations + +import argparse +import getpass +import hashlib +import ipaddress +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +import yaml + + +_LOCAL_IPS: set[str] | None = None + + +def load_yaml(path: str) -> dict[str, Any]: + """Load a YAML mapping from path.""" + data = yaml.safe_load(Path(path).expanduser().read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + raise SystemExit(f"Invalid YAML root in {path}: expected mapping") + return data + + +def readLocalIps() -> set[str]: + """Return IP addresses assigned to the host running the running scripts.""" + global _LOCAL_IPS + if _LOCAL_IPS is not None: + return _LOCAL_IPS + ips = {"127.0.0.1", "::1", "localhost"} + try: + output = subprocess.check_output( + ["ip", "-o", "addr", "show"], + text=True, + stderr=subprocess.DEVNULL, + ) + for line in output.splitlines(): + for token in line.split(): + if "/" not in token: + continue + address = token.split("/", 1)[0] + if address and (address[0].isdigit() or ":" in address): + ips.add(address) + except Exception: + pass + _LOCAL_IPS = ips + return ips + + +def get_nested(data: dict[str, Any], path: str, default: Any = None) -> Any: + """Read a dotted path from YAML, accepting camelCase and snake_case keys.""" + cur: Any = data + for part in path.split("."): + if not isinstance(cur, dict): + return default + candidates = [part, snake_case(part), camel_case(part)] + found = False + for candidate in candidates: + if candidate in cur: + cur = cur[candidate] + found = True + break + if not found: + return default + return cur + + +def normalizeRole(role: Any) -> str: + """Normalize a configK3s.yaml node role for master-node detection. + + Args: + role: Raw YAML role value from one node item. + """ + return str(role or "").strip().lower() + + +def getSetupNodes(setup: dict[str, Any]) -> list[dict[str, Any]]: + """Return node mappings from configK3s.yaml. + + Args: + setup: Parsed configK3s.yaml mapping. + """ + nodes = setup.get("nodes") or [] + if not isinstance(nodes, list): + raise SystemExit("configK3s.yaml field nodes must be a list") + invalid = [node for node in nodes if not isinstance(node, dict)] + if invalid: + raise SystemExit(f"configK3s.yaml node item must be a mapping: {invalid[0]}") + return nodes + + +def resolveNodeName(node: dict[str, Any], role: str, worker_index: int) -> str: + """Return the Kubernetes node-name used by the setup stage. + + Args: + node: Raw configK3s.yaml node mapping. + role: Normalized node role. + worker_index: 1-based worker index for unnamed worker nodes. + """ + name = node.get("name") + if name: + return str(name) + if role in {"master", "server", "control-plane", "control_plane"}: + return "seed-k3s-master" + return f"seed-k3s-worker{worker_index}" + + +def resolveNodeSshUser(setup: dict[str, Any], node: dict[str, Any]) -> str: + """Return SSH user for one node, with top-level ssh.user fallback.""" + return str(get_nested(node, "ssh.user") or get_nested(setup, "ssh.user") or "ubuntu") + + +def resolveNodeSshKey(setup: dict[str, Any], node: dict[str, Any]) -> str: + """Return SSH key for one node, with top-level ssh.key fallback.""" + return str(Path(str(get_nested(node, "ssh.key") or get_nested(setup, "ssh.key") or "~/.ssh/id_ed25519")).expanduser()) + + +def resolveNodeConnection(node: dict[str, Any], ip: str, ssh_user: str) -> str: + """Return local/ssh connection mode for one node. + + Args: + node: Raw configK3s.yaml node mapping. + ip: Node management IP. + ssh_user: Resolved SSH user. + """ + raw = node.get("connection") or node.get("connect") + if raw: + value = str(raw).strip().lower() + if value in {"local", "localhost"}: + return "local" + if value in {"ssh", "remote"}: + return "ssh" + raise SystemExit(f"Unsupported node connection for {ip}: {raw}") + if node.get("local") is True: + return "local" + if ip in readLocalIps() and ssh_user == getpass.getuser(): + return "local" + return "ssh" + + +def resolvedSetupNodes(setup: dict[str, Any]) -> list[dict[str, str]]: + """Return normalized node access records from configK3s.yaml.""" + records: list[dict[str, str]] = [] + worker_index = 0 + for node in getSetupNodes(setup): + role = normalizeRole(node.get("role")) + if role in {"worker", "agent"}: + worker_index += 1 + name = resolveNodeName(node, role, worker_index) + ip = str(node.get("ip") or node.get("managementIp") or node.get("management_ip") or "") + if not ip: + raise SystemExit(f"configK3s.yaml node requires ip: {node}") + ssh_user = resolveNodeSshUser(setup, node) + ssh_key = resolveNodeSshKey(setup, node) + records.append( + { + "name": name, + "role": role, + "ip": ip, + "sshUser": ssh_user, + "sshKey": ssh_key, + "connection": resolveNodeConnection(node, ip, ssh_user), + } + ) + return records + + +def findMasterNode(setup: dict[str, Any]) -> dict[str, Any] | None: + """Find the single master node used as the registry and build SSH target. + + Args: + setup: Parsed configK3s.yaml mapping. + """ + masters = [ + node + for node in resolvedSetupNodes(setup) + if normalizeRole(node.get("role")) in {"master", "server", "control-plane", "control_plane"} + ] + if len(masters) > 1: + names = ", ".join(str(node.get("name") or node.get("ip") or "") for node in masters) + raise SystemExit(f"configK3s.yaml must contain exactly one master node, got {len(masters)}: {names}") + return masters[0] if masters else None + + +def resolveRegistryHost(setup: dict[str, Any], master_node: dict[str, Any] | None) -> str: + """Resolve registry host from explicit registry.host or master node IP. + + Args: + setup: Parsed configK3s.yaml mapping. + master_node: Master node mapping returned by findMasterNode(). + """ + explicit_host = get_nested(setup, "registry.host") + if explicit_host: + return str(explicit_host) + if master_node and master_node.get("ip"): + return str(master_node["ip"]) + raise SystemExit("Cannot resolve registry host: set registry.host or provide one role=master node with ip") + + +def resolveSshUser(setup: dict[str, Any], master_node: dict[str, Any] | None) -> str: + """Resolve SSH user for the registry/build host. + + Args: + setup: Parsed configK3s.yaml mapping. + master_node: Master node mapping returned by findMasterNode(). + """ + explicit_user = get_nested(setup, "ssh.user") + if explicit_user: + return str(explicit_user) + master_user = (master_node or {}).get("sshUser") or get_nested(master_node or {}, "ssh.user") + if master_user: + return str(master_user) + return "ubuntu" + + +def resolveSshKey(setup: dict[str, Any], master_node: dict[str, Any] | None) -> str: + """Resolve SSH private key path for the registry/build host. + + Args: + setup: Parsed configK3s.yaml mapping. + master_node: Master node mapping returned by findMasterNode(). + """ + explicit_key = get_nested(setup, "ssh.key") + if explicit_key: + return str(Path(str(explicit_key)).expanduser()) + master_key = (master_node or {}).get("sshKey") or get_nested(master_node or {}, "ssh.key") + if master_key: + return str(Path(str(master_key)).expanduser()) + return str(Path("~/.ssh/id_ed25519").expanduser()) + + +def running_context(config_path: str) -> dict[str, str]: + """Resolve all running-stage values from configRunning.yaml.""" + running_config_path = Path(config_path).expanduser().resolve() + running = load_yaml(str(running_config_path)) + setup_config_path = Path(str(running.get("setupConfig") or running_config_path.parent / "../setup/configK3s.yaml")).expanduser() + if not setup_config_path.is_absolute(): + setup_config_path = (running_config_path.parent / setup_config_path).resolve() + setup = load_yaml(str(setup_config_path)) if setup_config_path.exists() else {} + output_dir = Path(str(running.get("outputDir") or running_config_path.parent / "../output")).expanduser() + if not output_dir.is_absolute(): + output_dir = (running_config_path.parent / output_dir).resolve() + master_node = findMasterNode(setup) if setup else None + cluster_name = str(setup.get("clusterName") or setup.get("cluster_name") or "seedemu-k3s") + registry_host = resolveRegistryHost(setup, master_node) + registry_port = str(get_nested(setup, "registry.port", "5000")) + fabric_type = str(get_nested(setup, "fabric.type", "none")).strip().lower() + network_backend = "kube-ovn" if fabric_type in {"ovn", "kube-ovn"} else "macvlan" + manifest_path = resolveManifestPath(running, output_dir, network_backend) + default_cni_master = ( + str(get_nested(setup, "fabric.bridgeName", "br-seedemu")) + if fabric_type in {"linux-vxlan", "vxlan", "linux_vxlan"} + else "ens2" + ) + attached_cni_type = str( + get_nested( + setup, + "cni.localLinkCniType", + get_nested(setup, "fabric.attachedCniType", os.environ.get("SEED_LOCAL_LINK_CNI_TYPE", "kube-ovn")), + ) + ) + return { + "setupConfig": str(setup_config_path), + "outputDir": str(output_dir), + "manifest": str(manifest_path), + "imagesYaml": str(resolveImagesPath(output_dir)), + "kustomization": str(output_dir / "kustomization.yaml"), + "imageRegistryPrefix": str(running.get("imageRegistryPrefix") or "seedemu"), + "registryPrefix": f"{registry_host}:{registry_port}", + "kubeconfig": str( + Path( + str( + get_nested( + setup, + "outputs.kubeconfig", + setup_config_path.parent / f"{cluster_name}.kubeconfig.yaml", + ) + ) + ).expanduser() + ), + "sshUser": resolveSshUser(setup, master_node), + "sshKey": resolveSshKey(setup, master_node), + "masterConnection": str((master_node or {}).get("connection") or "ssh"), + "cniMasterInterface": str(get_nested(setup, "cni.defaultMasterInterface", default_cni_master)), + "networkBackend": network_backend, + "attachedCniType": attached_cni_type, + "rolloutTimeoutSeconds": str(running.get("rolloutTimeoutSeconds") or "1800"), + } + + +def resolveManifestPath(running: dict[str, Any], output_dir: Path, network_backend: str) -> Path: + """Resolve the compile manifest consumed by the running stage. + + Args: + running: Parsed configRunning.yaml mapping. + output_dir: Compile output directory. + network_backend: Resolved backend, for example macvlan or kube-ovn. + + Kube-OVN compiler output is allowed to skip the historical k8s.yaml + intermediate and write k8s.kube-ovn.yaml directly. The fallback keeps older + compile outputs working. + """ + configured = running.get("manifest") + if configured: + manifest_path = Path(str(configured)).expanduser() + if not manifest_path.is_absolute(): + manifest_path = (output_dir / manifest_path).resolve() + return manifest_path + + kube_ovn_manifest = output_dir / "k8s.kube-ovn.yaml" + default_manifest = output_dir / "k8s.yaml" + if network_backend in {"kube-ovn", "ovn"} and kube_ovn_manifest.exists(): + return kube_ovn_manifest + return default_manifest + + +def resolveImagesPath(output_dir: Path) -> Path: + """Return the generated image metadata file for the compile output. + + Args: + output_dir: Compile output directory. + """ + images_yaml = output_dir / "images.yaml" + if images_yaml.exists(): + return images_yaml + return output_dir / "images.txt" + + +def config_value(args: argparse.Namespace) -> None: + """Print one resolved value from configRunning.yaml.""" + values = running_context(args.config) + if args.key not in values: + raise SystemExit(f"Unknown config key: {args.key}") + print(values[args.key]) + + +def node_access(args: argparse.Namespace) -> None: + """Print node access rows consumed by preflight/build scripts.""" + running_config_path = Path(args.config).expanduser().resolve() + running = load_yaml(str(running_config_path)) + setup_config_path = Path(str(running.get("setupConfig") or running_config_path.parent / "../setup/configK3s.yaml")).expanduser() + if not setup_config_path.is_absolute(): + setup_config_path = (running_config_path.parent / setup_config_path).resolve() + setup = load_yaml(str(setup_config_path)) + records = resolvedSetupNodes(setup) + if args.name: + records = [record for record in records if record["name"] == args.name] + if not records: + raise SystemExit(f"node not found in configK3s.yaml: {args.name}") + for record in records: + print( + "\t".join( + [ + record["name"], + record["ip"], + record["connection"], + record["sshUser"], + record["sshKey"], + ] + ) + ) + + +def split_repo_tag(image: str) -> tuple[str, str]: + tail = image.rsplit("/", 1)[-1] + if ":" in tail: + return image.rsplit(":", 1) + return image, "latest" + + +def strip_prefix(image: str, prefix: str) -> str: + prefix = prefix.rstrip("/") + if image.startswith(prefix + "/"): + return image[len(prefix) + 1 :] + return image.split("/", 1)[-1] + + +def load_images(path: str) -> list[dict[str, str]]: + """Load image metadata from images.yaml or legacy images.txt. + + Args: + path: Compiler image metadata file. + """ + image_path = Path(path) + text = image_path.read_text(encoding="utf-8") + if image_path.name == "images.txt": + return imagesFromText(text) + data = yaml.safe_load(text) or {} + if isinstance(data, dict): + images = data.get("images", []) + if isinstance(images, list): + return images + return imagesFromText(text) + + +def imagesFromText(text: str) -> list[dict[str, str]]: + """Parse one-image-reference-per-line metadata from KubernetesCompiler. + + Args: + text: Contents of images.txt. + """ + images: list[dict[str, str]] = [] + for line in text.splitlines(): + image = line.strip() + if not image or image.startswith("#"): + continue + images.append({"name": image, "context": contextFromImage(image)}) + return images + + +def contextFromImage(image: str) -> str: + """Infer a Docker build context directory from an image reference. + + Args: + image: Full image reference generated by the compiler. + """ + repo = image.rsplit("/", 1)[-1] + if ":" in repo: + repo = repo.rsplit(":", 1)[0] + return repo + + +def mapped_images(args: argparse.Namespace) -> None: + registry = args.registry_prefix.rstrip("/") + logical_prefix = args.image_registry_prefix.rstrip("/") + for item in load_images(args.images_yaml): + logical = item["name"].strip() + context = item["context"].strip() + print(f"{registry}/{strip_prefix(logical, logical_prefix)}\t{context}") + + +def render_kustomization(args: argparse.Namespace) -> None: + """Render kustomization.yaml for images and network backend adaptation. + + Args: + args.images_yaml: images.yaml generated by compile. + args.manifest: Manifest generated by compile. + args.image_registry_prefix: Logical compiler image prefix. + args.registry_prefix: Real registry host:port. + args.network_backend: "macvlan" or "kube-ovn". + args.cni_master_interface: Optional macvlan parent interface override. + args.output: Destination kustomization.yaml. + """ + registry = args.registry_prefix.rstrip("/") + logical_prefix = args.image_registry_prefix.rstrip("/") + images = [] + for item in load_images(args.images_yaml): + logical = item["name"].strip() + repo, tag = split_repo_tag(strip_prefix(logical, logical_prefix)) + logical_repo, _ = split_repo_tag(logical) + images.append({"name": logical_repo, "newName": f"{registry}/{repo}", "newTag": tag}) + output_path = Path(args.output) + manifest_path = Path(args.manifest) + network_backend = str(args.network_backend or "macvlan").strip().lower() + if network_backend in {"kube-ovn", "ovn"}: + rendered_manifest = output_path.parent / "k8s.kube-ovn.yaml" + if manifest_path.resolve() != rendered_manifest.resolve(): + renderKubeOvnManifest( + args.manifest, + rendered_manifest, + attached_cni_type=args.attached_cni_type, + cni_master_interface=args.cni_master_interface, + ) + payload = {"resources": [rendered_manifest.name], "images": images} + else: + payload = {"resources": namespaceResources(manifest_path, output_path.parent) + ["k8s.yaml"], "images": images} + patches = networkAttachmentPatches(args.manifest, args.cni_master_interface) + if patches: + payload["patches"] = patches + output_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + +def renderKubeOvnManifest( + manifest_path: str, + output_path: Path, + attached_cni_type: str | None = None, + cni_master_interface: str | None = None, +) -> None: + """Render a SeedEMU manifest that uses Kube-OVN managed secondary networks. + + Args: + manifest_path: Source k8s.yaml generated by the compiler. + output_path: Destination manifest referenced by kustomization.yaml. + attached_cni_type: Secondary CNI type. ``kube-ovn`` creates overlay + attached NICs; ``macvlan`` keeps macvlan dataplane and uses + Kube-OVN only for IPAM. + cni_master_interface: Optional macvlan master interface override. + + The compiler emits macvlan NADs plus Multus network-selection annotations + with static `ips` values in CIDR form. The renderer creates matching + Subnets and rewrites Pod template annotations without changing compiler + output. In macvlan mode, Kube-OVN acts as the centralized IPAM plugin and + avoids creating thousands of OVN logical switches/routes for scale tests. + """ + docs = loadManifestDocs(Path(manifest_path)) + + namespace_name = findNamespaceName(docs) + attached_cni = resolveAttachedCniType(attached_cni_type) + use_ovn_attached = isKubeOvnAttachedCni(attached_cni) + vpc_name = kubeOvnResourceName("vpc", namespace_name) + rendered: list[dict[str, Any]] = [] + vpc_written = False + namespace_written = any(doc.get("kind") == "Namespace" for doc in docs) + if not namespace_written: + rendered.append(namespaceResource(namespace_name)) + + for doc in docs: + if use_ovn_attached and doc.get("kind") == "Namespace" and not vpc_written: + rendered.append(doc) + rendered.append(kubeOvnVpc(namespace_name, vpc_name)) + vpc_written = True + continue + if doc.get("kind") == "NetworkAttachmentDefinition": + converted = convertNetworkAttachmentToKubeOvn( + doc, + namespace_name, + vpc_name, + attached_cni, + cni_master_interface, + ) + rendered.extend(converted) + continue + convertWorkloadAnnotationsToKubeOvn(doc, namespace_name, attached_cni) + rendered.append(doc) + + if use_ovn_attached and not vpc_written: + insert_at = 1 if rendered and rendered[0].get("kind") == "Namespace" else 0 + rendered.insert(insert_at, kubeOvnVpc(namespace_name, vpc_name)) + + output_path.write_text(yaml.safe_dump_all(rendered, sort_keys=False), encoding="utf-8") + + +def namespaceResource(namespace_name: str) -> dict[str, Any]: + """Return a Kubernetes Namespace resource for compiler outputs without one.""" + return {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace_name}} + + +def namespaceResources(manifest_path: Path, output_dir: Path) -> list[str]: + """Write namespace.yaml when the manifest lacks an explicit Namespace. + + Args: + manifest_path: Source manifest path. + output_dir: Directory that will contain kustomization.yaml. + """ + docs = loadManifestDocs(manifest_path) + if any(doc.get("kind") == "Namespace" for doc in docs): + return [] + namespace_name = findNamespaceName(docs) + namespace_path = output_dir / "namespace.yaml" + namespace_path.write_text(yaml.safe_dump(namespaceResource(namespace_name), sort_keys=False), encoding="utf-8") + return [namespace_path.name] + + +def resolveAttachedCniType(attached_cni_type: str | None = None) -> str: + """Return the configured secondary CNI type for Kube-OVN rendering.""" + value = ( + attached_cni_type + or os.environ.get("SEED_LOCAL_LINK_CNI_TYPE") + or os.environ.get("SEED_CNI_TYPE") + or "kube-ovn" + ) + return str(value).strip().lower().replace("_", "-") + + +def loadManifestDocs(manifest_path: Path) -> list[dict[str, Any]]: + """Load Kubernetes documents from a manifest path. + + Args: + manifest_path: YAML manifest path. + """ + with open(manifest_path, "r", encoding="utf-8") as fh: + return [doc for doc in yaml.safe_load_all(fh) if isinstance(doc, dict)] + + +def isKubeOvnAttachedCni(attached_cni_type: str) -> bool: + """Return true when secondary NICs should use the Kube-OVN CNI directly.""" + return attached_cni_type in {"kube-ovn", "ovn"} + + +def findNamespaceName(docs: list[dict[str, Any]]) -> str: + """Return the manifest namespace used by SeedEMU workload resources.""" + for doc in docs: + if doc.get("kind") == "Namespace": + name = (doc.get("metadata") or {}).get("name") + if name: + return str(name) + for doc in docs: + metadata = doc.get("metadata") or {} + name = metadata.get("namespace") + if name: + return str(name) + raise SystemExit("Cannot determine namespace for Kube-OVN manifest rendering") + + +def kubeOvnResourceName(prefix: str, value: str) -> str: + """Return a DNS-safe Kube-OVN cluster-scoped resource name.""" + safe = "".join(ch if ch.isalnum() or ch == "-" else "-" for ch in value.lower()).strip("-") + digest = hashlib.sha1(value.encode("utf-8")).hexdigest()[:8] + base = f"{prefix}-{digest}-{safe}" + return base[:63].rstrip("-") + + +def kubeOvnVpc(namespace_name: str, vpc_name: str) -> dict[str, Any]: + """Return a Kube-OVN Vpc that isolates one SeedEMU namespace.""" + return { + "apiVersion": "kubeovn.io/v1", + "kind": "Vpc", + "metadata": {"name": vpc_name}, + "spec": {"namespaces": [namespace_name]}, + } + + +def convertNetworkAttachmentToKubeOvn( + doc: dict[str, Any], + default_namespace: str, + vpc_name: str, + attached_cni_type: str, + cni_master_interface: str | None, +) -> list[dict[str, Any]]: + """Return [Subnet, NAD] for one compiler-generated macvlan NAD. + + Args: + doc: Original NetworkAttachmentDefinition document. + default_namespace: Namespace to use when the NAD omits metadata.namespace. + vpc_name: Namespace-scoped Kube-OVN VPC name. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. + """ + metadata = doc.get("metadata") or {} + nad_name = str(metadata.get("name") or "") + namespace_name = str(metadata.get("namespace") or default_namespace) + if not nad_name: + return [doc] + annotations = metadata.get("annotations") or {} + prefix = annotations.get("org.seedsecuritylabs.seedemu.meta.prefix") + if not prefix: + raise SystemExit(f"NAD {namespace_name}/{nad_name} lacks SeedEMU prefix annotation") + + use_ovn_attached = isKubeOvnAttachedCni(attached_cni_type) + provider = kubeOvnProviderName(nad_name, namespace_name, attached_cni_type) + subnet_name = kubeOvnResourceName("subnet", f"{namespace_name}-{nad_name}") + subnet = { + "apiVersion": "kubeovn.io/v1", + "kind": "Subnet", + "metadata": {"name": subnet_name}, + "spec": { + "protocol": "IPv4", + "provider": provider, + "cidrBlock": str(prefix), + "gateway": firstUsableIp(str(prefix)), + "natOutgoing": False, + "private": False, + }, + } + if use_ovn_attached: + subnet["spec"]["vpc"] = vpc_name + subnet["spec"]["gatewayType"] = "distributed" + + converted = dict(doc) + converted["spec"] = {"config": renderConvertedNadConfig(doc, provider, attached_cni_type, cni_master_interface)} + return [subnet, converted] + + +def renderConvertedNadConfig( + doc: dict[str, Any], + provider: str, + attached_cni_type: str, + cni_master_interface: str | None, +) -> str: + """Return NetworkAttachmentDefinition spec.config for the selected CNI. + + Args: + doc: Original NetworkAttachmentDefinition document. + provider: Kube-OVN provider string used to match the Subnet. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + cni_master_interface: Optional macvlan parent interface override. + """ + if isKubeOvnAttachedCni(attached_cni_type): + config = { + "cniVersion": "0.3.1", + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + spec = doc.get("spec") or {} + raw_config = spec.get("config") + try: + config = json.loads(raw_config) if isinstance(raw_config, str) else {} + except json.JSONDecodeError: + config = {} + if not isinstance(config, dict): + config = {} + config.setdefault("cniVersion", "0.3.1") + config["type"] = attached_cni_type + if attached_cni_type == "macvlan": + if cni_master_interface: + config["master"] = cni_master_interface + config.setdefault("mode", "bridge") + config["ipam"] = { + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": provider, + } + return json.dumps(config, separators=(",", ":")) + + +def convertWorkloadAnnotationsToKubeOvn( + doc: dict[str, Any], + default_namespace: str, + attached_cni_type: str, +) -> None: + """Rewrite Multus static IP annotations for Kube-OVN attached NICs. + + Args: + doc: Manifest document to mutate in place. Deployments and raw Pods are + supported because both can contain Multus network annotations. + default_namespace: Namespace to use when a network selection omits it. + attached_cni_type: Secondary CNI type, for example kube-ovn or macvlan. + + SeedEMU's compiler writes Multus `ips: ["10.x.x.x/24"]`, which is correct + for static CNI IPAM. Kube-OVN IPAM uses per-provider annotations and + expects plain IP values. Raw Pods use `ip_address`; workload templates use + `ip_pool`, otherwise kube-ovn-controller rejects Deployment pods during + address allocation. + """ + kind = str(doc.get("kind") or "") + annotations = getPodTemplateAnnotations(doc) + if not annotations: + return + raw_networks = annotations.get("k8s.v1.cni.cncf.io/networks") + if not isinstance(raw_networks, str): + return + try: + networks = json.loads(raw_networks) + except json.JSONDecodeError: + return + if not isinstance(networks, list): + return + + changed = False + for item in networks: + if not isinstance(item, dict): + continue + ips = item.pop("ips", None) + if not ips: + continue + nad_name, nad_namespace = parseNetworkSelection(item, default_namespace) + if not nad_name: + continue + ip_values = [stripCidr(str(ip_value)) for ip_value in ips if str(ip_value).strip()] + if not ip_values: + continue + static_field = kubeOvnStaticAddressField(kind) + provider = kubeOvnProviderName(nad_name, nad_namespace, attached_cni_type) + subnet_name = kubeOvnResourceName("subnet", f"{nad_namespace}-{nad_name}") + annotations[f"{provider}.kubernetes.io/{static_field}"] = ",".join(ip_values) + annotations[f"{provider}.kubernetes.io/logical_switch"] = subnet_name + changed = True + + if changed: + annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(networks, separators=(",", ":")) + + +def kubeOvnProviderName(nad_name: str, namespace_name: str, attached_cni_type: str) -> str: + """Return the provider name used by Kube-OVN for one NetworkAttachmentDefinition.""" + provider = f"{nad_name}.{namespace_name}" + if isKubeOvnAttachedCni(attached_cni_type): + provider = f"{provider}.ovn" + return provider + + +def kubeOvnStaticAddressField(kind: str) -> str: + """Return the Kube-OVN fixed-address annotation field for a resource kind. + + Args: + kind: Kubernetes resource kind containing the Multus annotation. + """ + if kind == "Pod": + return "ip_address" + return "ip_pool" + + +def getPodTemplateAnnotations(doc: dict[str, Any]) -> dict[str, str] | None: + """Return pod-level annotations from a workload or Pod document. + + Args: + doc: Kubernetes resource document. + """ + kind = doc.get("kind") + if kind == "Pod": + metadata = doc.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + if kind in {"Deployment", "DaemonSet", "StatefulSet", "Job"}: + template = doc.setdefault("spec", {}).setdefault("template", {}) + metadata = template.setdefault("metadata", {}) + annotations = metadata.setdefault("annotations", {}) + return annotations if isinstance(annotations, dict) else None + return None + + +def parseNetworkSelection(item: dict[str, Any], default_namespace: str) -> tuple[str, str]: + """Return (nad_name, namespace) from one Multus network selection item. + + Args: + item: One object from `k8s.v1.cni.cncf.io/networks`. + default_namespace: Namespace fallback when the item omits namespace. + """ + raw_name = str(item.get("name") or "") + namespace = str(item.get("namespace") or default_namespace) + if "/" in raw_name: + namespace, raw_name = raw_name.split("/", 1) + return raw_name, namespace + + +def stripCidr(ip_value: str) -> str: + """Return the host IP part from a CIDR or plain IP string.""" + return ip_value.strip().split("/", 1)[0] + + +def firstUsableIp(cidr: str) -> str: + """Return the first usable IPv4 address in a CIDR block.""" + network = ipaddress.ip_network(cidr, strict=False) + if network.version != 4: + raise SystemExit(f"Kube-OVN renderer currently supports IPv4 only: {cidr}") + hosts = network.hosts() + try: + return str(next(hosts)) + except StopIteration: + return str(network.network_address) + + +def networkAttachmentPatches(manifest_path: str, cni_master_interface: str) -> list[dict[str, Any]]: + """Return kustomize JSON6902 patches for NetworkAttachmentDefinition master. + + Args: + manifest_path: Source k8s.yaml path. + cni_master_interface: Physical parent interface for macvlan networks. + + Compile output is intentionally registry/fabric agnostic. The running + stage rewrites only the macvlan parent interface, leaving the original + k8s.yaml untouched and making physical/KVM deployments selectable by YAML. + """ + if not cni_master_interface: + return [] + patches: list[dict[str, Any]] = [] + with open(manifest_path, "r", encoding="utf-8") as fh: + for doc in yaml.safe_load_all(fh): + if not isinstance(doc, dict) or doc.get("kind") != "NetworkAttachmentDefinition": + continue + metadata = doc.get("metadata") or {} + name = metadata.get("name") + if not name: + continue + spec = doc.get("spec") or {} + raw_config = spec.get("config") + if not isinstance(raw_config, str): + continue + try: + cni_config = json.loads(raw_config) + except json.JSONDecodeError: + continue + if not isinstance(cni_config, dict) or cni_config.get("type") != "macvlan": + continue + if cni_config.get("master") == cni_master_interface: + continue + cni_config["master"] = cni_master_interface + target = { + "group": "k8s.cni.cncf.io", + "version": "v1", + "kind": "NetworkAttachmentDefinition", + "name": str(name), + } + namespace_name = metadata.get("namespace") + if namespace_name: + target["namespace"] = str(namespace_name) + patch = [ + { + "op": "replace", + "path": "/spec/config", + "value": json.dumps(cni_config, separators=(",", ":")), + } + ] + patches.append({"target": target, "patch": yaml.safe_dump(patch, sort_keys=False)}) + return patches + + +def deployment_names(args: argparse.Namespace) -> None: + with open(args.manifest, "r", encoding="utf-8") as fh: + for doc in yaml.safe_load_all(fh): + if isinstance(doc, dict) and doc.get("kind") == "Deployment": + name = (doc.get("metadata") or {}).get("name") + if name: + print(name) + + +def namespace(args: argparse.Namespace) -> None: + print(findNamespaceName(loadManifestDocs(Path(args.manifest)))) + + +def validate_manifest(args: argparse.Namespace) -> None: + seen = {} + duplicate_errors = [] + with open(args.manifest, "r", encoding="utf-8") as fh: + for index, doc in enumerate(yaml.safe_load_all(fh), 1): + if not isinstance(doc, dict): + continue + kind = doc.get("kind") + metadata = doc.get("metadata") or {} + name = metadata.get("name") + namespace_name = metadata.get("namespace") or "" + if not kind or not name: + continue + key = (kind, namespace_name, name) + if key in seen: + duplicate_errors.append( + f"{kind}/{name} namespace={namespace_name or ''} " + f"appears in docs {seen[key]} and {index}" + ) + else: + seen[key] = index + + if duplicate_errors: + print(f"Duplicate Kubernetes resources in {args.manifest}:", flush=True) + for error in duplicate_errors[:30]: + print(f" {error}", flush=True) + if len(duplicate_errors) > 30: + print(f" ... {len(duplicate_errors) - 30} more", flush=True) + raise SystemExit(1) + + +def snake_case(value: str) -> str: + out = [] + for char in value: + if char.isupper(): + out.append("_") + out.append(char.lower()) + else: + out.append(char) + return "".join(out).lstrip("_") + + +def camel_case(value: str) -> str: + parts = value.split("_") + return parts[0] + "".join(part[:1].upper() + part[1:] for part in parts[1:]) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + config = subparsers.add_parser("config-value") + config.add_argument("--config", required=True) + config.add_argument("--key", required=True) + config.set_defaults(func=config_value) + + access = subparsers.add_parser("node-access") + access.add_argument("--config", required=True) + access.add_argument("--name") + access.set_defaults(func=node_access) + + mapped = subparsers.add_parser("mapped-images") + mapped.add_argument("--images-yaml", required=True) + mapped.add_argument("--image-registry-prefix", required=True) + mapped.add_argument("--registry-prefix", required=True) + mapped.set_defaults(func=mapped_images) + + kustomization = subparsers.add_parser("kustomization") + kustomization.add_argument("--images-yaml", required=True) + kustomization.add_argument("--manifest", required=True) + kustomization.add_argument("--image-registry-prefix", required=True) + kustomization.add_argument("--registry-prefix", required=True) + kustomization.add_argument("--network-backend", default="macvlan") + kustomization.add_argument("--cni-master-interface", default="") + kustomization.add_argument("--attached-cni-type", default=None) + kustomization.add_argument("--output", required=True) + kustomization.set_defaults(func=render_kustomization) + + deployments = subparsers.add_parser("deployment-names") + deployments.add_argument("--manifest", required=True) + deployments.set_defaults(func=deployment_names) + + ns = subparsers.add_parser("namespace") + ns.add_argument("--manifest", required=True) + ns.set_defaults(func=namespace) + + validate = subparsers.add_parser("validate-manifest") + validate.add_argument("--manifest", required=True) + validate.set_defaults(func=validate_manifest) + + args = parser.parse_args() + args.func(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/running/manageRunningStage.py b/seedemu/k8sTools/resources/running/manageRunningStage.py new file mode 100755 index 000000000..3896579f7 --- /dev/null +++ b/seedemu/k8sTools/resources/running/manageRunningStage.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +"""Run the native Kubernetes workload stage without a Makefile. + +The setup stage writes a small ``configRunning.yaml`` that points at: + +- ``configK3s.yaml`` for kubeconfig, registry, SSH, and fabric settings. +- the compiler output directory for ``k8s.yaml``/``k8s.kube-ovn.yaml`` and + ``images.yaml``. + +This script intentionally keeps the old Makefile target names as subcommands +(``preflight``, ``build``, ``up``, ``clean``) while implementing the orchestration +in Python. Low-level tools such as ``kubectl``, ``ssh``, ``tar``, and ``docker`` +are still executed as external commands because they are the actual system +interfaces for Kubernetes and image builds. +""" +from __future__ import annotations + +import argparse +import os +import shlex +import shutil +import subprocess +import sys +import time +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +HELPER = SCRIPT_DIR / "manageK8sManifest.py" + + +def runCommand(args: list[str], *, cwd: str | Path | None = None, stdin=None) -> subprocess.CompletedProcess: + """Run one command and fail immediately on non-zero exit. + + Args: + args: Command argv. Shell expansion is avoided unless the caller + explicitly invokes a shell program such as ``ssh`` remote command. + cwd: Optional working directory. + stdin: Optional stdin pipe for streaming tar data to SSH. + """ + print("+ " + " ".join(str(arg) for arg in args)) + return subprocess.run(args, cwd=str(cwd) if cwd is not None else None, stdin=stdin, check=True) + + +def helperOutput(args: list[str]) -> str: + """Run ``manageK8sManifest.py`` and return stripped stdout. + + Args: + args: Helper subcommand arguments without the Python executable. + """ + return subprocess.check_output(["python3", str(HELPER), *args], text=True).strip() + + +def context(config: Path) -> dict[str, str]: + """Resolve all running-stage settings from configRunning.yaml. + + Args: + config: Path to configRunning.yaml. + """ + keys = [ + "outputDir", + "manifest", + "imagesYaml", + "kustomization", + "imageRegistryPrefix", + "registryPrefix", + "kubeconfig", + "sshUser", + "sshKey", + "masterConnection", + "cniMasterInterface", + "networkBackend", + "attachedCniType", + "rolloutTimeoutSeconds", + ] + return {key: helperOutput(["config-value", "--config", str(config), "--key", key]) for key in keys} + + +def checkOutput(config: Path) -> dict[str, str]: + """Validate compiler output and return the resolved running context. + + Args: + config: Path to configRunning.yaml. + """ + values = context(config) + manifest = Path(values["manifest"]) + images_yaml = Path(values["imagesYaml"]) + if not manifest.is_file(): + raise SystemExit(f"Missing {manifest}. Run compile first.") + if not images_yaml.is_file(): + raise SystemExit(f"Missing {images_yaml}. Run compile first.") + runCommand(["python3", str(HELPER), "validate-manifest", "--manifest", str(manifest)]) + return values + + +def preflight(config: Path) -> None: + """Validate cluster readiness before image build or deployment. + + Args: + config: Path to configRunning.yaml. + """ + checkOutput(config) + runCommand(["python3", str(SCRIPT_DIR / "validateClusterPreflight.py"), "--config", str(config)]) + + +def copyTreeContents(source: Path, target: Path) -> None: + """Copy one directory's contents into a clean target directory. + + Args: + source: Existing source directory. + target: Destination directory to create. + """ + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True, exist_ok=True) + for item in source.iterdir(): + destination = target / item.name + if item.is_dir(): + shutil.copytree(item, destination) + else: + shutil.copy2(item, destination) + + +def streamTarToRemote(source: Path, ssh_command: list[str], remote_extract_command: str) -> None: + """Stream a local directory to a remote extraction command over SSH. + + Args: + source: Directory whose contents should be archived. + ssh_command: Base SSH argv including target host. + remote_extract_command: Remote shell command that extracts stdin tar. + """ + print("+ " + " ".join(["tar", "-C", str(source), "-czf", "-", "."]) + " | " + " ".join(ssh_command + [remote_extract_command])) + producer = subprocess.Popen(["tar", "-C", str(source), "-czf", "-", "."], stdout=subprocess.PIPE) + try: + assert producer.stdout is not None + subprocess.run([*ssh_command, remote_extract_command], stdin=producer.stdout, check=True) + finally: + if producer.stdout is not None: + producer.stdout.close() + return_code = producer.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, ["tar", "-C", str(source), "-czf", "-", "."]) + + +def firstFromImage(dockerfile: Path) -> str | None: + """Return the first Dockerfile FROM image, handling optional flags. + + Args: + dockerfile: Dockerfile to inspect. + """ + for raw_line in dockerfile.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if not parts or parts[0].upper() != "FROM": + continue + index = 1 + while index < len(parts) and parts[index].startswith("--"): + index += 1 + if index < len(parts): + return parts[index] + return None + + +def isCompilerHashBaseTag(image: str) -> bool: + """Return true for generated compiler hash base tags. + + Args: + image: Docker image reference from a FROM line. + """ + tag = image.removesuffix(":latest") + return len(tag) == 32 and all(char in "0123456789abcdef" for char in tag) + + +def externalBuildBaseImages(output_dir: Path) -> list[str]: + """Collect non-generated Docker FROM images needed by remote buildx. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + """ + images: list[str] = [] + seen: set[str] = set() + for dockerfile in sorted(output_dir.rglob("Dockerfile")): + image = firstFromImage(dockerfile) + if not image: + continue + if image == "scratch" or image.startswith("$") or "${" in image: + continue + if isCompilerHashBaseTag(image): + continue + if image not in seen: + seen.add(image) + images.append(image) + return images + + +def dockerImageExists(image: str) -> bool: + """Return true when the local Docker daemon has one image reference. + + Args: + image: Docker image reference. + """ + return subprocess.run( + ["docker", "image", "inspect", image], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def dockerIoMirrorRef(image: str) -> str | None: + """Return a Docker Hub mirror reference for unqualified Docker Hub images. + + Args: + image: Docker image reference. + """ + if "/" in image: + first = image.split("/", 1)[0] + if "." in first or ":" in first or first == "localhost": + return None + return f"docker.m.daocloud.io/{image}" + return f"docker.m.daocloud.io/library/{image}" + + +def ensureLocalDockerImage(image: str) -> None: + """Ensure the local Docker daemon has one image, pulling if required. + + Args: + image: Docker image reference. + """ + if dockerImageExists(image): + print(f"[k8s_build] local Docker already has {image}") + return + + print(f"[k8s_build] pulling build base image on local host: {image}") + result = subprocess.run(["docker", "pull", image], check=False) + if result.returncode == 0: + return + + mirror_image = dockerIoMirrorRef(image) + if mirror_image is None: + raise subprocess.CalledProcessError(result.returncode, ["docker", "pull", image]) + + print(f"[k8s_build] pulling build base image from mirror: {mirror_image}") + runCommand(["docker", "pull", mirror_image]) + runCommand(["docker", "tag", mirror_image, image]) + + +def remoteDockerImageExists(image: str, ssh_command: list[str]) -> bool: + """Return true when the remote Docker daemon has one image reference. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + return subprocess.run( + [*ssh_command, f"sudo -n docker image inspect {shlex.quote(image)} >/dev/null 2>&1"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + +def streamDockerImageToRemote(image: str, ssh_command: list[str]) -> None: + """Load one local Docker image into the remote Docker daemon over SSH. + + Args: + image: Docker image reference. + ssh_command: Base SSH argv including target host. + """ + if remoteDockerImageExists(image, ssh_command): + print(f"[k8s_build] registry host Docker already has {image}") + return + + ensureLocalDockerImage(image) + print("+ " + " ".join(["docker", "save", image]) + " | " + " ".join(ssh_command + ["sudo -n docker load"])) + producer = subprocess.Popen(["docker", "save", image], stdout=subprocess.PIPE) + try: + assert producer.stdout is not None + subprocess.run([*ssh_command, "sudo -n docker load"], stdin=producer.stdout, check=True) + finally: + if producer.stdout is not None: + producer.stdout.close() + return_code = producer.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, ["docker", "save", image]) + + +def ensureBuildBaseImages(output_dir: Path, ssh_command: list[str] | None) -> None: + """Ensure Dockerfile FROM images are available where buildx will run. + + Args: + output_dir: Compiler output directory containing Docker build contexts. + ssh_command: Base SSH argv for a remote registry host, or None when the + registry/build host is local. + """ + images = externalBuildBaseImages(output_dir) + if not images: + return + print("[k8s_build] ensuring Docker build base images: " + ", ".join(images)) + if ssh_command is None: + for image in images: + ensureLocalDockerImage(image) + return + for image in images: + streamDockerImageToRemote(image, ssh_command) + + +def buildImages(config: Path) -> None: + """Stage compiler output on the registry node and build/push images. + + Args: + config: Path to configRunning.yaml. + """ + values = checkOutput(config) + output_dir = Path(values["outputDir"]) + registry_prefix = values["registryPrefix"].rstrip("/") + registry_host = registry_prefix.split("/", 1)[0].split(":", 1)[0] + remote_dir = f"/tmp/seedemu-native-build-{time.strftime('%Y%m%d_%H%M%S')}-{os.getpid()}" + build_command = [ + "python3", + "./buildRegistryImages.py", + "--output-dir", + f"{remote_dir}/output", + "--registry-prefix", + registry_prefix, + "--image-registry-prefix", + values["imageRegistryPrefix"], + ] + + if values["masterConnection"] == "local": + print(f"[k8s_build] master is local; staging compile output in {remote_dir}/output") + remote_root = Path(remote_dir) + if remote_root.exists(): + shutil.rmtree(remote_root) + (remote_root / "output").mkdir(parents=True) + (remote_root / "running").mkdir(parents=True) + copyTreeContents(output_dir, remote_root / "output") + copyTreeContents(SCRIPT_DIR, remote_root / "running") + ensureBuildBaseImages(output_dir, None) + print("[k8s_build] running local build") + runCommand(["sudo", "-n", *build_command], cwd=remote_root / "running") + return + + ssh_target = f"{values['sshUser']}@{registry_host}" + ssh_command = [ + "ssh", + "-i", + values["sshKey"], + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "LogLevel=ERROR", + "-o", + "BatchMode=yes", + ssh_target, + ] + print(f"[k8s_build] uploading compile output to {ssh_target}:{remote_dir}/output") + ensureBuildBaseImages(output_dir, ssh_command) + runCommand( + [ + *ssh_command, + f"rm -rf {shlex.quote(remote_dir)} && mkdir -p {shlex.quote(remote_dir + '/output')} {shlex.quote(remote_dir + '/running')}", + ] + ) + streamTarToRemote(output_dir, ssh_command, f"tar -C {shlex.quote(remote_dir + '/output')} -xzf -") + streamTarToRemote(SCRIPT_DIR, ssh_command, f"tar -C {shlex.quote(remote_dir + '/running')} -xzf -") + remote_build = " ".join(shlex.quote(part) for part in build_command) + print(f"[k8s_build] running build on {ssh_target}") + runCommand([*ssh_command, f"cd {shlex.quote(remote_dir + '/running')} && sudo -n {remote_build}"]) + + +def renderKustomization(config: Path) -> dict[str, str]: + """Render kustomization.yaml and return the resolved running context. + + Args: + config: Path to configRunning.yaml. + """ + values = checkOutput(config) + runCommand( + [ + "python3", + str(HELPER), + "kustomization", + "--images-yaml", + values["imagesYaml"], + "--manifest", + values["manifest"], + "--image-registry-prefix", + values["imageRegistryPrefix"], + "--registry-prefix", + values["registryPrefix"], + "--network-backend", + values["networkBackend"], + "--cni-master-interface", + values["cniMasterInterface"], + "--attached-cni-type", + values["attachedCniType"], + "--output", + values["kustomization"], + ] + ) + print(f"[k8s_up] wrote {values['kustomization']}") + return values + + +def namespaceForManifest(manifest: str) -> str: + """Return namespace from one Kubernetes manifest path.""" + return helperOutput(["namespace", "--manifest", manifest]) + + +def deploy(config: Path) -> None: + """Apply workload resources and wait for all SeedEMU pods to become ready. + + Args: + config: Path to configRunning.yaml. + """ + values = renderKustomization(config) + namespace = namespaceForManifest(values["manifest"]) + print("=== native-k8s deploy ===") + print(f"output_dir={values['outputDir']}") + print(f"manifest={values['manifest']}") + print(f"kustomization={values['kustomization']}") + print(f"network_backend={values['networkBackend']}") + print(f"kubeconfig={values['kubeconfig']}") + print(f"namespace={namespace}") + runCommand(["kubectl", "--kubeconfig", values["kubeconfig"], "apply", "-k", values["outputDir"]]) + waitReady(config) + + +def waitReady(config: Path) -> None: + """Wait for deployment rollouts and SeedEMU pods. + + Args: + config: Path to configRunning.yaml. + """ + values = checkOutput(config) + namespace = namespaceForManifest(values["manifest"]) + names = helperOutput(["deployment-names", "--manifest", values["manifest"]]).splitlines() + for name in names: + if not name: + continue + print(f"[rollout] deployment/{name}") + runCommand( + [ + "kubectl", + "--kubeconfig", + values["kubeconfig"], + "-n", + namespace, + "rollout", + "status", + f"deployment/{name}", + f"--timeout={values['rolloutTimeoutSeconds']}s", + ] + ) + runCommand( + [ + "kubectl", + "--kubeconfig", + values["kubeconfig"], + "-n", + namespace, + "wait", + "--for=condition=Ready", + "pod", + "-l", + "seedemu.io/workload=seedemu", + f"--timeout={values['rolloutTimeoutSeconds']}s", + ] + ) + print("Deploy completed successfully.") + + +def clean(config: Path) -> None: + """Delete workload resources and namespace. + + Args: + config: Path to configRunning.yaml. + """ + values = checkOutput(config) + namespace = namespaceForManifest(values["manifest"]) + if Path(values["kustomization"]).is_file(): + print(f"[clean] deleting resources from {values['kustomization']}") + subprocess.run( + [ + "kubectl", + "--kubeconfig", + values["kubeconfig"], + "delete", + "-k", + values["outputDir"], + "--ignore-not-found=true", + "--wait=false", + ], + check=False, + ) + else: + print("[clean] kustomization not found; deleting namespace only") + exists = subprocess.run( + ["kubectl", "--kubeconfig", values["kubeconfig"], "get", "namespace", namespace], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + if exists: + print(f"[clean] deleting namespace {namespace}") + runCommand(["kubectl", "--kubeconfig", values["kubeconfig"], "delete", "namespace", namespace, "--wait=false"]) + else: + print(f"[clean] namespace {namespace} does not exist") + + +def parseArgs(argv: list[str] | None) -> argparse.Namespace: + """Parse CLI arguments for the running-stage replacement. + + Args: + argv: Optional argv override for tests. + """ + parser = argparse.ArgumentParser(prog="manageRunningStage.py") + parser.add_argument( + "--config", + default=str(SCRIPT_DIR / "configRunning.yaml"), + help="Path to configRunning.yaml.", + ) + sub = parser.add_subparsers(dest="command", required=True) + for name in ["check-output", "preflight", "build", "render-kustomization", "up", "wait", "clean"]: + sub.add_parser(name) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Run one running-stage command.""" + args = parseArgs(argv) + config = Path(args.config).expanduser().resolve() + if args.command == "check-output": + checkOutput(config) + elif args.command == "preflight": + preflight(config) + elif args.command == "build": + buildImages(config) + elif args.command == "render-kustomization": + renderKustomization(config) + elif args.command == "up": + deploy(config) + elif args.command == "wait": + waitReady(config) + elif args.command == "clean": + clean(config) + else: + raise SystemExit(f"unsupported command: {args.command}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/seedemu/k8sTools/resources/running/validateClusterPreflight.py b/seedemu/k8sTools/resources/running/validateClusterPreflight.py new file mode 100755 index 000000000..7bc9d8eb5 --- /dev/null +++ b/seedemu/k8sTools/resources/running/validateClusterPreflight.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Python entrypoint for validateClusterPreflight with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that compile output, kubeconfig, registry, remote Docker/buildx, and +# all Kubernetes nodes are ready before running make build/up. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${SCRIPT_DIR}/configRunning.yaml" +HELPER="${SCRIPT_DIR}/manageK8sManifest.py" + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +OUTPUT_DIR="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key outputDir)" +MANIFEST="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key manifest)" +IMAGES_YAML="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imagesYaml)" +KUBECONFIG_PATH="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key kubeconfig)" +REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key registryPrefix)" +IMAGE_REGISTRY_PREFIX="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key imageRegistryPrefix)" +SSH_USER="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshUser)" +SSH_KEY="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key sshKey)" +MASTER_CONNECTION="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key masterConnection)" +NETWORK_BACKEND="$(python3 "${HELPER}" config-value --config "${CONFIG_PATH}" --key networkBackend)" + +ssh_args=(-n -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8) + +fail() { + echo "[preflight][ERROR] $*" >&2 + exit 1 +} + +requireCommand() { + # Args: + # $1: command name expected in PATH + command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" +} + +requireFile() { + # Args: + # $1: required file path + [ -f "$1" ] || fail "missing file: $1" +} + +registry_ref="${REGISTRY_PREFIX#http://}" +registry_ref="${registry_ref#https://}" +registry_ref="${registry_ref%%/*}" +registry_host="${registry_ref%%:*}" +registry_url="http://${registry_ref}" + +declare -A node_config_ip=() +declare -A node_connection=() +declare -A node_user=() +declare -A node_key=() +while IFS=$'\t' read -r access_name access_ip access_connection access_user access_key; do + [ -n "${access_name}" ] || continue + node_config_ip["${access_name}"]="${access_ip}" + node_connection["${access_name}"]="${access_connection}" + node_user["${access_name}"]="${access_user}" + node_key["${access_name}"]="${access_key}" +done < <(python3 "${HELPER}" node-access --config "${CONFIG_PATH}") + +echo "=== native-k8s preflight ===" +echo "config=${CONFIG_PATH}" +echo "output_dir=${OUTPUT_DIR}" +echo "manifest=${MANIFEST}" +echo "images_yaml=${IMAGES_YAML}" +echo "kubeconfig=${KUBECONFIG_PATH}" +echo "registry_prefix=${REGISTRY_PREFIX}" +echo "image_registry_prefix=${IMAGE_REGISTRY_PREFIX}" +echo "network_backend=${NETWORK_BACKEND}" + +echo "[1/7] Local tools and compile output" +requireCommand python3 +requireCommand kubectl +requireCommand curl +requireCommand ssh +requireFile "${HELPER}" +requireFile "${MANIFEST}" +requireFile "${IMAGES_YAML}" +requireFile "${KUBECONFIG_PATH}" +[ -d "${OUTPUT_DIR}" ] || fail "missing output directory: ${OUTPUT_DIR}" +[ -s "${MANIFEST}" ] || fail "empty manifest: ${MANIFEST}" +[ -s "${IMAGES_YAML}" ] || fail "empty images yaml: ${IMAGES_YAML}" + +namespace="$(python3 "${HELPER}" namespace --manifest "${MANIFEST}")" +[ -n "${namespace}" ] || fail "cannot determine namespace from ${MANIFEST}" +image_count="$(python3 "${HELPER}" mapped-images --images-yaml "${IMAGES_YAML}" --image-registry-prefix "${IMAGE_REGISTRY_PREFIX}" --registry-prefix "${REGISTRY_PREFIX}" | wc -l)" +[ "${image_count}" -gt 0 ] || fail "no images found in ${IMAGES_YAML}" +echo "namespace=${namespace}" +echo "image_count=${image_count}" + +echo "[2/7] Kubeconfig and API server" +kubectl --kubeconfig "${KUBECONFIG_PATH}" version --client=true >/dev/null +kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide + +echo "[3/7] Node readiness" +not_ready="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ + | awk '$2 != "True" {print $1}' +)" +[ -z "${not_ready}" ] || fail "not-ready nodes: ${not_ready//$'\n'/ }" + +echo "[4/7] kube-system baseline" +kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -o wide +bad_system_pods="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods --no-headers 2>/dev/null \ + | awk '$3 !~ /^(Running|Completed)$/ {print $1 ":" $3}' +)" +[ -z "${bad_system_pods}" ] || fail "kube-system has unhealthy pods: ${bad_system_pods//$'\n'/ }" +if [ "${NETWORK_BACKEND}" = "kube-ovn" ]; then + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status deployment/kube-ovn-controller --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/kube-ovn-cni --timeout=300s + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/ovs-ovn --timeout=300s +fi + +echo "[5/7] Namespace baseline" +if kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; then + workload_count="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" \ + get pods,deployments.apps,replicasets.apps,statefulsets.apps,daemonsets.apps,jobs.batch,cronjobs.batch \ + --ignore-not-found --no-headers 2>/dev/null | wc -l + )" + if [ "${workload_count}" -gt 0 ]; then + fail "namespace ${namespace} already has ${workload_count} workload resources; run make clean or wait for previous cleanup before make up" + fi + echo "namespace ${namespace} already exists without workload resources; continuing" +else + echo "namespace ${namespace} is absent" +fi + +echo "[6/7] Registry health" +code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" +case "${code}" in + 200|401) echo "local registry_http=${code}" ;; + *) fail "registry ${registry_url}/v2/ is not healthy from local host, http=${code}" ;; +esac + +ssh_target="${SSH_USER}@${registry_host}" +if [ "${MASTER_CONNECTION}" = "local" ]; then + echo "registry-host local registry_http=${code}" +else + requireFile "${SSH_KEY}" + ssh "${ssh_args[@]}" "${ssh_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/'" \ + | awk '{print "registry-host registry_http="$0; if ($0 != "200" && $0 != "401") exit 1}' \ + || fail "registry ${registry_url}/v2/ is not healthy from ${ssh_target}" +fi + +echo "[7/7] Remote build prerequisites and node registry reachability" +if [ "${MASTER_CONNECTION}" = "local" ]; then + sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null \ + || fail "docker/buildx is not ready on local registry host" +else + ssh "${ssh_args[@]}" "${ssh_target}" "sudo -n docker version >/dev/null && sudo -n docker buildx version >/dev/null" \ + || fail "docker/buildx is not ready on ${ssh_target}" +fi +echo "registry-host docker/buildx ok" + +while IFS=$'\t' read -r node_name node_ip; do + [ -n "${node_name}" ] || continue + node_access="${node_connection[$node_name]:-}" + [ -n "${node_access}" ] || fail "node ${node_name} is not present in configK3s.yaml" + target_ip="${node_ip:-${node_config_ip[$node_name]}}" + [ -n "${target_ip}" ] || fail "cannot determine InternalIP for node ${node_name}" + if [ "${node_access}" = "local" ]; then + code="$(curl -s -o /dev/null -w '%{http_code}' "${registry_url}/v2/" || true)" + else + requireFile "${node_key[$node_name]}" + node_target="${node_user[$node_name]}@${target_ip}" + node_ssh_args=(-n -i "${node_key[$node_name]}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=8) + code="$(ssh "${node_ssh_args[@]}" "${node_target}" "curl -s -o /dev/null -w '%{http_code}' '${registry_url}/v2/' || true")" + fi + printf '%s\t%s\tregistry_http=%s\n' "${node_name}" "${node_ip}" "${code}" + case "${code}" in + 200|401) ;; + *) fail "registry ${registry_url}/v2/ is not reachable from ${node_name} (${node_ip}), http=${code}" ;; + esac +done < <( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' +) + +echo "Preflight completed" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/README.md b/seedemu/k8sTools/resources/setup/README.md new file mode 100644 index 000000000..b8cd8cbb4 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/README.md @@ -0,0 +1,90 @@ +# k8sTools Setup Resources + +这个目录是 `seedemu.k8sTools` 的内部资源目录。用户通常不会直接看到它，因为 +`K8sTools.build()` 会把这些资源复制到临时目录中执行，完成后只保留用户指定的 +`configK3s.yaml`、`kubeconfig.yaml`，以及可选的 `inventory.yaml`。 + +所有用户侧和内部操作入口都是 Python 文件。部分复杂系统操作仍保留原 shell +命令序列，但正文内嵌在对应 Python 入口中，不再依赖相邻 `.sh` 资源文件。 +这些入口仍会调用系统工具，例如 `virsh`、`docker`、`kubectl`、`ssh`、 +`ansible-playbook` 和 `helm`；这些命令才是真正的 KVM、K3s、OVN/OVS +和镜像操作接口。 + +## 资源分组 + +| 路径 | 作用 | +| --- | --- | +| `applyK3sCluster.py` | 读取 `configK3s.yaml`，安装或重装 K3s，配置 master registry，导入 bootstrap 镜像，安装 Multus，并在 `fabric.type=ovn` 时调用 Kube-OVN 安装入口。 | +| `destroyPhysicalCluster.py` | 按 `configK3s.yaml` 清理 K3s、registry、kubeconfig/inventory，并调用对应 fabric 清理入口。 | +| `preparePhysicalNodes.py` | 已有物理机预检查：验证 SSH、`sudo -n`、基础命令和可选 VXLAN underlay 接口。 | +| `manageK3sConfig.py` | K3s 阶段 YAML 解析器。负责生成 Ansible inventory、shell 变量、fabric 参数、node-name 和 running 阶段需要的 registry/kubeconfig 信息。 | +| `ansible/k3s-install.yml` | K3s 安装 playbook 模板。它不是运行时生成文件，必须作为资源保留。 | +| `kvm/prepareHostAssets.py` | 准备 Ubuntu cloud image 和 Docker 镜像 tar 缓存。 | +| `kvm/createKvmVms.py` | 创建本机 libvirt/KVM VM，生成 `configK3s.yaml` 和 `kvmState.yaml`。 | +| `kvm/tuneVmLimits.py` | 通过 SSH 对 VM 写入文件句柄、netns、邻居表、cni0 hash 等高密度实验限制。 | +| `kvm/destroyKvmVms.py` | 根据 `kvmState.yaml` 清理 VM、磁盘、cloud-init 和 DHCP reservation。 | +| `kvm/manageKvmConfig.py` | KVM YAML 解析器，负责避开已有 VM/IP/MAC 冲突并生成 VM 计划。 | +| `multiHostKvm/prepareKvmHypervisors.py` | 多物理机 KVM 前置准备：验证 hypervisor、创建 routed libvirt network、配置跨 VM subnet 路由或 tunnel。 | +| `multiHostKvm/createMultiHostKvmVms.py` | 在多台 hypervisor 上创建 VM，生成全局 `configK3s.yaml` 和 `multiHostKvmState.yaml`。 | +| `multiHostKvm/destroyMultiHostKvmVms.py` | 根据 `multiHostKvmState.yaml` 到每台 hypervisor 清理 VM 和路由/network 状态。 | +| `multiHostKvm/manageMultiHostKvmConfig.py` | 多物理机 KVM YAML 解析器。 | +| `ovn/installKubeOvnFabric.py` | 安装 Kube-OVN non-primary CNI，并把 Kube-OVN/OVS 镜像导入各节点 containerd。 | +| `ovn/validateKubeOvnFabric.py` | 验证 Kube-OVN CRD、controller、CNI DaemonSet、OVS/OVN DaemonSet 是否就绪。 | +| `ovn/cleanKubeOvnFabric.py` | 在 K8s API 可用时卸载 Kube-OVN，并清理节点上的 OVN/OVS runtime 文件。 | +| `vxlan/configureLinuxVxlanFabric.py` | 为物理机 macvlan 模式创建 `br-seedemu` 和 VXLAN tunnel。 | +| `vxlan/validateLinuxVxlanFabric.py` | 验证 Linux VXLAN bridge 与 macvlan 双向连通；失败时自动调用清理入口。 | +| `vxlan/cleanLinuxVxlanFabric.py` | 删除 Linux VXLAN bridge、VXLAN 和测试 macvlan 接口。 | + +## build 阶段调用关系 + +`K8sTools.build --input configKvmOvn.yaml` 的核心顺序： + +```text +kvm/prepareHostAssets.py +kvm/createKvmVms.py +kvm/tuneVmLimits.py +applyK3sCluster.py +ovn/installKubeOvnFabric.py +``` + +`K8sTools.build --input configK3sOvn.yaml` 的核心顺序： + +```text +preparePhysicalNodes.py +applyK3sCluster.py +ovn/installKubeOvnFabric.py +``` + +`K8sTools.build --input configMultiHostKvmOvn.yaml` 的核心顺序： + +```text +multiHostKvm/prepareKvmHypervisors.py +multiHostKvm/createMultiHostKvmVms.py +kvm/tuneVmLimits.py +applyK3sCluster.py +ovn/installKubeOvnFabric.py +``` + +## 关键输入和输出 + +| 文件 | 说明 | +| --- | --- | +| 用户输入 YAML | `kind: kvmOvn`、`kind: physicalOvn` 或 `kind: multiHostKvmOvn`。 | +| `configK3s.yaml` | `build` 的持久输出，也是 `up` 和 `destroy` 的配置来源。 | +| `kubeconfig.yaml` | `build` 的持久输出，供 `kubectl` 和 `up/down` 使用。 | +| `kvmState.yaml` | 临时 KVM 清理状态，会嵌入最终 `configK3s.yaml` 的 `k8sTools.destroy` 元数据中。 | +| `multiHostKvmState.yaml` | 临时多物理机 KVM 清理状态，会嵌入最终 `configK3s.yaml`。 | +| `inventory.yaml` | 可选持久集群 inventory；通过 `k8sTools.py build --inventory inventory.yaml` 写出，包含节点角色、管理 IP 和 CPU/memory/disk 容量。 | + +## 手动调试提示 + +如果需要调试内部资源，优先在 `k8sTools.py build --keep-temp` 失败后进入保留的 +临时目录，再执行对应 Python 入口，例如： + +```bash +python3 ./kvm/createKvmVms.py ./kvm.yaml +python3 ./applyK3sCluster.py ./configK3s.yaml +python3 ./ovn/validateKubeOvnFabric.py ./configK3s.yaml +``` + +正常用户流程不需要执行这些内部文件，直接使用例子目录中的 `k8sTools.py build/up/down/destroy`。 diff --git a/seedemu/k8sTools/resources/setup/_embeddedShell.py b/seedemu/k8sTools/resources/setup/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/setup/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml new file mode 100644 index 000000000..16ffa5f2e --- /dev/null +++ b/seedemu/k8sTools/resources/setup/ansible/k3s-install.yml @@ -0,0 +1,663 @@ +--- +# K3s Installation Playbook for SEED Emulator +# Installs K3s with Multus + Macvlan CNI for L2 connectivity + +- name: Install K3s Master + hosts: master + become: yes + tasks: + - name: Optionally uninstall existing K3s server + shell: | + if [ -x /usr/local/bin/k3s-uninstall.sh ]; then + /usr/local/bin/k3s-uninstall.sh + fi + args: + executable: /bin/bash + when: seed_k3s_force_reinstall | bool + + - name: Check if K3s server already installed + stat: + path: /etc/rancher/k3s/k3s.yaml + register: k3s_server_yaml + + - name: Ensure k3s config directory exists + file: + path: /etc/rancher/k3s + state: directory + mode: "0755" + + - name: Render K3s server config + copy: + dest: /etc/rancher/k3s/config.yaml + mode: "0644" + content: | + write-kubeconfig-mode: "644" + flannel-backend: "{{ seed_k3s_flannel_backend }}" + disable: + - traefik + cluster-cidr: "{{ seed_k3s_cluster_cidr }}" + service-cidr: "{{ seed_k3s_service_cidr }}" + node-name: "{{ inventory_hostname }}" + kubelet-arg: + - "max-pods={{ seed_k3s_max_pods }}" + kube-controller-manager-arg: + - "node-cidr-mask-size-ipv4={{ seed_k3s_node_cidr_mask_size_ipv4 }}" + register: k3s_server_config + + - name: Configure registry mirror (insecure HTTP) + copy: + dest: /etc/rancher/k3s/registries.yaml + mode: "0644" + content: | + mirrors: + "{{ seed_registry_host }}:{{ seed_registry_port }}": + endpoint: + - "http://{{ seed_registry_host }}:{{ seed_registry_port }}" + "docker.io": + endpoint: + - "{{ seed_docker_io_mirror_endpoint }}" + "registry-1.docker.io": + endpoint: + - "{{ seed_docker_io_mirror_endpoint }}" + register: k3s_registries + + - name: Restart K3s server if registry config changed + service: + name: k3s + state: restarted + when: k3s_server_yaml.stat.exists and k3s_registries.changed + + - name: Restart K3s server if server config changed + service: + name: k3s + state: restarted + when: k3s_server_yaml.stat.exists and k3s_server_config.changed + + - name: Check Docker command on master + shell: command -v docker + register: docker_command + failed_when: false + changed_when: false + + - name: Install Docker (used to host local registry container) + shell: | + set -euo pipefail + waitAptLocks() { + for _ in $(seq 1 180); do + if ! fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "apt/dpkg locks did not clear before Docker install" >&2 + return 1 + } + waitAptLocks + apt-get update -o Acquire::Retries=3 + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + -o DPkg::Lock::Timeout=300 \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confold \ + --no-install-recommends docker.io + args: + executable: /bin/bash + when: docker_command.rc != 0 + + - name: Configure Docker daemon for local insecure registry + copy: + dest: /etc/docker/daemon.json + mode: "0644" + content: | + { + "insecure-registries": ["{{ seed_registry_host }}:{{ seed_registry_port }}"] + } + register: docker_daemon_config + + - name: Ensure Docker service is running + service: + name: docker + state: started + enabled: yes + + - name: Restart Docker if daemon config changed + service: + name: docker + state: restarted + when: docker_daemon_config.changed + + - name: Install K3s server + shell: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_install_version }} \ + INSTALL_K3S_ARTIFACT_URL={{ seed_k3s_artifact_url }} \ + sh - + args: + creates: /etc/rancher/k3s/k3s.yaml + + - name: Wait for K3s to be ready + shell: k3s kubectl get nodes + register: nodes + until: nodes.rc == 0 + retries: 30 + delay: 5 + + - name: Fetch K3s binary for offline worker installs + fetch: + src: /usr/local/bin/k3s + dest: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + flat: yes + + - name: Install CNI plugins required by Multus (macvlan/ipvlan/static) + shell: | + set -euo pipefail + wait_apt_locks() { + for _ in $(seq 1 150); do + if ! fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "apt/dpkg locks did not clear in time" >&2 + return 1 + } + if dpkg -s containernetworking-plugins >/dev/null 2>&1; then + exit 0 + fi + wait_apt_locks + apt-get update -o Acquire::Retries=3 || true + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends containernetworking-plugins + args: + executable: /bin/bash + + - name: Ensure /opt/cni/bin exists + file: + path: /opt/cni/bin + state: directory + mode: "0755" + + - name: Link required CNI plugin binaries into /opt/cni/bin + file: + src: "/usr/lib/cni/{{ item }}" + dest: "/opt/cni/bin/{{ item }}" + state: link + force: yes + loop: + - macvlan + - ipvlan + - static + + - name: Ensure Multus kubeconfig compatibility path exists on K3s master + shell: | + set -euo pipefail + sudo -n mkdir -p /etc/cni/net.d + sudo -n rm -rf /etc/cni/net.d/multus.d + sudo -n ln -s /var/lib/rancher/k3s/agent/etc/cni/net.d/multus.d /etc/cni/net.d/multus.d + args: + executable: /bin/bash + + - name: Get K3s token + shell: cat /var/lib/rancher/k3s/server/node-token + register: k3s_token + + - name: Store token + set_fact: + k3s_server_token: "{{ k3s_token.stdout }}" + + - name: Write bundled Multus CNI manifest + copy: + dest: /tmp/seedemu-multus-daemonset.yml + mode: "0644" + content: | + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: network-attachment-definitions.k8s.cni.cncf.io + spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - nad + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + properties: + config: + type: string + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: multus + rules: + - apiGroups: + - k8s.cni.cncf.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: multus + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus + subjects: + - kind: ServiceAccount + name: multus + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: multus + namespace: kube-system + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: multus-cni-config + namespace: kube-system + labels: + tier: node + app: multus + data: + cni-conf.json: | + { + "name": "multus-cni-network", + "type": "multus", + "capabilities": { + "portMappings": true + }, + "delegates": [ + { + "cniVersion": "0.3.1", + "name": "default-cni-network", + "plugins": [ + { + "type": "flannel", + "name": "flannel.1", + "delegate": { + "isDefaultGateway": true, + "hairpinMode": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + ], + "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" + } + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: kube-multus-ds + namespace: kube-system + labels: + tier: node + app: multus + name: multus + spec: + selector: + matchLabels: + name: multus + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: multus + name: multus + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + serviceAccountName: multus + containers: + - name: kube-multus + image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot + command: + - /thin_entrypoint + args: + - --multus-conf-file=/tmp/multus-conf/00-multus.conf + - --multus-autoconfig-dir=/host/etc/cni/net.d + - --cni-conf-dir=/host/etc/cni/net.d + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 100m + memory: 50Mi + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + - name: cnibin + mountPath: /host/opt/cni/bin + - name: multus-cfg + mountPath: /tmp/multus-conf + initContainers: + - name: install-multus-binary + image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot + command: + - /install_multus + args: + - --type + - thin + resources: + requests: + cpu: 10m + memory: 15Mi + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + mountPropagation: Bidirectional + terminationGracePeriodSeconds: 10 + volumes: + - name: cni + hostPath: + path: /etc/cni/net.d + - name: cnibin + hostPath: + path: /opt/cni/bin + - name: multus-cfg + configMap: + name: multus-cni-config + items: + - key: cni-conf.json + path: 00-multus.conf + + - name: Install Multus CNI + shell: k3s kubectl apply -f /tmp/seedemu-multus-daemonset.yml + + - name: Patch Multus ClusterRole for pod list/watch (required by k3s Multus pod-wait logic) + shell: | + cat </dev/null 2>&1; then + break + fi + sleep 2 + done + conf_dir="/var/lib/rancher/k3s/agent/etc/cni/net.d" + if [ ! -d "${conf_dir}" ]; then + conf_dir="$(awk -F'"' '/conf_dir[[:space:]]*=/{print $2; exit}' /var/lib/rancher/k3s/agent/etc/containerd/config.toml 2>/dev/null || true)" + fi + bin_dir="$(awk -F'"' '/bin_dir[[:space:]]*=/{print $2; exit}' /var/lib/rancher/k3s/agent/etc/containerd/config.toml 2>/dev/null || true)" + if [ -z "${bin_dir}" ]; then + if [ -d /var/lib/rancher/k3s/data/cni ]; then + bin_dir="/var/lib/rancher/k3s/data/cni" + elif [ -d /var/lib/rancher/k3s/data/current/bin ]; then + bin_dir="/var/lib/rancher/k3s/data/current/bin" + fi + fi + if [ -z "${conf_dir}" ] || [ -z "${bin_dir}" ]; then + echo "Failed to detect k3s conf_dir/bin_dir" >&2 + exit 1 + fi + k3s kubectl -n kube-system patch ds kube-multus-ds --type='strategic' -p "{\"spec\":{\"template\":{\"spec\":{\"volumes\":[{\"name\":\"cni\",\"hostPath\":{\"path\":\"${conf_dir}\"}},{\"name\":\"cnibin\",\"hostPath\":{\"path\":\"${bin_dir}\"}}]}}}}" + args: + executable: /bin/bash + + - name: Label master node + shell: k3s kubectl label node {{ inventory_hostname }} seedemu.io/as-group={{ seedemu_as_group }} --overwrite + +- name: Install K3s Workers + hosts: workers + become: yes + tasks: + - name: Optionally uninstall existing K3s agent + shell: | + if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then + /usr/local/bin/k3s-agent-uninstall.sh + fi + args: + executable: /bin/bash + when: seed_k3s_force_reinstall | bool + + - name: Check if K3s agent already installed + stat: + path: /var/lib/rancher/k3s/agent + register: k3s_agent_dir + + - name: Ensure k3s config directory exists + file: + path: /etc/rancher/k3s + state: directory + mode: "0755" + + - name: Render K3s agent config + copy: + dest: /etc/rancher/k3s/config.yaml + mode: "0644" + content: | + node-name: "{{ inventory_hostname }}" + kubelet-arg: + - "max-pods={{ seed_k3s_max_pods }}" + register: k3s_agent_config + + - name: Configure registry mirror (insecure HTTP) + copy: + dest: /etc/rancher/k3s/registries.yaml + mode: "0644" + content: | + mirrors: + "{{ seed_registry_host }}:{{ seed_registry_port }}": + endpoint: + - "http://{{ seed_registry_host }}:{{ seed_registry_port }}" + "docker.io": + endpoint: + - "{{ seed_docker_io_mirror_endpoint }}" + "registry-1.docker.io": + endpoint: + - "{{ seed_docker_io_mirror_endpoint }}" + register: k3s_registries + + - name: Restart K3s agent if registry config changed + service: + name: k3s-agent + state: restarted + when: k3s_agent_dir.stat.exists and k3s_registries.changed + + - name: Restart K3s agent if agent config changed + service: + name: k3s-agent + state: restarted + when: k3s_agent_dir.stat.exists and k3s_agent_config.changed + + - name: Copy K3s binary prepared on master to worker + copy: + src: "/tmp/seedemu-k3s-binary-{{ k3s_install_version }}/k3s" + dest: /usr/local/bin/k3s + mode: "0755" + when: not k3s_agent_dir.stat.exists or seed_k3s_force_reinstall | bool + + - name: Install K3s agent + shell: | + curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_DOWNLOAD=true \ + INSTALL_K3S_VERSION={{ k3s_install_version }} \ + INSTALL_K3S_ARTIFACT_URL={{ seed_k3s_artifact_url }} \ + K3S_URL=https://{{ hostvars[groups['master'][0]]['ansible_host'] }}:6443 \ + K3S_TOKEN={{ hostvars[groups['master'][0]]['k3s_server_token'] }} \ + sh - + args: + creates: /var/lib/rancher/k3s/agent + + - name: Install CNI plugins required by Multus (macvlan/ipvlan/static) + shell: | + set -euo pipefail + wait_apt_locks() { + for _ in $(seq 1 150); do + if ! fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "apt/dpkg locks did not clear in time" >&2 + return 1 + } + if dpkg -s containernetworking-plugins >/dev/null 2>&1; then + exit 0 + fi + wait_apt_locks + apt-get update -o Acquire::Retries=3 || true + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends containernetworking-plugins + args: + executable: /bin/bash + + - name: Ensure /opt/cni/bin exists + file: + path: /opt/cni/bin + state: directory + mode: "0755" + + - name: Link required CNI plugin binaries into /opt/cni/bin + file: + src: "/usr/lib/cni/{{ item }}" + dest: "/opt/cni/bin/{{ item }}" + state: link + force: yes + loop: + - macvlan + - ipvlan + - static + + - name: Ensure Multus kubeconfig compatibility path exists on K3s workers + shell: | + set -euo pipefail + sudo -n mkdir -p /etc/cni/net.d + sudo -n rm -rf /etc/cni/net.d/multus.d + sudo -n ln -s /var/lib/rancher/k3s/agent/etc/cni/net.d/multus.d /etc/cni/net.d/multus.d + args: + executable: /bin/bash + +- name: Configure Worker Labels + hosts: master + become: yes + tasks: + - name: Wait for workers to join + shell: k3s kubectl get nodes | grep -c Ready + register: ready_nodes + until: ready_nodes.stdout | int >= seed_k3s_expected_ready_nodes + retries: 30 + delay: 10 + + - name: Label worker nodes + shell: k3s kubectl label node {{ item }} seedemu.io/as-group={{ hostvars[item]['seedemu_as_group'] }} --overwrite + loop: "{{ groups['workers'] }}" + +- name: Setup Macvlan CNI + hosts: master + become: yes + tasks: + - name: Create Macvlan NetworkAttachmentDefinition + shell: | + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveInput() { + if [ "${CONFIG_PATH}" = "-h" ] || [ "${CONFIG_PATH}" = "--help" ]; then + usage + exit 0 + fi + if [ ! -s "${CONFIG_PATH}" ]; then + echo "configK3s.yaml not found or empty: ${CONFIG_PATH}" >&2 + exit 1 + fi +} + +helper() { + local command="$1" + shift + python3 "${HELPER}" --config "${CONFIG_PATH}" "${command}" "$@" +} + +loadConfigVars() { + local vars_output + if ! vars_output="$(helper shell-vars)"; then + return 1 + fi + eval "${vars_output}" + MASTER_IS_LOCAL=false + if [ "${k3sMasterConnection:-ssh}" = "local" ]; then + MASTER_IS_LOCAL=true + fi + SSH_OPTS=( + -i "${k3sSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +loadNodeSshContext() { + # Args: + # $1: node name from configK3s.yaml. + # Reads per-node ssh.user/key and prepares NODE_SSH_OPTS for node-specific + # SSH/scp operations. Master-only operations keep using SSH_OPTS. + local name="$1" + eval "$(helper node-ssh-vars --name "${name}")" + if [ "${nodeConnection:-ssh}" = "local" ]; then + NODE_SSH_OPTS=() + return 0 + fi + [ -f "${nodeSshKey}" ] || { + echo "SSH key not found for ${name}: ${nodeSshKey}" >&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +resolveSeedEmulatorDockerDir() { + # Resolve the host path containing seedemu-base and seedemu-router + # Dockerfiles. Existing YAML value wins; common source-tree locations are + # tried as fallbacks for physical-server workflows. + local cursor="${SCRIPT_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch" \ + "${REPO_ROOT}/../seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +runMasterCommand() { + # Args: + # $1: shell command to run on the K3s master. + # Uses local execution when configK3s.yaml points the master at this host; + # otherwise uses the configured master SSH account. + local command="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + bash -lc "${command}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "${command}" + fi +} + +copyFileToMaster() { + # Args: + # $1: local source path. + # $2: destination path on the master. + local source="$1" + local target="$2" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + cp "${source}" "${target}" + else + scp "${SSH_OPTS[@]}" "${source}" "${k3sUser}@${k3sMasterIp}:${target}" >/dev/null + fi +} + +readMasterFile() { + # Args: + # $1: absolute file path to read from the master with sudo. + local path="$1" + if [ "${MASTER_IS_LOCAL}" = "true" ]; then + sudo -n cat "${path}" + else + ssh "${SSH_OPTS[@]}" "${k3sUser}@${k3sMasterIp}" "sudo -n cat '${path}'" + fi +} + +runNodeCommand() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: shell command to run on that node. + local name="$1" + local ip="$2" + local command="$3" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + bash -lc "${command}" + else + ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "${command}" + fi +} + +copyFileToNode() { + # Args: + # $1: node name from configK3s.yaml. + # $2: node management IP. + # $3: local source path. + # $4: destination path on the node. + local name="$1" + local ip="$2" + local source="$3" + local target="$4" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + cp "${source}" "${target}" + else + scp "${NODE_SSH_OPTS[@]}" "${source}" "${nodeSshUser}@${ip}:${target}" >/dev/null + fi +} + +printPlan() { + echo "config=${CONFIG_PATH}" + echo "K3s node plan:" + helper nodes-tsv | awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' + echo "master=${k3sMasterName} (${k3sMasterIp})" + echo "master_connection=${k3sMasterConnection:-ssh}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + echo "kubeconfig=${outputKubeconfig}" +} + +verifyConnectivity() { + echo "[1/10] Verifying SSH and sudo on all nodes" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + echo " ${name} ${ip}" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + runWithTimeout 12s bash -lc "echo local-ok >/dev/null" + runWithTimeout 12s sudo -n true >/dev/null + else + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ssh-ok" >/dev/null + runWithTimeout 12s ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n true" >/dev/null + fi + done < <(helper nodes-tsv) +} + +runAnsibleInstall() { + echo "[2/10] Installing K3s via generated Ansible inventory" + local inventory_tmp playbook_tmp node_count + mkdir -p "${setupTmpDir}" + inventory_tmp="$(mktemp "${setupTmpDir}/ansible-inventory.XXXXXX.yml")" + playbook_tmp="$(mktemp "${setupTmpDir}/k3s-install.XXXXXX.yml")" + helper write-ansible-inventory --output "${inventory_tmp}" >/dev/null + node_count="$(helper nodes-tsv | wc -l | tr -d ' ')" + sed "s/ready_nodes.stdout | int >= 3/ready_nodes.stdout | int >= ${node_count}/" \ + "${PLAYBOOK_PATH}" > "${playbook_tmp}" + ANSIBLE_HOST_KEY_CHECKING=False \ + runWithTimeout "${ansibleTimeout}" \ + ansible-playbook \ + -i "${inventory_tmp}" \ + "${playbook_tmp}" \ + --ssh-common-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none" + rm -f "${inventory_tmp}" "${playbook_tmp}" +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + local mirror_ref="${HOST_DOCKER_IO_MIRROR}" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${mirror_ref}" "${image}" + else + printf '%s/library/%s\n' "${mirror_ref}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " host docker pull ${image}" >&2 + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " host docker pull ${mirror_image}" >&2 + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare image on host: ${image}" >&2 + echo "This script intentionally does not ask the VM to pull public images." >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + ensureHostDockerImage "${image}" + if [ -s "${tar_path}" ]; then + printf '%s\n' "${tar_path}" + return 0 + fi + echo " host docker save ${image} -> ${tar_path}" >&2 + docker save -o "${tar_path}" "${image}" + printf '%s\n' "${tar_path}" +} + +loadDockerImageToMaster() { + local image="$1" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + remote_tar="/tmp/$(basename "${tar_path}")" + echo " copy ${image} to ${k3sMasterName}:${remote_tar}" + copyFileToMaster "${tar_path}" "${remote_tar}" + runMasterCommand "sudo -n docker load -i '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +importK3sImageToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path remote_tar + tar_path="$(saveHostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${node_name}" "${node_ip}" "${tar_path}" +} + +importK3sImageTarToNode() { + local image="$1" + local node_name="$2" + local node_ip="$3" + local tar_path="$4" + local remote_tar + remote_tar="/tmp/$(basename "${tar_path}")" + echo " import ${image} to ${node_name}" + copyFileToNode "${node_name}" "${node_ip}" "${tar_path}" "${remote_tar}" + runNodeCommand "${node_name}" "${node_ip}" \ + "sudo -n k3s ctr -n k8s.io images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" +} + +ensureRegistry() { + echo "[3/10] Ensuring private registry on master" + echo " preparing ${REGISTRY_BOOTSTRAP_IMAGE} on host and loading it into master Docker" + loadDockerImageToMaster "${REGISTRY_BOOTSTRAP_IMAGE}" + runMasterCommand " + set -euo pipefail + if ! docker buildx version >/dev/null 2>&1; then + sudo -n apt-get update >/dev/null + sudo -n apt-get install -y docker-buildx >/dev/null + fi + sudo -n docker rm -f registry >/dev/null 2>&1 || true + sudo -n docker image inspect '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + sudo -n docker run -d --network host --restart=always --name registry \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:${registryPort} '${REGISTRY_BOOTSTRAP_IMAGE}' >/dev/null + " +} + +ensureSeedemuHostBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedBaseSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + echo " host docker build ${seedRouterSourceImage}" >&2 + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareMasterWorkloadBuildImages() { + echo "[4/10] Preparing workload build base images on master Docker" + [ -d "${seedEmulatorDockerDir}/seedemu-base" ] || { + echo "Missing host seedemu base image directory: ${seedEmulatorDockerDir}/seedemu-base" >&2 + exit 1 + } + [ -d "${seedEmulatorDockerDir}/seedemu-router" ] || { + echo "Missing host seedemu router image directory: ${seedEmulatorDockerDir}/seedemu-router" >&2 + exit 1 + } + + ensureSeedemuHostBuildImages + loadDockerImageToMaster "${ubuntuBuildImage}" + loadDockerImageToMaster "${seedBaseSourceImage}" + loadDockerImageToMaster "${seedRouterSourceImage}" + + echo " ensuring stable compiler hash tags on master Docker" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' '${seedBaseHashImage}' + sudo -n docker tag '${seedRouterSourceImage}' '${seedRouterHashImage}' + " + + echo " pushing seedemu base/router images into master local registry" + runMasterCommand " + set -euo pipefail + sudo -n docker tag '${seedBaseSourceImage}' \ + '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedBaseSourceImage}' + sudo -n docker tag '${seedRouterSourceImage}' \ + '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + sudo -n docker push '127.0.0.1:${registryPort}/${seedRouterSourceImage}' + " +} + +fetchKubeconfig() { + echo "[5/10] Fetching kubeconfig" + mkdir -p "$(dirname "${outputKubeconfig}")" + readMasterFile "/etc/rancher/k3s/k3s.yaml" > "${outputKubeconfig}" + sed -i "s|127.0.0.1|${k3sMasterIp}|g" "${outputKubeconfig}" + echo "kubeconfig=${outputKubeconfig}" +} + +preloadK3sBootstrapImagesAllNodes() { + echo "[6/10] Preloading K3s system and Multus images from host into all K3s containerd nodes" + local preload_node_concurrency="${SEED_K8S_PRELOAD_NODE_CONCURRENCY:-3}" + local preload_failures=0 + if ! [[ "${preload_node_concurrency}" =~ ^[0-9]+$ ]] || [ "${preload_node_concurrency}" -lt 1 ]; then + echo "Invalid SEED_K8S_PRELOAD_NODE_CONCURRENCY: ${preload_node_concurrency}" >&2 + exit 1 + fi + echo " preload node concurrency=${preload_node_concurrency}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + saveHostImageTarball "${image}" >/dev/null + done + activePreloadJobs() { + jobs -rp | wc -l | tr -d ' ' + } + preloadNodeImages() { + local name="$1" + local ip="$2" + local image tar_path + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}" "${MULTUS_BOOTSTRAP_IMAGE}"; do + tar_path="$(hostImageTarball "${image}")" + importK3sImageTarToNode "${image}" "${name}" "${ip}" "${tar_path}" + done + } + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + preloadNodeImages "${name}" "${ip}" & + while [ "$(activePreloadJobs)" -ge "${preload_node_concurrency}" ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + done < <(helper nodes-tsv) + while [ "$(activePreloadJobs)" -gt 0 ]; do + if ! wait -n; then + preload_failures=$((preload_failures + 1)) + fi + done + if [ "${preload_failures}" -ne 0 ]; then + echo "K3s bootstrap image preload failed on ${preload_failures} node(s)" >&2 + exit 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=kube-dns \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l k8s-app=metrics-server \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l app=local-path-provisioner \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system delete pod -l name=multus \ + --force --grace-period=0 --wait=false >/dev/null 2>&1 || true +} + +installOvnFabricIfConfigured() { + # Install Kube-OVN after K3s and Multus are available. OVN is a Kubernetes + # CNI add-on, so it cannot be prepared in the physical-node preflight stage. + if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + return 0 + fi + echo "[7/10] Installing Kube-OVN non-primary CNI" + python3 "${SCRIPT_DIR}/ovn/installKubeOvnFabric.py" "${CONFIG_PATH}" +} + +applyNodeK3sTuning() { + local name="$1" + local ip="$2" + local remote_runner=() + echo " tuning K3s on ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${k3sMaxPods}" \ + "${kubeletRegistryQps}" \ + "${kubeletRegistryBurst}" \ + "${rebootAfterTuning}" <<'EOF_REMOTE' +set -euo pipefail +MAX_PODS="$1" +REGISTRY_QPS="$2" +REGISTRY_BURST="$3" +ASYNC_RESTART="$4" + +if [ ! -f /etc/sysctl.d/99-seed-vm-limits.conf ]; then + echo "warning: /etc/sysctl.d/99-seed-vm-limits.conf not found; run tuneVmLimits.py before large-scale experiments" >&2 +fi + +mkdir -p /etc/rancher/k3s +touch /etc/rancher/k3s/config.yaml +cfg=/etc/rancher/k3s/config.yaml +tmp=$(mktemp) +awk ' + /^[^[:space:]].*:/ { + if ($0 ~ /^(kubelet-arg|kube-apiserver-arg):/) {skip=1; next} + skip=0 + } + skip == 0 {print} +' "${cfg}" > "${tmp}" +cat >> "${tmp}" <> "${tmp}" <<'EOF_MASTER' +kube-apiserver-arg: + - "max-requests-inflight=1000" + - "max-mutating-requests-inflight=500" +EOF_MASTER +fi +cat "${tmp}" > "${cfg}" +rm -f "${tmp}" + +if [ "${ASYNC_RESTART}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true; systemctl restart containerd 2>/dev/null || true" >/dev/null 2>&1 || true +else + systemctl restart k3s 2>/dev/null || systemctl restart k3s-agent 2>/dev/null || true + systemctl restart containerd 2>/dev/null || true +fi +EOF_REMOTE +} + +applyTuningAllNodes() { + echo "[8/10] Applying K3s runtime tuning" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + applyNodeK3sTuning "${name}" "${ip}" + done < <(helper nodes-tsv) +} + +verifyCluster() { + echo "[9/10] Waiting for K3s nodes" + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Ready node --all --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" -n kube-system rollout status daemonset/kube-multus-ds --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o wide + kubectl --kubeconfig "${outputKubeconfig}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tPodCIDR: "}{.spec.podCIDR}{"\n"}{end}' + echo +} + +writeOutputs() { + echo "[10/10] Writing inventory output" + helper write-cluster-inventory >/dev/null + echo "inventory=${outputInventory}" + echo "kubeconfig=${outputKubeconfig}" + echo "registry=${registryHost}:${registryPort}" +} + +main() { + requireCommand python3 + requireCommand ansible-playbook + requireCommand docker + requireCommand scp + requireCommand ssh + requireCommand kubectl + requireCommand sed + [ -f "${PLAYBOOK_PATH}" ] || { + echo "K3s Ansible playbook not found: ${PLAYBOOK_PATH}" >&2 + exit 1 + } + + resolveInput + loadConfigVars + resolveSeedEmulatorDockerDir + + if [ "${MASTER_IS_LOCAL}" != "true" ] && [ ! -f "${k3sSshKey}" ]; then + echo "SSH key not found: ${k3sSshKey}" >&2 + exit 1 + fi + + printPlan + verifyConnectivity + runAnsibleInstall + ensureRegistry + prepareMasterWorkloadBuildImages + fetchKubeconfig + preloadK3sBootstrapImagesAllNodes + installOvnFabricIfConfigured + applyTuningAllNodes + verifyCluster + writeOutputs + echo "K3s cluster is ready." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py new file mode 100755 index 000000000..c695f6d97 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/destroyPhysicalCluster.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Python entrypoint for destroyPhysicalCluster with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove a physical SeedEMU K3s cluster and its optional fabric backend. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Runs K3s uninstall scripts on configured nodes, removes the local registry +# container on the master, deletes generated kubeconfig/inventory files, and +# removes the configured linux-vxlan or Kube-OVN fabric if present. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" shell-vars)" + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[cluster-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s <<<"$script" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"$script" +} + +uninstall_node=' +set -euo pipefail +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then + /usr/local/bin/k3s-agent-uninstall.sh || true +fi +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then + /usr/local/bin/k3s-uninstall.sh || true +fi +' + +remove_registry=' +set -euo pipefail +if command -v docker >/dev/null 2>&1; then + docker rm -f registry >/dev/null 2>&1 || true +fi +' + +echo "Cleaning physical K3s cluster from: $CONFIG_PATH" + +destroy_node_concurrency="${SEED_K8S_DESTROY_NODE_CONCURRENCY:-4}" +if ! [[ "$destroy_node_concurrency" =~ ^[0-9]+$ ]] || [ "$destroy_node_concurrency" -lt 1 ]; then + echo "Invalid SEED_K8S_DESTROY_NODE_CONCURRENCY: $destroy_node_concurrency" >&2 + exit 1 +fi + +activeNodeCleanJobs() { + jobs -rp | wc -l | tr -d ' ' +} + +if [ "${fabricType}" = "ovn" ] || [ "${fabricType}" = "kube-ovn" ]; then + if [ -x "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" ]; then + python3 "${SCRIPT_DIR}/ovn/cleanKubeOvnFabric.py" "$CONFIG_PATH" || true + else + echo "warning: OVN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +node_clean_failures=0 +echo "K3s node uninstall concurrency: ${destroy_node_concurrency}" +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "$HELPER" --config "$CONFIG_PATH" node-ssh-vars --name "$name")" + if [ "$role" = "master" ]; then + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$remove_registry" + fi + runNodeRootScript "$name" "$ip" "$nodeConnection" "$nodeSshUser" "$nodeSshKey" "$uninstall_node" & + while [ "$(activeNodeCleanJobs)" -ge "$destroy_node_concurrency" ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi + done +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +while [ "$(activeNodeCleanJobs)" -gt 0 ]; do + if ! wait -n; then + node_clean_failures=$((node_clean_failures + 1)) + fi +done + +if [ "$node_clean_failures" -ne 0 ]; then + echo "K3s node uninstall failed on ${node_clean_failures} node(s)" >&2 + exit 1 +fi + +if [ "${fabricType}" = "linux-vxlan" ]; then + if [ -x "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" ]; then + python3 "${SCRIPT_DIR}/vxlan/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + else + echo "warning: VXLAN cleanup script is not present in this generated setup directory" >&2 + fi +fi + +rm -f "$outputKubeconfig" "$outputInventory" +echo "Removed generated kubeconfig/inventory:" +echo " $outputKubeconfig" +echo " $outputInventory" +echo "Physical K3s cluster cleanup finished." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py new file mode 100755 index 000000000..fc6f27a93 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/createKvmVms.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""Python entrypoint for createKvmVms with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create or start KVM VMs from kvm.yaml. The script resolves non-conflicting +# VM names/IPs/MACs, writes user-facing configK3s.yaml plus internal +# kvmState.yaml, and waits for SSH. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +kvmBaseImageReuseMode="copy" +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 + +eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + +SSH_PUB_KEY="" +EXISTING_VMS_TSV="" +PLANNED_NODES_TSV="" +REUSING_KVM_STATE_PLAN="false" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${EXISTING_VMS_TSV}" ] && rm -f "${EXISTING_VMS_TSV}" || true + [ -n "${PLANNED_NODES_TSV}" ] && rm -f "${PLANNED_NODES_TSV}" || true +} + +domainExists() { + virsh dominfo "$1" >/dev/null 2>&1 +} + +domainRunning() { + virsh domstate "$1" 2>/dev/null | grep -qi "running" +} + +prepareDirs() { + mkdir -p "${kvmStorageDir}/base" + mkdir -p "${kvmDiskDir}" + mkdir -p "${kvmCloudInitDir}" + mkdir -p "$(dirname "${kvmBaseImagePath}")" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +ensureNetwork() { + if ! virsh net-info "${kvmNetwork}" >/dev/null 2>&1; then + echo "Libvirt network not found: ${kvmNetwork}" >&2 + exit 1 + fi + virsh net-start "${kvmNetwork}" >/dev/null 2>&1 || true + virsh net-autostart "${kvmNetwork}" >/dev/null 2>&1 || true +} + +collectExistingVms() { + EXISTING_VMS_TSV="$(mktemp "${kvmStorageDir}/existing-vms.XXXXXX.tsv")" + PLANNED_NODES_TSV="$(mktemp "${kvmStorageDir}/planned-vms.XXXXXX.tsv")" + + if [ -f "${outputKvmState}" ]; then + if ! python3 "${HELPER}" "${CONFIG_PATH}" validate-kvm-state --state "${outputKvmState}"; then + echo "Refusing to reuse stale KVM state: ${outputKvmState}" >&2 + echo "Remove it only if you intentionally want this setup directory to generate a new VM plan." >&2 + exit 1 + fi + REUSING_KVM_STATE_PLAN="true" + python3 "${HELPER}" "${CONFIG_PATH}" state-nodes-tsv --state "${outputKvmState}" > "${PLANNED_NODES_TSV}" + echo "Using existing KVM state: ${outputKvmState}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" + return + fi + + virsh list --all --name 2>/dev/null | awk 'NF {print $1 "\t\t"}' >> "${EXISTING_VMS_TSV}" + + virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + print("\t".join([ + host.get("name") or "", + host.get("ip") or "", + (host.get("mac") or "").lower(), + ])) +' >> "${EXISTING_VMS_TSV}" + + virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk ' + NR <= 2 {next} + NF >= 5 { + name=$6 + if (name == "-" || name == "") name="" + ip=$5 + sub("/.*", "", ip) + print name "\t" ip "\t" tolower($3) + } + ' >> "${EXISTING_VMS_TSV}" || true + + python3 "${HELPER}" "${CONFIG_PATH}" nodes-tsv --existing-tsv "${EXISTING_VMS_TSV}" > "${PLANNED_NODES_TSV}" + echo "Planned KVM nodes:" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s vcpus=%s memory_mb=%s disk_gb=%s\n", $1, $2, $3, $4, $5, $6, $7}' "${PLANNED_NODES_TSV}" +} + +downloadBaseImage() { + if [ -f "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Using existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Using existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + echo "Downloading Ubuntu cloud image to ${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +loadSshPubkey() { + if [ -f "${sshKey}.pub" ]; then + SSH_PUB_KEY="$(tr -d '\n' < "${sshKey}.pub")" + return + fi + if [ -f "${sshKey}" ]; then + SSH_PUB_KEY="$(ssh-keygen -y -f "${sshKey}" 2>/dev/null | tr -d '\n')" + [ -n "${SSH_PUB_KEY}" ] && return + fi + echo "Cannot read SSH key. Set ssh.key in ${CONFIG_PATH} to a valid private key." >&2 + exit 1 +} + +updateDhcpHost() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + virsh net-update "${kvmNetwork}" add ip-dhcp-host "${host_xml}" --live --config >/dev/null +} + +createVmCloudInit() { + local vm_name="$1" + local vm_dir="${kvmCloudInitDir}/${vm_name}" + mkdir -p "${vm_dir}" + + cat > "${vm_dir}/user-data.yaml" < "${vm_dir}/meta-data.yaml" </dev/null +} + +createOrStartVm() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vcpus="$4" + local memory_mb="$5" + local disk_gb="$6" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + local vm_cloud_dir="${kvmCloudInitDir}/${vm_name}" + + if domainExists "${vm_name}"; then + validateExistingDomainBelongsToSetup "${vm_name}" + if [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "Refusing to reuse existing VM '${vm_name}'. Set kvm.allowExisting: true only if this is intentional." >&2 + exit 1 + fi + echo "VM exists: ${vm_name}" + if ! domainRunning "${vm_name}"; then + virsh start "${vm_name}" >/dev/null + fi + return + fi + + echo "Creating VM: ${vm_name} ip=${vm_ip} vcpus=${vcpus} memory_mb=${memory_mb} disk_gb=${disk_gb}" + virt-install \ + --name "${vm_name}" \ + --memory "${memory_mb}" \ + --vcpus "${vcpus}" \ + --cpu host-passthrough \ + --import \ + --os-variant generic \ + --network "network=${kvmNetwork},model=virtio,mac=${vm_mac}" \ + --disk "path=${vm_disk},format=qcow2,bus=virtio" \ + --graphics none \ + --noautoconsole \ + --cloud-init "user-data=${vm_cloud_dir}/user-data.yaml,meta-data=${vm_cloud_dir}/meta-data.yaml" >/dev/null +} + +validateExistingDomainBelongsToSetup() { + # Args: + # $1: VM domain name that already exists. + # Uses kvmDiskDir from kvm.yaml to make stale kvmState.yaml files fail safe. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for existing VM ${vm_name}; refusing to reuse it." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to reuse existing VM ${vm_name}: disk is outside this setup diskDir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +checkNetworkConflict() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + + virsh net-dumpxml "${kvmNetwork}" | python3 -c ' +import sys +import xml.etree.ElementTree as ET + +name, ip, mac = sys.argv[1], sys.argv[2], sys.argv[3].lower() +root = ET.fromstring(sys.stdin.read()) +for host in root.findall(".//host"): + h_name = host.get("name") or "" + h_ip = host.get("ip") or "" + h_mac = (host.get("mac") or "").lower() + same_host = h_name == name and h_ip == ip and h_mac == mac + if h_ip == ip and not same_host: + raise SystemExit(f"IP {ip} is already reserved by host name={h_name} mac={h_mac}") + if h_mac == mac and not same_host: + raise SystemExit(f"MAC {mac} is already reserved by host name={h_name} ip={h_ip}") +' "${vm_name}" "${vm_ip}" "${vm_mac}" + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null | awk -v ip="${vm_ip}" -v mac="${vm_mac,,}" -v name="${vm_name}" ' + NR <= 2 {next} + { + lease_mac=tolower($3); lease_ip=$5; sub("/.*", "", lease_ip); lease_name=$6 + if ((lease_ip == ip || lease_mac == mac) && !(lease_ip == ip && lease_mac == mac && lease_name == name)) { + print "DHCP lease conflict: name=" lease_name " mac=" lease_mac " ip=" lease_ip > "/dev/stderr" + exit 1 + } + } + '; then + return + fi + exit 1 +} + +checkCreateConflicts() { + local vm_name="$1" + local vm_ip="$2" + local vm_mac="$3" + local vm_disk="${kvmDiskDir}/${vm_name}.qcow2" + + if domainExists "${vm_name}" && [ "${kvmAllowExisting}" != "true" ] && [ "${REUSING_KVM_STATE_PLAN}" != "true" ]; then + echo "VM name conflict: ${vm_name} already exists. Choose a new name or set kvm.allowExisting: true intentionally." >&2 + exit 1 + fi + if [ -e "${vm_disk}" ] && ! domainExists "${vm_name}" ]; then + echo "Disk conflict: ${vm_disk} already exists but VM ${vm_name} is not defined." >&2 + exit 1 + fi + checkNetworkConflict "${vm_name}" "${vm_ip}" "${vm_mac}" +} + +waitForSsh() { + local vm_name="$1" + local vm_ip="$2" + local elapsed=0 + while [ "${elapsed}" -lt "${kvmBootTimeoutSeconds}" ]; do + if ssh -o StrictHostKeyChecking=no \ + -n \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -o ConnectTimeout=5 \ + -i "${sshKey}" \ + "${sshUser}@${vm_ip}" "echo ok" >/dev/null 2>&1; then + echo "SSH ready: ${vm_name} (${vm_ip})" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + echo "Timeout waiting for SSH on ${vm_name} (${vm_ip})" >&2 + return 1 +} + +writeK3sConfig() { + # Args: none. + # Writes the user-facing configK3s.yaml from the transient planned node + # list. It intentionally omits KVM resource details and output paths. + python3 "${HELPER}" "${CONFIG_PATH}" write-k3s-config \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputK3sConfig}" + echo "K3s config: ${outputK3sConfig}" +} + +writeKvmState() { + # Args: none. + # Writes internal kvmState.yaml with VM MAC/resource/disk metadata. Cleanup + # uses this file instead of overloading the user-facing configK3s.yaml. + python3 "${HELPER}" "${CONFIG_PATH}" write-kvm-state \ + --nodes-tsv "${PLANNED_NODES_TSV}" \ + --output "${outputKvmState}" + echo "KVM state: ${outputKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand virsh + requireCommand virt-install + requireCommand qemu-img + requireCommand curl + requireCommand ssh + requireCommand ssh-keygen + requireCommand awk + + [ -f "${sshKey}" ] || { + echo "SSH key not found: ${sshKey}" >&2 + exit 1 + } + trap cleanupTmp EXIT + + prepareDirs + ensureNetwork + collectExistingVms + downloadBaseImage + loadSshPubkey + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + checkCreateConflicts "${name}" "${ip}" "${mac}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + updateDhcpHost "${name}" "${ip}" "${mac}" + createVmCloudInit "${name}" + createVmDisk "${name}" "${disk_gb}" + createOrStartVm "${name}" "${ip}" "${mac}" "${vcpus}" "${memory_mb}" "${disk_gb}" + done < "${PLANNED_NODES_TSV}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < "${PLANNED_NODES_TSV}" + + if [ "${kvmSkipK3sConfig}" = "true" ]; then + echo "Skipping host-local configK3s.yaml generation because kvm.skipK3sConfig=true." + else + writeK3sConfig + fi + writeKvmState + echo "KVM VMs are ready." + echo "Next step: ${SCRIPT_DIR}/tuneVmLimits.py ${outputK3sConfig}" +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py new file mode 100755 index 000000000..45a1cb0f6 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/destroyKvmVms.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Python entrypoint for destroyKvmVms with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs recorded in kvmState.yaml. +# Inputs: kvmState.yaml generated by createKvmVms.py. +# Outputs/side effects: destroys matching libvirt domains, removes DHCP +# reservations/leases, disks, cloud-init files, generated kubeconfig/inventory, +# and then verifies the cleanup. The disk-dir guard prevents deleting unrelated +# VMs whose names happen to match. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" +NODES_TSV="" +VERIFY_NODES_TSV="" +dnsmasqStatus="/var/lib/libvirt/dnsmasq/virbr0.status" +setupOutputGlob="${SETUP_DIR}/seedemu-k3s.*" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +cleanupTmp() { + [ -n "${NODES_TSV}" ] && rm -f "${NODES_TSV}" || true + [ -n "${VERIFY_NODES_TSV}" ] && rm -f "${VERIFY_NODES_TSV}" || true +} + +loadConfig() { + if [ ! -s "${CONFIG_PATH}" ]; then + echo "KVM state not found or empty: ${CONFIG_PATH}" >&2 + echo "Run createKvmVms.py first so it generates kvmState.yaml." >&2 + exit 1 + fi + eval "$(python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-vars --state "${CONFIG_PATH}")" + kvmBridgeName="$(virsh net-dumpxml "${kvmNetwork}" 2>/dev/null | python3 -c 'import sys, xml.etree.ElementTree as ET; data=sys.stdin.read(); print((ET.fromstring(data).find(".//bridge").get("name") if data.strip() else "") or "")' 2>/dev/null || true)" + if [ -n "${kvmBridgeName}" ]; then + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmBridgeName}.status" + else + dnsmasqStatus="/var/lib/libvirt/dnsmasq/${kvmNetwork}.status" + fi + NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-nodes.XXXXXX.tsv")" + VERIFY_NODES_TSV="$(mktemp "${SCRIPT_DIR}/destroy-verify-nodes.XXXXXX.tsv")" + python3 "${HELPER}" "${SETUP_DIR}/kvm.yaml" state-nodes-tsv --state "${CONFIG_PATH}" > "${NODES_TSV}" + cp "${NODES_TSV}" "${VERIFY_NODES_TSV}" +} + +printPlan() { + echo "Cleaning KVM nodes from: ${CONFIG_PATH}" + echo "KVM network: ${kvmNetwork}" + echo "Disk dir: ${kvmDiskDir}" + echo "Cloud-init dir: ${kvmCloudInitDir}" + awk -F '\t' '{printf " %-24s role=%-6s ip=%-15s mac=%s\n", $1, $2, $3, $4}' "${NODES_TSV}" +} + +cleanupDomainsReservationsAndFiles() { + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + echo "Cleaning VM ${name} (${ip}, ${mac})" + + if virsh dominfo "${name}" >/dev/null 2>&1; then + validateDestroyTargetBelongsToSetup "${name}" + virsh destroy "${name}" >/dev/null 2>&1 || true + virsh undefine "${name}" --nvram --remove-all-storage >/dev/null 2>&1 \ + || virsh undefine "${name}" --nvram >/dev/null 2>&1 \ + || virsh undefine "${name}" >/dev/null + fi + + if [ -n "${ip}" ] && [ -n "${mac}" ]; then + local host_xml="" + virsh net-update "${kvmNetwork}" delete ip-dhcp-host "${host_xml}" --live --config >/dev/null 2>&1 || true + fi + + rm -rf "${kvmCloudInitDir}/${name}" + rm -f "${kvmDiskDir}/${name}.qcow2" + done < "${NODES_TSV}" +} + +validateDestroyTargetBelongsToSetup() { + # Args: + # $1: VM domain name to validate before destructive cleanup. + # Uses kvmDiskDir from kvmState.yaml. This prevents a stale state file from + # deleting a pre-existing cluster VM whose disk lives elsewhere. + local vm_name="$1" + local disk_path="" + disk_path="$( + virsh domblklist "${vm_name}" --details 2>/dev/null \ + | awk '$2 == "disk" && $4 != "-" {print $4; exit}' + )" + if [ -z "${disk_path}" ]; then + echo "Cannot determine disk path for ${vm_name}; refusing destructive cleanup." >&2 + exit 1 + fi + case "${disk_path}" in + "${kvmDiskDir}"/*) ;; + *) + echo "Refusing to destroy ${vm_name}: disk is outside this setup disk dir." >&2 + echo " vm_disk=${disk_path}" >&2 + echo " setup_disk_dir=${kvmDiskDir}" >&2 + echo "This usually means kvmState.yaml is stale or points to an existing cluster." >&2 + exit 1 + ;; + esac +} + +cleanupDnsmasqStaleLeases() { + [ -f "${dnsmasqStatus}" ] || return 0 + + local tmp_nodes + tmp_nodes="$(mktemp)" + awk -F '\t' 'NF >= 4 {print $1 "\t" $3 "\t" tolower($4)}' "${NODES_TSV}" > "${tmp_nodes}" + + sudo cp "${dnsmasqStatus}" "${dnsmasqStatus}.bak.$(date +%Y%m%d_%H%M%S)" + sudo python3 - "${dnsmasqStatus}" "${tmp_nodes}" <<'PY' +import json +import sys +from pathlib import Path + +status_path = Path(sys.argv[1]) +nodes_path = Path(sys.argv[2]) + +remove_names = set() +remove_ips = set() +remove_macs = set() +for line in nodes_path.read_text(encoding="utf-8").splitlines(): + parts = line.split("\t") + if len(parts) < 3: + continue + name, ip, mac = parts[:3] + if name: + remove_names.add(name) + if ip: + remove_ips.add(ip) + if mac: + remove_macs.add(mac.lower()) + +try: + data = json.loads(status_path.read_text(encoding="utf-8") or "[]") +except json.JSONDecodeError: + raise SystemExit(f"Invalid dnsmasq status JSON: {status_path}") + +filtered = [ + item for item in data + if not ( + item.get("hostname") in remove_names + or item.get("ip-address") in remove_ips + or str(item.get("mac-address", "")).lower() in remove_macs + ) +] +status_path.write_text(json.dumps(filtered, indent=2) + "\n", encoding="utf-8") +PY + rm -f "${tmp_nodes}" +} + +cleanupSetupOutputs() { + echo "Cleaning stale setup outputs matching: ${setupOutputGlob}" + rm -f ${setupOutputGlob} + rm -f "${CONFIG_PATH}" "${outputK3sConfig}" "${outputKubeconfig}" "${outputInventory}" + rm -f "${SETUP_DIR}/configRunning.yaml" +} + +verifyClean() { + local failed=0 + + echo "Verifying cleanup..." + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + [ -n "${name}" ] || continue + if virsh dominfo "${name}" >/dev/null 2>&1; then + echo "Residual domain: ${name}" >&2 + failed=1 + fi + if [ -e "${kvmDiskDir}/${name}.qcow2" ]; then + echo "Residual disk: ${kvmDiskDir}/${name}.qcow2" >&2 + failed=1 + fi + if [ -e "${kvmCloudInitDir}/${name}" ]; then + echo "Residual cloud-init dir: ${kvmCloudInitDir}/${name}" >&2 + failed=1 + fi + + if virsh net-dumpxml "${kvmNetwork}" 2>/dev/null \ + | grep -E "]*(mac=['\"]${mac}['\"]|name=['\"]${name}['\"]|ip=['\"]${ip}['\"])" >/dev/null 2>&1; then + echo "Residual DHCP reservation in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + + if virsh net-dhcp-leases "${kvmNetwork}" 2>/dev/null \ + | awk -v name="${name}" -v ip="${ip}/24" -v mac="$(printf '%s' "${mac}" | tr '[:upper:]' '[:lower:]')" ' + NR > 2 { + row = tolower($0) + if (row ~ mac && $5 == ip && $6 == name) { + found = 1 + } + } + END { exit(found ? 0 : 1) } + '; then + echo "Residual DHCP lease in libvirt network ${kvmNetwork}: ${name} ${ip} ${mac}" >&2 + failed=1 + fi + done < "${VERIFY_NODES_TSV}" + + if [ -e "${CONFIG_PATH}" ]; then + echo "Residual KVM state: ${CONFIG_PATH}" >&2 + failed=1 + fi + + if compgen -G "${setupOutputGlob}" >/dev/null; then + echo "Residual setup output files matching ${setupOutputGlob}" >&2 + compgen -G "${setupOutputGlob}" >&2 || true + failed=1 + fi + + if [ "${failed}" -ne 0 ]; then + echo "Cleanup verification failed." >&2 + exit 1 + fi + echo "Cleanup verification passed." +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand awk + requireCommand grep + requireCommand mktemp + requireCommand python3 + requireCommand sudo + requireCommand virsh + + trap cleanupTmp EXIT + loadConfig + printPlan + cleanupDomainsReservationsAndFiles + cleanupDnsmasqStaleLeases + cleanupSetupOutputs + verifyClean +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py new file mode 100755 index 000000000..db26816ea --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/manageKvmConfig.py @@ -0,0 +1,858 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +import re +import shlex +from pathlib import Path +from typing import Any + +import yaml + + +SETUP_DIR = Path(__file__).resolve().parent +REPO_ROOT = Path.home() / "k8s" +SETUP_DATA_DIR = SETUP_DIR + + +def expand_path(value: str) -> str: + return str(Path(os.path.expandvars(os.path.expanduser(value))).resolve()) + + +def expand_path_list(value: Any) -> str: + """Expand a YAML scalar/list into the shell path-list used by KVM scripts.""" + if value is None: + return str((Path.home() / "k8s/output").resolve()) + if isinstance(value, list): + return " ".join(expand_path(str(item)) for item in value if str(item).strip()) + text = str(value).strip() + if not text: + return "" + return " ".join(expand_path(part) for part in text.split()) + + +def load_config(path: str) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise SystemExit(f"Invalid config root in {path}: expected mapping") + normalize_config(data) + return data + + +def normalize_config(data: dict[str, Any]) -> None: + """Accept new camelCase YAML while preserving older internal lookups.""" + if "clusterName" in data and "cluster_name" not in data: + data["cluster_name"] = data["clusterName"] + for section in ("master", "workers"): + value = data.get(section) + if isinstance(value, dict): + aliases = {"memoryMb": "memory_mb", "diskGb": "disk_gb", "namePrefix": "name_prefix"} + for new_key, old_key in aliases.items(): + if new_key in value and old_key not in value: + value[old_key] = value[new_key] + kvm = data.get("kvm") + if isinstance(kvm, dict): + aliases = { + "storageDir": "storage_dir", + "diskDir": "disk_dir", + "cloudInitDir": "cloud_init_dir", + "baseImagePath": "base_image_path", + "legacyBaseImagePath": "legacy_base_image_path", + "baseImageUrl": "base_image_url", + "ubuntuSeries": "ubuntu_series", + "bootTimeoutSeconds": "boot_timeout_seconds", + "allowExisting": "allow_existing", + "skipK3sConfig": "skip_k3s_config", + } + for new_key, old_key in aliases.items(): + if new_key in kvm and old_key not in kvm: + kvm[old_key] = kvm[new_key] + outputs = data.get("outputs") + if isinstance(outputs, dict): + aliases = {"tmpDir": "tmp_dir", "k3sConfig": "k3s_config", "kvmState": "kvm_state", "runningConfig": "running_config"} + for new_key, old_key in aliases.items(): + if new_key in outputs and old_key not in outputs: + outputs[old_key] = outputs[new_key] + + +def get_nested(data: dict[str, Any], path: str, default: Any = None) -> Any: + cur: Any = data + for part in path.split("."): + if not isinstance(cur, dict): + return default + for candidate in (part, _snakeCase(part), _camelCase(part)): + if candidate in cur: + cur = cur[candidate] + break + else: + return default + return cur + + +def _snakeCase(value: str) -> str: + """Return a snake_case spelling for one YAML path component.""" + return re.sub(r"(? str: + """Return a camelCase spelling for one YAML path component.""" + parts = value.split("_") + return parts[0] + "".join(part[:1].upper() + part[1:] for part in parts[1:]) + + +def ubuntu_image_defaults(data: dict[str, Any]) -> tuple[str, str]: + series = str(get_nested(data, "kvm.ubuntu_series", "jammy")) + if series == "jammy": + return ( + "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img", + str(SETUP_DIR / "base/jammy-server-cloudimg-amd64.img"), + ) + if series == "noble": + return ( + "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img", + str(SETUP_DIR / "base/noble-server-cloudimg-amd64.img"), + ) + raise SystemExit(f"Unsupported kvm.ubuntu_series: {series}") + + +def default_seedemu_docker_dir() -> str: + """Return a likely source-tree path containing SeedEMU base/router images.""" + candidates = [] + for base in (SETUP_DIR, *SETUP_DIR.parents): + candidates.append(base / "docker_images/multiarch") + candidates.extend( + [ + Path.home() / "seed-emulator/docker_images/multiarch", + Path.home() / "seed-emulator-k8s-new/docker_images/multiarch", + Path.home() / "k8s/seed-emulator/docker_images/multiarch", + ] + ) + for candidate in candidates: + if (candidate / "seedemu-base").is_dir() and (candidate / "seedemu-router").is_dir(): + return str(candidate.resolve()) + return str(candidates[0].expanduser().resolve()) + + +def normalize_node(node: dict[str, Any], defaults: dict[str, Any] | None = None) -> dict[str, Any]: + defaults = defaults or {} + out = dict(defaults) + out.update(node) + required = ["name", "role", "ip", "mac", "vcpus", "memory_mb", "disk_gb"] + missing = [key for key in required if key not in out or out[key] in ("", None)] + if missing: + raise SystemExit(f"Node is missing required fields {missing}: {out}") + out["role"] = "master" if str(out["role"]) in {"master", "control-plane"} else "worker" + out["vcpus"] = int(out["vcpus"]) + out["memory_mb"] = int(out["memory_mb"]) + out["disk_gb"] = int(out["disk_gb"]) + return out + + +def read_existing_nodes(path: str | None) -> list[dict[str, str]]: + if not path: + return [] + existing_path = Path(path) + if not existing_path.exists(): + return [] + out: list[dict[str, str]] = [] + with existing_path.open("r", encoding="utf-8") as handle: + for raw in handle: + raw = raw.strip() + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + while len(parts) < 3: + parts.append("") + name, ip, mac = [part.strip() for part in parts[:3]] + if name or ip or mac: + out.append({"name": name, "ip": ip, "mac": mac.lower()}) + return out + + +def split_ipv4(ip: str) -> tuple[str, int] | None: + match = re.fullmatch(r"(\d+\.\d+\.\d+)\.(\d+)", ip) + if not match: + return None + octet = int(match.group(2)) + if octet < 1 or octet > 254: + return None + return match.group(1), octet + + +def mac_suffix(mac: str, prefix: str) -> int | None: + mac = mac.lower() + prefix = prefix.lower() + if not mac.startswith(prefix + ":"): + return None + suffix = mac.rsplit(":", 1)[-1] + try: + value = int(suffix, 16) + except ValueError: + return None + if value < 0 or value > 255: + return None + return value + + +def next_vm_name(base: str, used_names: set[str], first_number: int | None = None) -> str: + if first_number is None and base not in used_names: + used_names.add(base) + return base + number = first_number or 2 + while True: + candidate = f"{base}{number}" + if candidate not in used_names: + used_names.add(candidate) + return candidate + number += 1 + + +def next_worker_number(name_prefix: str, used_names: set[str]) -> int: + pattern = re.compile(rf"^{re.escape(name_prefix)}(\d+)$") + highest = 0 + for name in used_names: + match = pattern.fullmatch(name) + if match: + highest = max(highest, int(match.group(1))) + return highest + 1 + + +def next_ip(ip_prefix: str, start: int, used_ips: set[str]) -> str: + highest = start - 1 + for ip in used_ips: + parsed = split_ipv4(ip) + if parsed and parsed[0] == ip_prefix and parsed[1] >= start: + highest = max(highest, parsed[1]) + candidate = highest + 1 + while candidate <= 254: + ip = f"{ip_prefix}.{candidate}" + if ip not in used_ips: + used_ips.add(ip) + return ip + candidate += 1 + raise SystemExit(f"No available IPv4 address left in {ip_prefix}.0/24 starting at {start}") + + +def next_mac(mac_prefix: str, start: int, used_macs: set[str]) -> str: + highest = start - 1 + for mac in used_macs: + suffix = mac_suffix(mac, mac_prefix) + if suffix is not None and suffix >= start: + highest = max(highest, suffix) + candidate = highest + 1 + while candidate <= 255: + mac = f"{mac_prefix.lower()}:{candidate:02x}" + if mac not in used_macs: + used_macs.add(mac) + return mac + candidate += 1 + raise SystemExit(f"No available MAC suffix left for prefix {mac_prefix} starting at {start:02x}") + + +def auto_node( + role: str, + cfg: dict[str, Any], + used_names: set[str], + used_ips: set[str], + used_macs: set[str], + *, + default_name: str | None = None, + default_name_prefix: str | None = None, + worker_number: int | None = None, + default_ip_prefix: str, + default_ip_start: int, + default_mac_prefix: str, + default_mac_start: int, +) -> dict[str, Any]: + required = ["vcpus", "memory_mb", "disk_gb"] + missing = [key for key in required if cfg.get(key) in ("", None)] + if missing: + raise SystemExit(f"{role} config is missing required fields {missing}: {cfg}") + + if cfg.get("name"): + name = str(cfg["name"]) + if name in used_names: + raise SystemExit(f"Configured {role} name conflicts with existing/planned VM: {name}") + used_names.add(name) + elif role == "master": + name = next_vm_name(str(default_name or "seed-k3s-master"), used_names) + else: + prefix = str(default_name_prefix or "seed-k3s-worker") + number = worker_number or next_worker_number(prefix, used_names) + name = next_vm_name(prefix, used_names, first_number=number) + + ip = str(cfg["ip"]) if cfg.get("ip") else next_ip( + str(cfg.get("ip_prefix", default_ip_prefix)), + int(cfg.get("ip_start", default_ip_start)), + used_ips, + ) + if cfg.get("ip") and ip in used_ips: + raise SystemExit(f"Configured {role} IP conflicts with existing/planned VM: {ip}") + used_ips.add(ip) + + mac = str(cfg["mac"]).lower() if cfg.get("mac") else next_mac( + str(cfg.get("mac_prefix", default_mac_prefix)), + int(cfg.get("mac_start", default_mac_start)), + used_macs, + ) + if cfg.get("mac") and mac in used_macs: + raise SystemExit(f"Configured {role} MAC conflicts with existing/planned VM: {mac}") + used_macs.add(mac) + + return normalize_node( + { + "name": name, + "role": role, + "ip": ip, + "mac": mac, + "vcpus": cfg["vcpus"], + "memory_mb": cfg["memory_mb"], + "disk_gb": cfg["disk_gb"], + } + ) + + +def nodes(data: dict[str, Any], existing: list[dict[str, str]] | None = None) -> list[dict[str, Any]]: + existing = existing or [] + used_names = {item["name"] for item in existing if item.get("name")} + used_ips = {item["ip"] for item in existing if item.get("ip")} + used_macs = {item["mac"].lower() for item in existing if item.get("mac")} + explicit = data.get("nodes") + if explicit: + out = [normalize_node(item) for item in explicit] + else: + out = [] + master_cfg = data.get("master") + if "master" in data and master_cfg is not None: + out.append( + auto_node( + "master", + master_cfg or {}, + used_names, + used_ips, + used_macs, + default_name=str(get_nested(data, "defaults.master_name", "seed-k3s-master")), + default_ip_prefix=str(get_nested(data, "defaults.ip_prefix", "192.168.122")), + default_ip_start=int(get_nested(data, "defaults.master_ip_start", 110)), + default_mac_prefix=str(get_nested(data, "defaults.mac_prefix", "52:54:00:64:10")), + default_mac_start=int(get_nested(data, "defaults.master_mac_start", 0x10)), + ) + ) + workers_cfg = data.get("workers") or {} + if "workers" in data and data.get("workers") is not None and "count" not in workers_cfg: + raise SystemExit("workers.count is required when workers is configured") + count = int(workers_cfg.get("count", 0)) + if count < 0: + raise SystemExit("workers.count must be >= 0") + name_prefix = str(workers_cfg.get("name_prefix", get_nested(data, "defaults.worker_name_prefix", "seed-k3s-worker"))) + next_number = next_worker_number(name_prefix, used_names) + for idx in range(1, count + 1): + out.append( + auto_node( + "worker", + workers_cfg, + used_names, + used_ips, + used_macs, + default_name_prefix=name_prefix, + worker_number=next_number + idx - 1, + default_ip_prefix=str(workers_cfg.get("ip_prefix", get_nested(data, "defaults.ip_prefix", "192.168.122"))), + default_ip_start=int(workers_cfg.get("ip_start", get_nested(data, "defaults.worker_ip_start", 111))), + default_mac_prefix=str(workers_cfg.get("mac_prefix", get_nested(data, "defaults.mac_prefix", "52:54:00:64:10"))), + default_mac_start=int(workers_cfg.get("mac_start", get_nested(data, "defaults.worker_mac_start", 0x11))), + ) + ) + if not out: + raise SystemExit("No VMs requested: provide master and/or workers.count > 0") + + masters = [node for node in out if node["role"] == "master"] + if len(masters) > 1: + raise SystemExit(f"Expected at most one master node, got {len(masters)}") + names = [node["name"] for node in out] + ips = [node["ip"] for node in out] + macs = [node["mac"] for node in out] + for label, values in (("name", names), ("ip", ips), ("mac", macs)): + dup = sorted({value for value in values if values.count(value) > 1}) + if dup: + raise SystemExit(f"Duplicate node {label}: {dup}") + return out + + +def master_node(data: dict[str, Any]) -> dict[str, Any]: + return [node for node in nodes(data) if node["role"] == "master"][0] + + +def optional_master_node(data: dict[str, Any], existing: list[dict[str, str]] | None = None) -> dict[str, Any] | None: + masters = [node for node in nodes(data, existing) if node["role"] == "master"] + return masters[0] if masters else None + + +def install_version(data: dict[str, Any]) -> str: + version = str(get_nested(data, "k3s.version", "v1.28.5+k3s1")) + artifact = str(get_nested(data, "k3s.artifact_url", "https://rancher-mirror.rancher.cn/k3s")) + configured = get_nested(data, "k3s.install_version") + if configured: + return str(configured) + if "rancher-mirror.rancher.cn/k3s" in artifact: + return version.replace("+", "-") + return version + + +def shell_env(args: argparse.Namespace) -> None: + kvm_env(args) + + +def kvm_env(args: argparse.Namespace) -> None: + data = load_config(args.config) + base_url, base_path = ubuntu_image_defaults(data) + existing = read_existing_nodes(args.existing_tsv) + master = optional_master_node(data, existing) + cluster_name = str(data.get("cluster_name", "seedemu-k3s")) + legacy_base_image = get_nested( + data, + "kvm.legacy_base_image_path", + REPO_ROOT / f"output/kvm_lab/base/{Path(base_path).name}", + ) + storage_dir = expand_path(str(get_nested(data, "kvm.storage_dir", SETUP_DIR))) + disk_dir = expand_path(str(get_nested(data, "kvm.disk_dir", SETUP_DATA_DIR / "disks"))) + cloud_init_dir = expand_path(str(get_nested(data, "kvm.cloud_init_dir", SETUP_DIR / "cloud-init"))) + values = { + "clusterName": cluster_name, + "kvmNetwork": get_nested(data, "kvm.network", "default"), + "kvmStorageDir": storage_dir, + "kvmDiskDir": disk_dir, + "kvmCloudInitDir": cloud_init_dir, + "kvmUbuntuSeries": get_nested(data, "kvm.ubuntu_series", "jammy"), + "kvmBaseImageUrl": get_nested(data, "kvm.base_image_url", base_url), + "kvmBaseImagePath": expand_path(str(get_nested(data, "kvm.base_image_path", base_path))), + "kvmLegacyBaseImagePath": expand_path(str(legacy_base_image)) if legacy_base_image else "", + "kvmBaseImageSearchDirs": expand_path_list(get_nested(data, "kvm.base_image_search_dirs")), + "kvmBootTimeoutSeconds": get_nested(data, "kvm.boot_timeout_seconds", 300), + "kvmAllowExisting": str(get_nested(data, "kvm.allow_existing", False)).lower(), + "kvmSkipK3sConfig": str(get_nested(data, "kvm.skip_k3s_config", False)).lower(), + "sshUser": get_nested(data, "ssh.user", "ubuntu"), + "sshKey": expand_path(str(get_nested(data, "ssh.key", "~/.ssh/id_ed25519"))), + "masterName": master["name"] if master else "", + "masterIp": master["ip"] if master else "", + "outputK3sConfig": expand_path(str(get_nested(data, "outputs.k3s_config", SETUP_DIR / "configK3s.yaml"))), + "outputKvmState": expand_path(str(get_nested(data, "outputs.kvm_state", SETUP_DIR / "kvmState.yaml"))), + "outputKubeconfig": expand_path(str(get_nested(data, "outputs.kubeconfig", SETUP_DIR / f"{cluster_name}.kubeconfig.yaml"))), + "outputInventory": expand_path(str(get_nested(data, "outputs.inventory", SETUP_DIR / f"{cluster_name}.inventory.yaml"))), + "seedEmulatorDockerDir": expand_path( + str( + get_nested( + data, + "seedemu.dockerImagesDir", + get_nested(data, "seedemu.docker_images_dir", default_seedemu_docker_dir()), + ) + ) + ), + } + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def nodes_tsv(args: argparse.Namespace) -> None: + for node in nodes(load_config(args.config), read_existing_nodes(args.existing_tsv)): + print( + "\t".join( + str(node[key]) + for key in ("name", "role", "ip", "mac", "vcpus", "memory_mb", "disk_gb") + ) + ) + + +def write_inventory(args: argparse.Namespace) -> None: + data = load_config(args.config) + cluster_name = str(data.get("cluster_name", "seedemu-k3s")) + master = master_node(data) + output = Path(args.output or expand_path(str(get_nested(data, "outputs.inventory", SETUP_DIR / f"{cluster_name}.inventory.yaml")))) + payload = { + "cluster_name": cluster_name, + "reference_cluster": False, + "runtime": "k3s", + "max_validated_topology_size": int(get_nested(data, "max_validated_topology_size", 12000)), + "k3s": { + "cluster_cidr": get_nested(data, "k3s.cluster_cidr", "10.42.0.0/16"), + "service_cidr": get_nested(data, "k3s.service_cidr", "10.43.0.0/16"), + "node_cidr_mask_size_ipv4": int(get_nested(data, "k3s.node_cidr_mask_size_ipv4", 20)), + "max_pods": int(get_nested(data, "k3s.max_pods", 4000)), + }, + "network_tuning": { + "cni0_hash_max": int(get_nested(data, "tuning.cni0_hash_max", 16384)), + "user_max_net_namespaces": int(get_nested(data, "tuning.user_max_net_namespaces", 65536)), + "neigh_gc_thresh1": int(get_nested(data, "tuning.neigh_gc_thresh1", 1048576)), + "neigh_gc_thresh2": int(get_nested(data, "tuning.neigh_gc_thresh2", 4194304)), + "neigh_gc_thresh3": int(get_nested(data, "tuning.neigh_gc_thresh3", 8388608)), + "netdev_max_backlog": int(get_nested(data, "tuning.netdev_max_backlog", 1000000)), + "optmem_max": int(get_nested(data, "tuning.optmem_max", 25165824)), + }, + "ssh": { + "user": get_nested(data, "ssh.user", "ubuntu"), + "default_key_path": get_nested(data, "ssh.key", "~/.ssh/id_ed25519"), + }, + "registry": { + "host": get_nested(data, "registry.host", master["ip"]), + "port": int(get_nested(data, "registry.port", 5000)), + }, + "cni": {"default_master_interface": get_nested(data, "cni.default_master_interface", "ens2")}, + "nodes": [ + { + "name": node["name"], + "role": node["role"], + "management_ip": node["ip"], + "runtime": "k3s", + "resources": { + "vcpus": node["vcpus"], + "memory_mb": node["memory_mb"], + "disk_gb": node["disk_gb"], + }, + "labels": {"kubernetes.io/hostname": node["name"]}, + } + for node in nodes(data) + ], + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + print(output) + + +def read_nodes_tsv(path: str) -> list[dict[str, Any]]: + """Read a transient node TSV produced by nodes-tsv. + + Args: + path: TSV path with name, role, ip, mac, vcpus, memory_mb, disk_gb. + """ + out: list[dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as handle: + for raw in handle: + raw = raw.strip() + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + while len(parts) < 7: + parts.append("") + name, role, ip, mac, vcpus, memory_mb, disk_gb = parts[:7] + out.append( + { + "name": name, + "role": role, + "ip": ip, + "mac": mac, + "vcpus": int(vcpus or 0), + "memoryMb": int(memory_mb or 0), + "diskGb": int(disk_gb or 0), + } + ) + return out + + +def to_k3s_node_payload(node: dict[str, Any]) -> dict[str, Any]: + """Normalize KVM node records to configK3s.yaml node shape. + + Args: + node: Node mapping from kvm.yaml expansion or transient TSV. + """ + return { + "name": node["name"], + "role": node["role"], + "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), + "ssh": { + "user": node.get("sshUser", ""), + "key": node.get("sshKey", ""), + }, + } + + +def to_kvm_state_node_payload(node: dict[str, Any]) -> dict[str, Any]: + """Normalize KVM node records to kvmState.yaml node shape. + + Args: + node: Node mapping from kvm.yaml expansion or transient TSV. + """ + return { + "name": node["name"], + "role": node["role"], + "ip": node["ip"], + "mac": node.get("mac", ""), + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), + } + + +def k3s_passthrough_sections(data: dict[str, Any], cluster_name: str) -> dict[str, Any]: + """Return kvm.yaml sections that must survive into configK3s.yaml. + + Args: + data: Normalized kvm.yaml mapping. + cluster_name: Effective cluster name used for default output paths. + + KVM creation owns VM resources, but the next K3s stage still needs backend + selection and output paths. Keep only K3s-facing sections so configK3s.yaml + stays simple for users and does not repeat CPU/memory/disk settings. + """ + payload: dict[str, Any] = {} + for section in ("registry", "k3s", "fabric", "ovn", "cni", "tuning", "seedemu"): + value = data.get(section) + if isinstance(value, dict) and value: + payload[section] = value + + outputs = data.get("outputs") + if isinstance(outputs, dict): + output_payload: dict[str, Any] = {} + tmp_dir = get_nested(data, "outputs.tmp_dir") + kubeconfig = get_nested(data, "outputs.kubeconfig") + inventory = get_nested(data, "outputs.inventory") + if tmp_dir: + output_payload["tmpDir"] = expand_path(str(tmp_dir)) + if kubeconfig: + output_payload["kubeconfig"] = expand_path(str(kubeconfig)) + else: + output_payload["kubeconfig"] = expand_path(str(SETUP_DIR / f"{cluster_name}.kubeconfig.yaml")) + if inventory: + output_payload["inventory"] = expand_path(str(inventory)) + else: + output_payload["inventory"] = expand_path(str(SETUP_DIR / f"{cluster_name}.inventory.yaml")) + payload["outputs"] = output_payload + return payload + + +def write_k3s_config(args: argparse.Namespace) -> None: + """Write configK3s.yaml after KVM node names/IPs are resolved. + + Args: + args.config: Source kvm.yaml. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. + args.output: Destination configK3s.yaml path. + """ + data = load_config(args.config) + cluster_name = str(data.get("cluster_name", "seedemu-k3s")) + ssh_user = str(get_nested(data, "ssh.user", "ubuntu")) + ssh_key = expand_path(str(get_nested(data, "ssh.key", "~/.ssh/id_ed25519"))) + resolved_nodes = [ + to_k3s_node_payload({**node, "sshUser": ssh_user, "sshKey": ssh_key}) + for node in (read_nodes_tsv(args.nodes_tsv) if args.nodes_tsv else nodes(data)) + ] + masters = [node for node in resolved_nodes if node["role"] == "master"] + if len(masters) != 1: + raise SystemExit(f"Expected exactly one master in configK3s.yaml nodes, got {len(masters)}") + output = Path(args.output or expand_path(str(get_nested(data, "outputs.k3s_config", SETUP_DIR / "configK3s.yaml")))) + payload = { + "clusterName": cluster_name, + "nodes": resolved_nodes, + } + payload.update(k3s_passthrough_sections(data, cluster_name)) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + print(output) + + +def write_kvm_state(args: argparse.Namespace) -> None: + """Write kvmState.yaml after KVM node names/IPs are resolved. + + Args: + args.config: Source kvm.yaml. + args.nodes_tsv: Transient node TSV generated by createKvmVms.py. + args.output: Destination kvmState.yaml path. + """ + data = load_config(args.config) + cluster_name = str(data.get("cluster_name", "seedemu-k3s")) + resolved_nodes = [ + to_kvm_state_node_payload(node) + for node in (read_nodes_tsv(args.nodes_tsv) if args.nodes_tsv else nodes(data)) + ] + output = Path(args.output or expand_path(str(get_nested(data, "outputs.kvm_state", SETUP_DIR / "kvmState.yaml")))) + payload = { + "clusterName": cluster_name, + "kvm": { + "network": get_nested(data, "kvm.network", "default"), + "diskDir": expand_path(str(get_nested(data, "kvm.disk_dir", SETUP_DATA_DIR / "disks"))), + "cloudInitDir": expand_path(str(get_nested(data, "kvm.cloud_init_dir", SETUP_DIR / "cloud-init"))), + }, + "outputs": { + "k3sConfig": expand_path(str(get_nested(data, "outputs.k3s_config", SETUP_DIR / "configK3s.yaml"))), + "tmpDir": expand_path(str(get_nested(data, "outputs.tmp_dir", SETUP_DIR / "tmp"))), + "kubeconfig": expand_path(str(get_nested(data, "outputs.kubeconfig", SETUP_DIR / f"{cluster_name}.kubeconfig.yaml"))), + "inventory": expand_path(str(get_nested(data, "outputs.inventory", SETUP_DIR / f"{cluster_name}.inventory.yaml"))), + }, + "nodes": resolved_nodes, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + print(output) + + +def load_kvm_state(path: str) -> dict[str, Any]: + """Load kvmState.yaml and validate its root mapping.""" + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise SystemExit(f"Invalid kvmState.yaml root: {path}") + if not isinstance(data.get("nodes"), list): + raise SystemExit(f"Invalid kvmState.yaml nodes list: {path}") + return data + + +def kvm_state_nodes(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return normalized nodes from kvmState.yaml.""" + return [to_kvm_state_node_payload(node) for node in data["nodes"]] + + +def state_nodes_tsv(args: argparse.Namespace) -> None: + """Print kvmState.yaml nodes as TSV.""" + for node in kvm_state_nodes(load_kvm_state(args.state)): + print( + "\t".join( + str(node[key]) + for key in ("name", "role", "ip", "mac", "vcpus", "memoryMb", "diskGb") + ) + ) + + +def state_vars(args: argparse.Namespace) -> None: + """Print shell assignments derived from kvmState.yaml.""" + data = load_kvm_state(args.state) + cluster_name = str(data.get("clusterName") or data.get("cluster_name") or "seedemu-k3s") + values = { + "clusterName": cluster_name, + "kvmNetwork": get_nested(data, "kvm.network", "default"), + "kvmDiskDir": expand_path(str(get_nested(data, "kvm.diskDir", SETUP_DATA_DIR / "disks"))), + "kvmCloudInitDir": expand_path(str(get_nested(data, "kvm.cloudInitDir", SETUP_DIR / "cloud-init"))), + "outputK3sConfig": expand_path(str(get_nested(data, "outputs.k3sConfig", SETUP_DIR / "configK3s.yaml"))), + "outputKubeconfig": expand_path(str(get_nested(data, "outputs.kubeconfig", SETUP_DIR / f"{cluster_name}.kubeconfig.yaml"))), + "outputInventory": expand_path(str(get_nested(data, "outputs.inventory", SETUP_DIR / f"{cluster_name}.inventory.yaml"))), + } + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def validate_kvm_state(args: argparse.Namespace) -> None: + """Validate that an existing kvmState.yaml still matches kvm.yaml. + + Args: + args.config: Source kvm.yaml. + args.state: Existing kvmState.yaml that createKvmVms.py may reuse. + """ + data = load_config(args.config) + expected = nodes(data) + resolved = kvm_state_nodes(load_kvm_state(args.state)) + + def signature(items: list[dict[str, Any]]) -> list[tuple[str, int, int, int]]: + result = [] + for item in items: + result.append( + ( + str(item["role"]), + int(item.get("vcpus") or 0), + int(item.get("memoryMb") or item.get("memory_mb") or 0), + int(item.get("diskGb") or item.get("disk_gb") or 0), + ) + ) + return sorted(result) + + if signature(expected) != signature(resolved): + raise SystemExit( + "kvmState.yaml node plan does not match current kvm.yaml. " + f"expected={signature(expected)} resolved={signature(resolved)}" + ) + + +def write_ansible_inventory(args: argparse.Namespace) -> None: + data = load_config(args.config) + all_nodes = nodes(data) + master = [node for node in all_nodes if node["role"] == "master"][0] + workers = [node for node in all_nodes if node["role"] == "worker"] + payload = { + "all": { + "vars": { + "ansible_user": get_nested(data, "ssh.user", "ubuntu"), + "ansible_ssh_private_key_file": expand_path(str(get_nested(data, "ssh.key", "~/.ssh/id_ed25519"))), + "k3s_version": get_nested(data, "k3s.version", "v1.28.5+k3s1"), + "k3s_install_version": install_version(data), + "seed_registry_host": get_nested(data, "registry.host", master["ip"]), + "seed_registry_port": get_nested(data, "registry.port", 5000), + "seed_docker_io_mirror_endpoint": get_nested(data, "registry.docker_io_mirror_endpoint", "https://docker.m.daocloud.io"), + "seed_k3s_artifact_url": get_nested(data, "k3s.artifact_url", "https://rancher-mirror.rancher.cn/k3s"), + "seed_cni_master_interface": get_nested(data, "cni.default_master_interface", "ens2"), + "seed_k3s_cluster_cidr": get_nested(data, "k3s.cluster_cidr", "10.42.0.0/16"), + "seed_k3s_service_cidr": get_nested(data, "k3s.service_cidr", "10.43.0.0/16"), + "seed_k3s_node_cidr_mask_size_ipv4": get_nested(data, "k3s.node_cidr_mask_size_ipv4", 20), + "seed_k3s_max_pods": get_nested(data, "k3s.max_pods", 4000), + "seed_k3s_force_reinstall": bool(get_nested(data, "k3s.force_reinstall", False)), + }, + "children": { + "master": { + "hosts": { + master["name"]: { + "ansible_host": master["ip"], + "k3s_role": "server", + "seedemu_as_group": "master", + } + } + }, + "workers": { + "hosts": { + node["name"]: { + "ansible_host": node["ip"], + "k3s_role": "agent", + "seedemu_as_group": f"worker-{idx}", + } + for idx, node in enumerate(workers, start=1) + } + }, + }, + } + } + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + print(output) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("config") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("shell-vars").set_defaults(func=shell_env) + kvm_env_parser = sub.add_parser("kvm-vars") + kvm_env_parser.add_argument("--existing-tsv") + kvm_env_parser.set_defaults(func=kvm_env) + nodes_parser = sub.add_parser("nodes-tsv") + nodes_parser.add_argument("--existing-tsv") + nodes_parser.set_defaults(func=nodes_tsv) + inv = sub.add_parser("write-inventory") + inv.add_argument("--output") + inv.set_defaults(func=write_inventory) + k3s = sub.add_parser("write-k3s-config") + k3s.add_argument("--nodes-tsv") + k3s.add_argument("--output") + k3s.set_defaults(func=write_k3s_config) + state = sub.add_parser("write-kvm-state") + state.add_argument("--nodes-tsv") + state.add_argument("--output") + state.set_defaults(func=write_kvm_state) + state_nodes = sub.add_parser("state-nodes-tsv") + state_nodes.add_argument("--state", required=True) + state_nodes.set_defaults(func=state_nodes_tsv) + state_env = sub.add_parser("state-vars") + state_env.add_argument("--state", required=True) + state_env.set_defaults(func=state_vars) + validate = sub.add_parser("validate-kvm-state") + validate.add_argument("--state", required=True) + validate.set_defaults(func=validate_kvm_state) + ansible = sub.add_parser("write-ansible-inventory") + ansible.add_argument("--output", required=True) + ansible.set_defaults(func=write_ansible_inventory) + args = parser.parse_args() + args.func(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py new file mode 100755 index 000000000..73f5e5e27 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/prepareHostAssets.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Python entrypoint for prepareHostAssets with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare host-side assets required before KVM creation: +# Ubuntu cloud image, registry/K3s bootstrap image tarballs, and SeedEMU base +# image tags. All user-configurable paths come from kvm.yaml. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageKvmConfig.py" + +HOST_IMAGE_CACHE_DIR="${SCRIPT_DIR}/image-cache" +hostDockerIoMirror="docker.m.daocloud.io" +dockerPullTimeoutSeconds=180 +baseImageLowSpeedSeconds=60 +baseImageLowSpeedLimit=1024 +prepareForce="false" +kvmBaseImageReuseMode="copy" +REGISTRY_BOOTSTRAP_IMAGE="registry:2" +MULTUS_BOOTSTRAP_IMAGE="ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot" +K3S_SYSTEM_BOOTSTRAP_IMAGES=( + "rancher/mirrored-coredns-coredns:1.10.1" + "rancher/mirrored-metrics-server:v0.6.3" + "rancher/local-path-provisioner:v0.0.24" +) +seedEmulatorDockerDir="${HOME}/seed-emulator/docker_images/multiarch" +kvmBaseImageSearchDirs="${HOME}/k8s/output" +seedBaseSourceImage="handsonsecurity/seedemu-multiarch-base:buildx-latest" +seedRouterSourceImage="handsonsecurity/seedemu-multiarch-router:buildx-latest" +seedBaseHashImage="98a2693c996c2294358552f48373498d:latest" +seedRouterHashImage="39e016aa9e819f203ebc1809245a5818:latest" +ubuntuBuildImage="ubuntu:20.04" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +resolveSeedEmulatorDockerDir() { + # Resolve the source-tree path containing seedemu-base and seedemu-router. + # kvm.yaml seedemu.dockerImagesDir wins through manageKvmConfig.py; the + # ancestor scan keeps examples runnable directly from this repository. + local cursor="${SETUP_DIR}" + while [ "${cursor}" != "/" ]; do + if [ -d "${cursor}/docker_images/multiarch/seedemu-base" ] && + [ -d "${cursor}/docker_images/multiarch/seedemu-router" ]; then + seedEmulatorDockerDir="${cursor}/docker_images/multiarch" + return 0 + fi + cursor="$(dirname "${cursor}")" + done + local candidate + for candidate in \ + "${seedEmulatorDockerDir}" \ + "${HOME}/seed-emulator-k8s-new/docker_images/multiarch" \ + "${HOME}/k8s/seed-emulator/docker_images/multiarch"; do + if [ -d "${candidate}/seedemu-base" ] && [ -d "${candidate}/seedemu-router" ]; then + seedEmulatorDockerDir="$(cd "${candidate}" && pwd)" + return 0 + fi + done +} + +imageTarName() { + printf '%s\n' "$1" | sed 's|[^A-Za-z0-9_.-]|_|g' +} + +dockerIoMirrorRef() { + local image="$1" + + if [[ "${image}" == */*/* ]]; then + return 1 + fi + + if [[ "${image}" == */* ]]; then + printf '%s/%s\n' "${hostDockerIoMirror}" "${image}" + else + printf '%s/library/%s\n' "${hostDockerIoMirror}" "${image}" + fi +} + +ensureHostDockerImage() { + local image="$1" + local mirror_image="" + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo " docker pull ${image}" + if runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${image}" >/dev/null; then + return 0 + fi + + if mirror_image="$(dockerIoMirrorRef "${image}")"; then + echo " docker pull ${mirror_image}" + runWithTimeout "${dockerPullTimeoutSeconds}s" docker pull "${mirror_image}" >/dev/null + docker tag "${mirror_image}" "${image}" >/dev/null + return 0 + fi + + echo "Failed to prepare Docker image on host: ${image}" >&2 + return 1 +} + +hostImageTarball() { + local image="$1" + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + printf '%s/%s.tar\n' "${HOST_IMAGE_CACHE_DIR}" "$(imageTarName "${image}")" +} + +saveHostImageTarball() { + local image="$1" + local tar_path + tar_path="$(hostImageTarball "${image}")" + if [ "${prepareForce}" != "true" ] && [ -s "${tar_path}" ]; then + echo " cache exists: ${tar_path}" + return + fi + ensureHostDockerImage "${image}" + echo " docker save ${image} -> ${tar_path}" + docker save -o "${tar_path}" "${image}" +} + +validBaseImage() { + # Args: + # $1: qcow2 image path to validate. + local image_path="$1" + [ -s "${image_path}" ] && qemu-img info "${image_path}" >/dev/null 2>&1 +} + +downloadBaseImageFile() { + # Args: + # $1: URL, $2: destination path. + # Downloads through a resumable .part file and validates qcow2 metadata + # before publishing the final path. + local image_url="$1" + local image_path="$2" + local partial_path="${image_path}.part" + mkdir -p "$(dirname "${image_path}")" + curl -fL \ + --retry 5 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --speed-time "${baseImageLowSpeedSeconds}" \ + --speed-limit "${baseImageLowSpeedLimit}" \ + -C - \ + -o "${partial_path}" \ + "${image_url}" + validBaseImage "${partial_path}" || { + echo "Downloaded base image is not a valid qcow2 file: ${partial_path}" >&2 + return 1 + } + mv -f "${partial_path}" "${image_path}" +} + +prepareBaseImage() { + mkdir -p "$(dirname "${kvmBaseImagePath}")" + + if [ -e "${kvmBaseImagePath}" ]; then + if validBaseImage "${kvmBaseImagePath}"; then + echo "Base image already exists: ${kvmBaseImagePath}" + return + fi + echo "Removing incomplete or invalid base image: ${kvmBaseImagePath}" + rm -f "${kvmBaseImagePath}" + fi + + if [ -n "${kvmLegacyBaseImagePath:-}" ] && [ -f "${kvmLegacyBaseImagePath}" ]; then + echo "Reusing existing base image from ${kvmLegacyBaseImagePath}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${kvmLegacyBaseImagePath}" "${kvmBaseImagePath}" + fi + return + fi + + local output_image="" + local search_dir="" + for search_dir in ${kvmBaseImageSearchDirs}; do + [ -d "${search_dir}" ] || continue + output_image="$(find "${search_dir}" -type f -name "$(basename "${kvmBaseImagePath}")" -print -quit 2>/dev/null || true)" + if [ -n "${output_image}" ]; then + echo "Reusing existing base image from ${output_image}" + if [ "${kvmBaseImageReuseMode}" = "symlink" ]; then + ln -s "${output_image}" "${kvmBaseImagePath}" + else + cp --reflink=auto "${output_image}" "${kvmBaseImagePath}" + fi + return + fi + done + + echo "Downloading Ubuntu cloud image:" + echo " url=${kvmBaseImageUrl}" + echo " output=${kvmBaseImagePath}" + downloadBaseImageFile "${kvmBaseImageUrl}" "${kvmBaseImagePath}" +} + +prepareSeedemuBuildImages() { + ensureHostDockerImage "${ubuntuBuildImage}" + + if ! docker image inspect "${seedBaseSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-base" ]; then + echo " docker build ${seedBaseSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedBaseSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-base" >/dev/null + else + ensureHostDockerImage "${seedBaseSourceImage}" + fi + fi + + if ! docker image inspect "${seedRouterSourceImage}" >/dev/null 2>&1; then + if [ -d "${seedEmulatorDockerDir}/seedemu-router" ]; then + echo " docker build ${seedRouterSourceImage}" + DOCKER_BUILDKIT=1 docker build -t "${seedRouterSourceImage}" \ + "${seedEmulatorDockerDir}/seedemu-router" >/dev/null + else + ensureHostDockerImage "${seedRouterSourceImage}" + fi + fi + + docker tag "${seedBaseSourceImage}" "${seedBaseHashImage}" >/dev/null + docker tag "${seedRouterSourceImage}" "${seedRouterHashImage}" >/dev/null +} + +prepareImageCache() { + mkdir -p "${HOST_IMAGE_CACHE_DIR}" + prepareSeedemuBuildImages + + saveHostImageTarball "${REGISTRY_BOOTSTRAP_IMAGE}" + saveHostImageTarball "${MULTUS_BOOTSTRAP_IMAGE}" + for image in "${K3S_SYSTEM_BOOTSTRAP_IMAGES[@]}"; do + saveHostImageTarball "${image}" + done + saveHostImageTarball "${ubuntuBuildImage}" + saveHostImageTarball "${seedBaseSourceImage}" + saveHostImageTarball "${seedRouterSourceImage}" + + echo " hash tags prepared locally, not saved into image-cache:" + echo " ${seedBaseHashImage}" + echo " ${seedRouterHashImage}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + + requireCommand python3 + requireCommand curl + requireCommand docker + requireCommand find + requireCommand qemu-img + + eval "$(python3 "${HELPER}" "${CONFIG_PATH}" kvm-vars)" + resolveSeedEmulatorDockerDir + + echo "Preparing setup assets using config: ${CONFIG_PATH}" + echo "base_image_path=${kvmBaseImagePath}" + echo "image_cache_dir=${HOST_IMAGE_CACHE_DIR}" + echo "seedemu_docker_dir=${seedEmulatorDockerDir}" + prepareBaseImage + prepareImageCache + echo "Setup assets are ready." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py new file mode 100755 index 000000000..3ec843914 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/kvm/tuneVmLimits.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Python entrypoint for tuneVmLimits with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Tune OS-level limits on each VM listed in configK3s.yaml. +# Inputs: configK3s.yaml with nodes and SSH settings. +# Outputs/side effects: writes sysctl, limits, systemd drop-ins, and cni0 +# hash_max service on each VM through SSH. No K3s installation is performed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +# Keep these defaults in the script, not in configK3s.yaml. The YAML describes VMs; +# this script describes the current high-density OS limit policy. +vmLimitNofile="10485760" +vmLimitNproc="4194304" +vmLimitMaxNetNamespaces="65536" +vmLimitNeighGcThresh1="1048576" +vmLimitNeighGcThresh2="4194304" +vmLimitNeighGcThresh3="8388608" +vmLimitNetdevMaxBacklog="1000000" +vmLimitOptmemMax="25165824" +vmLimitCni0HashMax="16384" +vmLimitReboot="false" + +NODE_SSH_OPTS=() +nodeSshUser="" +nodeSshKey="" +nodeConnection="" + +usage() { + cat <&2 + exit 1 + } + NODE_SSH_OPTS=( + -i "${nodeSshKey}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o IdentitiesOnly=yes + -o IdentityAgent=none + -o ConnectTimeout=10 + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + ) +} + +requireCommand() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +waitForSsh() { + local name="$1" + local ip="$2" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + sudo -n true >/dev/null 2>&1 || { + echo "Local sudo failed for ${name} (${ip})" >&2 + return 1 + } + return 0 + fi + if ssh -n "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "echo ok" >/dev/null 2>&1; then + return 0 + fi + echo "SSH failed for ${name} (${ip})" >&2 + return 1 +} + +unlockOneVm() { + local name="$1" + local ip="$2" + local remote_runner=() + echo "Unlocking VM limits: ${name} (${ip})" + loadNodeSshContext "${name}" + if [ "${nodeConnection:-ssh}" = "local" ]; then + remote_runner=(sudo -n bash -s --) + else + remote_runner=(ssh "${NODE_SSH_OPTS[@]}" "${nodeSshUser}@${ip}" "sudo -n bash -s" --) + fi + "${remote_runner[@]}" \ + "${vmLimitNofile}" \ + "${vmLimitNproc}" \ + "${vmLimitMaxNetNamespaces}" \ + "${vmLimitNeighGcThresh1}" \ + "${vmLimitNeighGcThresh2}" \ + "${vmLimitNeighGcThresh3}" \ + "${vmLimitNetdevMaxBacklog}" \ + "${vmLimitOptmemMax}" \ + "${vmLimitCni0HashMax}" \ + "${vmLimitReboot}" <<'EOF_REMOTE' +set -euo pipefail + +LIMIT_NOFILE="$1" +LIMIT_NPROC="$2" +MAX_NET_NS="$3" +NEIGH1="$4" +NEIGH2="$5" +NEIGH3="$6" +NETDEV_BACKLOG="$7" +OPTMEM_MAX="$8" +CNI0_HASH_MAX="$9" +DO_REBOOT="${10}" + +appendIfMissing() { + local file="$1" + local pattern="$2" + local line="$3" + touch "${file}" + if ! grep -qF "${pattern}" "${file}"; then + printf '%s\n' "${line}" >> "${file}" + fi +} + +echo ">> limits.conf" +appendIfMissing /etc/security/limits.conf "* soft nofile" "* soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* hard nofile" "* hard nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "* soft nproc" "* soft nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "* hard nproc" "* hard nproc ${LIMIT_NPROC}" +appendIfMissing /etc/security/limits.conf "root soft nofile" "root soft nofile ${LIMIT_NOFILE}" +appendIfMissing /etc/security/limits.conf "root hard nofile" "root hard nofile ${LIMIT_NOFILE}" + +echo ">> sysctl" +cat > /etc/sysctl.d/99-seed-vm-limits.conf </dev/null 2>&1 || true +ip -s -s neigh flush all >/dev/null 2>&1 || true + +echo ">> systemd default limits" +mkdir -p /etc/systemd/system.conf.d /etc/systemd/user.conf.d +cat > /etc/systemd/system.conf.d/99-seed-vm-limits.conf < /etc/systemd/user.conf.d/99-seed-vm-limits.conf <> known service drop-ins if present" +for service in k3s k3s-agent containerd docker; do + if systemctl list-unit-files | grep -q "^${service}.service"; then + mkdir -p "/etc/systemd/system/${service}.service.d" + cat > "/etc/systemd/system/${service}.service.d/99-seed-vm-limits.conf" <> cni0 hash_max service for future K3s/flannel bridge" +cat > /usr/local/sbin/seed-vm-cni0-hashmax.sh <<'EOF_TUNE' +#!/usr/bin/env bash +set -euo pipefail +target="${1:-16384}" +for _ in $(seq 1 60); do + if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + printf '%s\n' "${target}" > /sys/class/net/cni0/bridge/hash_max + exit 0 + fi + sleep 2 +done +exit 0 +EOF_TUNE +chmod +x /usr/local/sbin/seed-vm-cni0-hashmax.sh + +cat > /etc/systemd/system/seed-vm-cni0-hashmax.service </dev/null 2>&1 || true +if [ -w /sys/class/net/cni0/bridge/hash_max ]; then + /usr/local/sbin/seed-vm-cni0-hashmax.sh "${CNI0_HASH_MAX}" >/dev/null 2>&1 || true +fi + +if [ "${DO_REBOOT}" = "true" ]; then + systemd-run --on-active=2 /bin/bash -c "reboot" >/dev/null 2>&1 || true +fi + +echo "VM limit unlock completed on $(hostname)" +EOF_REMOTE +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + echo "Using K3s config: ${CONFIG_PATH}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + waitForSsh "${name}" "${ip}" + done < <(nodesInput) + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + unlockOneVm "${name}" "${ip}" + done < <(nodesInput) + + echo "VM limit unlock completed for all nodes in ${CONFIG_PATH}" +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/manageK3sConfig.py b/seedemu/k8sTools/resources/setup/manageK3sConfig.py new file mode 100644 index 000000000..3566b2c08 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/manageK3sConfig.py @@ -0,0 +1,736 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import getpass +import os +import re +import shlex +import subprocess +from pathlib import Path +from typing import Any + +import yaml + + +SETUP_DIR = Path(__file__).resolve().parent +_LOCAL_IPS: set[str] | None = None + + +def expandPath(value: str) -> str: + """Expand ~ and return an absolute path string.""" + return str(Path(os.path.expanduser(value)).resolve()) + + +def readLocalIps() -> set[str]: + """Return IP addresses configured on the host running this helper. + + The K3s setup scripts use this to detect the common physical-server case + where the configured master node is the current machine and should be + handled with local commands instead of SSH-to-self. + """ + global _LOCAL_IPS + if _LOCAL_IPS is not None: + return _LOCAL_IPS + ips = {"127.0.0.1", "::1", "localhost"} + try: + output = subprocess.check_output( + ["ip", "-o", "addr", "show"], + text=True, + stderr=subprocess.DEVNULL, + ) + for line in output.splitlines(): + parts = line.split() + for token in parts: + if "/" not in token: + continue + address = token.split("/", 1)[0] + if address and (address[0].isdigit() or ":" in address): + ips.add(address) + except Exception: + pass + _LOCAL_IPS = ips + return ips + + +def defaultSeedEmulatorDockerDir() -> str: + """Return the best-known host path for SeedEMU base/router Dockerfiles.""" + candidates = [] + for base in (SETUP_DIR, *SETUP_DIR.parents): + candidates.append(base / "docker_images/multiarch") + candidates.extend( + [ + Path.home() / "seed-emulator/docker_images/multiarch", + Path.home() / "seed-emulator-k8s-new/docker_images/multiarch", + Path.home() / "k8s/seed-emulator/docker_images/multiarch", + ] + ) + for candidate in candidates: + if (candidate / "seedemu-base").is_dir() and (candidate / "seedemu-router").is_dir(): + return str(candidate.resolve()) + return str(candidates[0].expanduser().resolve()) + + +def getNested(data: dict[str, Any], path: str, default: Any = None) -> Any: + """Read a dotted YAML path, accepting both camelCase and snake_case keys.""" + cur: Any = data + for part in path.split("."): + if not isinstance(cur, dict): + return default + candidates = [part, _snakeCase(part), _camelCase(part)] + found = False + for candidate in candidates: + if candidate in cur: + cur = cur[candidate] + found = True + break + if not found: + return default + return cur + + +def loadYaml(path: str) -> dict[str, Any]: + """Load configK3s.yaml and validate that it is a mapping.""" + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise SystemExit(f"Invalid YAML root in {path}: expected mapping") + return data + + +def normalizeRole(name: str, role: str | None, index: int) -> str: + """Normalize user node roles; role must be explicitly provided.""" + if not role: + raise SystemExit(f"Node {name or index} requires role: master or worker") + if role in {"master", "control-plane", "server"}: + return "master" + if role in {"worker", "agent"}: + return "worker" + raise SystemExit(f"Unsupported role for node {name or index}: {role}") + + +def yamlNodes(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return normalized nodes from configK3s.yaml. + + Each node requires an IP address and should carry its own ssh.user/key. + Missing names become seed-k3s-master/seed-k3s-workerN. A top-level ssh + block is still accepted as a compatibility fallback, but generated configs + write SSH settings per node. + """ + raw_nodes = data.get("nodes") or [] + if not isinstance(raw_nodes, list): + raise SystemExit("configK3s.yaml field nodes must be a list") + out: list[dict[str, Any]] = [] + for index, item in enumerate(raw_nodes): + if not isinstance(item, dict): + raise SystemExit(f"Invalid node item: {item}") + ip = str(item.get("ip") or item.get("managementIp") or item.get("management_ip") or "") + if not ip: + raise SystemExit(f"Each k3s node requires an ip: {item}") + role = normalizeRole(str(item.get("name") or ""), str(item.get("role") or ""), index) + default_name = "seed-k3s-master" if role == "master" else f"seed-k3s-worker{sum(1 for node in out if node['role'] == 'worker') + 1}" + name = str(item.get("name") or default_name) + ssh_user, ssh_key = nodeSshSettings(data, item, name) + connection = nodeConnection(item, ip, ssh_user) + out.append( + { + "name": name, + "role": role, + "ip": ip, + "mac": str(item.get("mac") or ""), + "vcpus": int(item.get("vcpus") or 0), + "memoryMb": int(item.get("memoryMb") or item.get("memory_mb") or 0), + "diskGb": int(item.get("diskGb") or item.get("disk_gb") or 0), + "sshUser": ssh_user, + "sshKey": expandPath(ssh_key), + "connection": connection, + } + ) + validateNodes(out) + return out + + +def nodeSshSettings(data: dict[str, Any], item: dict[str, Any], node_name: str) -> tuple[str, str]: + """Return SSH user/key for one node. + + Args: + data: Full configK3s.yaml mapping. + item: One raw node mapping. + node_name: Normalized node name used for error messages. + """ + ssh = item.get("ssh") if isinstance(item.get("ssh"), dict) else {} + user = ssh.get("user") or getNested(data, "ssh.user") + key = ssh.get("key") or getNested(data, "ssh.key") + if not user or not key: + raise SystemExit( + f"Node {node_name} requires ssh.user and ssh.key. " + "Use nodes[].ssh.{user,key}; top-level ssh is accepted only as a fallback." + ) + return str(user), str(key) + + +def nodeConnection(item: dict[str, Any], ip: str, ssh_user: str) -> str: + """Return how setup scripts should reach one node. + + Args: + item: Raw node mapping from configK3s.yaml. + ip: Normalized management IP. + ssh_user: Normalized SSH user. + + YAML may explicitly set nodes[].connection to "local" or "ssh". If it is + omitted, this helper treats a node as local only when its IP is assigned to + the current host and the SSH user equals the current local user. + """ + raw = item.get("connection") or item.get("connect") + if raw: + normalized = str(raw).strip().lower() + if normalized in {"local", "localhost"}: + return "local" + if normalized in {"ssh", "remote"}: + return "ssh" + raise SystemExit(f"Unsupported node connection for {ip}: {raw}") + if item.get("local") is True: + return "local" + if ip in readLocalIps() and ssh_user == getpass.getuser(): + return "local" + return "ssh" + + +def validateNodes(nodes: list[dict[str, Any]]) -> None: + """Validate that the selected node set can form one K3s cluster.""" + if not nodes: + raise SystemExit("No nodes selected for K3s cluster") + masters = [node for node in nodes if node["role"] == "master"] + if len(masters) != 1: + names = ", ".join(node["name"] for node in masters) or "none" + raise SystemExit(f"Expected exactly one master node, got {len(masters)}: {names}") + seen_names: set[str] = set() + seen_ips: set[str] = set() + for node in nodes: + if node["name"] in seen_names: + raise SystemExit(f"Duplicate node name: {node['name']}") + if node["ip"] in seen_ips: + raise SystemExit(f"Duplicate node ip: {node['ip']}") + seen_names.add(node["name"]) + seen_ips.add(node["ip"]) + + +def installVersion(data: dict[str, Any]) -> str: + """Return the K3s install version expected by the configured artifact URL.""" + version = str(getNested(data, "k3s.version", "v1.28.5+k3s1")) + artifact = str(getNested(data, "k3s.artifactUrl", "https://rancher-mirror.rancher.cn/k3s")) + configured = getNested(data, "k3s.installVersion") + if configured: + return str(configured) + if "rancher-mirror.rancher.cn/k3s" in artifact: + return version.replace("+", "-") + return version + + +def masterNode(nodes: list[dict[str, Any]]) -> dict[str, Any]: + """Return the unique master node.""" + return [node for node in nodes if node["role"] == "master"][0] + + +def configValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, Any]: + """Build the local shell variable values consumed by applyK3sCluster.py.""" + cluster_name = str(data.get("clusterName") or data.get("cluster_name") or "seedemu-k3s") + master = masterNode(nodes) + registry_host = getNested(data, "registry.host", master["ip"]) + fabric_type = str(getNested(data, "fabric.type", "none")) + default_cni_master = getNested(data, "fabric.bridgeName", "br-seedemu") if fabric_type == "linux-vxlan" else "ens2" + return { + "clusterName": cluster_name, + "setupTmpDir": expandPath(str(getNested(data, "outputs.tmpDir", SETUP_DIR / "tmp"))), + "k3sUser": master["sshUser"], + "k3sSshKey": master["sshKey"], + "k3sMasterName": master["name"], + "k3sMasterIp": master["ip"], + "k3sMasterConnection": master["connection"], + "k3sVersion": getNested(data, "k3s.version", "v1.28.5+k3s1"), + "k3sInstallVersion": installVersion(data), + "k3sArtifactUrl": getNested(data, "k3s.artifactUrl", "https://rancher-mirror.rancher.cn/k3s"), + "k3sForceReinstall": str(getNested(data, "k3s.forceReinstall", True)).lower(), + "k3sClusterCidr": getNested(data, "k3s.clusterCidr", "10.42.0.0/16"), + "k3sServiceCidr": getNested(data, "k3s.serviceCidr", "10.43.0.0/16"), + "k3sFlannelBackend": getNested(data, "k3s.flannelBackend", "host-gw"), + "k3sNodeCidrMaskSizeIpv4": getNested(data, "k3s.nodeCidrMaskSizeIpv4", 20), + "k3sMaxPods": getNested(data, "k3s.maxPods", 4000), + "kubeletRegistryQps": getNested(data, "k3s.kubeletRegistryQps", 100), + "kubeletRegistryBurst": getNested(data, "k3s.kubeletRegistryBurst", 20), + "registryHost": registry_host, + "registryPort": getNested(data, "registry.port", 5000), + "fabricType": fabric_type, + "dockerIoMirrorEndpoint": getNested(data, "registry.dockerIoMirrorEndpoint", "https://docker.m.daocloud.io"), + "seedEmulatorDockerDir": expandPath(str(getNested(data, "seedemu.dockerImagesDir", defaultSeedEmulatorDockerDir()))), + "cniMasterInterface": getNested(data, "cni.defaultMasterInterface", default_cni_master), + "cni0HashMax": getNested(data, "tuning.cni0HashMax", 16384), + "userMaxNetNamespaces": getNested(data, "tuning.userMaxNetNamespaces", 65536), + "neighGcThresh1": getNested(data, "tuning.neighGcThresh1", 1048576), + "neighGcThresh2": getNested(data, "tuning.neighGcThresh2", 4194304), + "neighGcThresh3": getNested(data, "tuning.neighGcThresh3", 8388608), + "netdevMaxBacklog": getNested(data, "tuning.netdevMaxBacklog", 1000000), + "optmemMax": getNested(data, "tuning.optmemMax", 25165824), + "rebootAfterTuning": str(getNested(data, "tuning.rebootAfterTuning", False)).lower(), + "outputKubeconfig": expandPath(str(getNested(data, "outputs.kubeconfig", SETUP_DIR / f"{cluster_name}.kubeconfig.yaml"))), + "outputInventory": expandPath(str(getNested(data, "outputs.inventory", SETUP_DIR / f"{cluster_name}.inventory.yaml"))), + } + + +def fabricValues(data: dict[str, Any]) -> dict[str, Any]: + """Return normalized Linux fabric settings from configK3s.yaml. + + Args: + data: Parsed configK3s.yaml mapping. + + The first implementation intentionally supports a two-node Linux VXLAN + fabric. Larger physical fabrics should use a real L2/VLAN/EVPN/OVS design + instead of an implicit command-driven full mesh. + """ + values = { + "fabricType": str(getNested(data, "fabric.type", "none")), + "fabricBridgeName": str(getNested(data, "fabric.bridgeName", "br-seedemu")), + "fabricVxlanName": str(getNested(data, "fabric.vxlanName", "vxseed0")), + "fabricMacvlanTestName": str(getNested(data, "fabric.macvlanTestName", "macseed0")), + "fabricVni": int(getNested(data, "fabric.vni", 4242)), + "fabricDstPort": int(getNested(data, "fabric.dstPort", 4789)), + "fabricMtu": int(getNested(data, "fabric.mtu", 1450)), + } + validateInterfaceName(values["fabricBridgeName"], "fabric.bridgeName") + validateInterfaceName(values["fabricVxlanName"], "fabric.vxlanName") + validateInterfaceName(values["fabricMacvlanTestName"], "fabric.macvlanTestName") + return values + + +def ovnValues(data: dict[str, Any], nodes: list[dict[str, Any]]) -> dict[str, Any]: + """Return Kube-OVN non-primary CNI settings from configK3s.yaml. + + Args: + data: Parsed configK3s.yaml mapping. + nodes: Normalized node list used to discover the master IP. + + These values are consumed by ovn/installKubeOvnFabric.py. Kube-OVN runs as + a secondary CNI while K3s/flannel remains the primary eth0 network. + """ + vals = configValues(data, nodes) + return { + "ovnChartVersion": str(getNested(data, "ovn.chartVersion", "v1.15.12")), + "ovnHelmRepoName": str(getNested(data, "ovn.helmRepoName", "kubeovn")), + "ovnHelmRepoUrl": str(getNested(data, "ovn.helmRepoUrl", "https://kubeovn.github.io/kube-ovn/")), + "ovnReleaseName": str(getNested(data, "ovn.releaseName", "kube-ovn")), + "ovnNamespace": str(getNested(data, "ovn.namespace", "kube-system")), + "ovnTunnelType": str(getNested(data, "ovn.tunnelType", "geneve")), + "ovnIface": str(getNested(data, "ovn.iface", "")), + "ovnPodCidr": str(getNested(data, "ovn.podCidr", "172.28.0.0/16")), + "ovnPodGateway": str(getNested(data, "ovn.podGateway", "172.28.0.1")), + "ovnJoinCidr": str(getNested(data, "ovn.joinCidr", "100.64.0.0/16")), + "ovnServiceCidr": str(getNested(data, "ovn.serviceCidr", vals["k3sServiceCidr"])), + "ovnCniConfDir": str(getNested(data, "ovn.cniConfDir", "/var/lib/rancher/k3s/agent/etc/cni/net.d")), + "ovnMountCniConfDir": str(getNested(data, "ovn.mountCniConfDir", "/var/lib/rancher/k3s/agent/etc/cni/net.d")), + "ovnCniBinDir": str(getNested(data, "ovn.cniBinDir", "/var/lib/rancher/k3s/data/cni")), + "ovnHelmCacheDir": expandPath(str(getNested(data, "ovn.helmCacheDir", Path(vals["setupTmpDir"]) / "helm"))), + "ovnMasterNodes": str(getNested(data, "ovn.masterNodes", vals["k3sMasterIp"])), + } + + +def validateInterfaceName(name: str, field: str) -> None: + """Validate Linux interface-name length before ip-link commands run.""" + if not name: + raise SystemExit(f"{field} must not be empty") + if len(name) > 15: + raise SystemExit(f"{field}={name!r} is too long for Linux IFNAMSIZ; use at most 15 characters") + + +def sshOptions(key_path: str) -> list[str]: + """Return SSH options used for non-interactive node inspection. + + Args: + key_path: Private key path from nodes[].ssh.key. + """ + return [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "LogLevel=ERROR", + "-o", + "ConnectTimeout=10", + "-i", + key_path, + ] + + +def parseRouteInterface(output: str, node_name: str, peer_ip: str) -> str: + """Extract the outbound Linux interface from `ip -o route get` output. + + Args: + output: Text returned by `ip -o route get `. + node_name: Node name used for error messages. + peer_ip: Peer management IP used for error messages. + """ + match = re.search(r"(?:^|\s)dev\s+(\S+)", output) + if not match: + raise SystemExit( + f"Could not detect underlay interface for {node_name}; " + f"`ip -o route get {peer_ip}` returned: {output.strip()!r}. " + "Set fabric.nodes..underlayInterface explicitly." + ) + return match.group(1) + + +def detectUnderlayInterface(node: dict[str, Any], peer_ip: str) -> str: + """Detect the interface a node uses to reach its VXLAN peer. + + Args: + node: Normalized node mapping from yamlNodes(). + peer_ip: Management IP of the opposite VXLAN endpoint. + + For local nodes this runs `ip -o route get` directly. For remote nodes it + runs the same command through SSH using nodes[].ssh.{user,key}. This keeps + user YAML minimal while still allowing explicit underlayInterface override. + """ + command = f"ip -o route get {shlex.quote(peer_ip)}" + try: + if node["connection"] == "local": + output = subprocess.check_output( + ["bash", "-lc", command], + stdin=subprocess.DEVNULL, + text=True, + stderr=subprocess.STDOUT, + ) + else: + output = subprocess.check_output( + [ + "ssh", + *sshOptions(node["sshKey"]), + f"{node['sshUser']}@{node['ip']}", + command, + ], + stdin=subprocess.DEVNULL, + text=True, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as exc: + raise SystemExit( + f"Could not detect underlay interface for {node['name']} via route to {peer_ip}. " + f"Command output: {(exc.output or '').strip()!r}. " + "Set fabric.nodes..underlayInterface explicitly." + ) from exc + underlay = parseRouteInterface(output, node["name"], peer_ip) + validateInterfaceName(underlay, f"fabric.nodes.{node['name']}.underlayInterface") + return underlay + + +def fabricNodeRows(data: dict[str, Any], nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return per-node rows consumed by VXLAN fabric shell scripts. + + Args: + data: Parsed configK3s.yaml mapping. + nodes: Normalized K3s node list. + + The user may set fabric.nodes..underlayInterface explicitly. If it is + omitted, the helper detects the underlay by asking the node which interface + it uses to route to the peer management IP. Test IPs are internal defaults + and are only used by validateLinuxVxlanFabric.py. + """ + values = fabricValues(data) + if values["fabricType"] == "none": + return [] + if values["fabricType"] != "linux-vxlan": + return [] + if len(nodes) != 2: + raise SystemExit("fabric.type=linux-vxlan currently supports exactly two nodes") + rows: list[dict[str, Any]] = [] + fabric_nodes = getNested(data, "fabric.nodes", {}) + if fabric_nodes is None: + fabric_nodes = {} + if not isinstance(fabric_nodes, dict): + raise SystemExit("fabric.nodes must be a mapping keyed by node name or node IP") + defaults = [ + ("172.31.252.1/30", "172.31.253.1/30"), + ("172.31.252.2/30", "172.31.253.2/30"), + ] + for index, node in enumerate(nodes): + node_cfg = fabric_nodes.get(node["name"]) or fabric_nodes.get(node["ip"]) or {} + if not isinstance(node_cfg, dict): + raise SystemExit(f"fabric.nodes.{node['name']} must be a mapping") + peer = nodes[1 - index] + underlay = str(node_cfg.get("underlayInterface") or node_cfg.get("underlay_interface") or "") + if not underlay: + underlay = detectUnderlayInterface(node, peer["ip"]) + validateInterfaceName(underlay, f"fabric.nodes.{node['name']}.underlayInterface") + rows.append( + { + "name": node["name"], + "role": node["role"], + "ip": node["ip"], + "connection": node["connection"], + "sshUser": node["sshUser"], + "sshKey": node["sshKey"], + "underlayInterface": underlay, + "bridgeTestIp": str(node_cfg.get("bridgeTestIp") or node_cfg.get("bridge_test_ip") or defaults[index][0]), + "macvlanTestIp": str(node_cfg.get("macvlanTestIp") or node_cfg.get("macvlan_test_ip") or defaults[index][1]), + "peerName": peer["name"], + "peerIp": peer["ip"], + } + ) + return rows + + +def commandFabricShellVars(args: argparse.Namespace) -> None: + """Print shell assignments for the optional physical L2 fabric.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + values = fabricValues(data) + rows = fabricNodeRows(data, nodes) + values["fabricNodeCount"] = len(rows) + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def commandOvnShellVars(args: argparse.Namespace) -> None: + """Print shell assignments for Kube-OVN non-primary CNI install scripts.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + values = ovnValues(data, nodes) + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def commandFabricNodesTsv(args: argparse.Namespace) -> None: + """Print per-node fabric rows as TSV for shell scripts.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + for row in fabricNodeRows(data, nodes): + print( + "\t".join( + str(row[key]) + for key in ( + "name", + "role", + "ip", + "connection", + "sshUser", + "sshKey", + "underlayInterface", + "bridgeTestIp", + "macvlanTestIp", + "peerName", + "peerIp", + ) + ) + ) + + +def commandShellVars(args: argparse.Namespace) -> None: + """Print local shell assignments derived from configK3s.yaml.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + for key, value in configValues(data, nodes).items(): + print(f"{key}={shlex.quote(str(value))}") + + +def commandNodesTsv(args: argparse.Namespace) -> None: + """Print normalized node rows as TSV.""" + for node in yamlNodes(loadYaml(args.config)): + print( + "\t".join( + str(node[key]) + for key in ("name", "role", "ip", "mac", "vcpus", "memoryMb", "diskGb") + ) + ) + + +def commandNodeSshVars(args: argparse.Namespace) -> None: + """Print shell assignments for one node's SSH settings.""" + nodes = yamlNodes(loadYaml(args.config)) + selected = next((node for node in nodes if node["name"] == args.name), None) + if selected is None: + raise SystemExit(f"Node not found in configK3s.yaml: {args.name}") + print(f"nodeSshUser={shlex.quote(str(selected['sshUser']))}") + print(f"nodeSshKey={shlex.quote(str(selected['sshKey']))}") + print(f"nodeConnection={shlex.quote(str(selected['connection']))}") + + +def ansibleHostVars(node: dict[str, Any], k3s_role: str, as_group: str) -> dict[str, Any]: + """Return Ansible host variables for one K3s node. + + Args: + node: Normalized node dictionary from yamlNodes(). + k3s_role: K3s role string consumed by ansible/k3s-install.yml. + as_group: Human-readable grouping label. + """ + payload: dict[str, Any] = {"k3s_role": k3s_role, "seedemu_as_group": as_group} + if node["connection"] == "local": + payload.update( + { + "ansible_host": node["ip"], + "ansible_connection": "local", + "ansible_python_interpreter": "/usr/bin/python3", + } + ) + else: + payload.update( + { + "ansible_host": node["ip"], + "ansible_user": node["sshUser"], + "ansible_ssh_private_key_file": node["sshKey"], + } + ) + return payload + + +def commandWriteAnsibleInventory(args: argparse.Namespace) -> None: + """Write the temporary Ansible inventory used by applyK3sCluster.py.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + vals = configValues(data, nodes) + master = masterNode(nodes) + workers = [node for node in nodes if node["role"] == "worker"] + payload = { + "all": { + "vars": { + "k3s_version": vals["k3sVersion"], + "k3s_install_version": vals["k3sInstallVersion"], + "seed_registry_host": vals["registryHost"], + "seed_registry_port": int(vals["registryPort"]), + "seed_docker_io_mirror_endpoint": vals["dockerIoMirrorEndpoint"], + "seed_k3s_artifact_url": vals["k3sArtifactUrl"], + "seed_cni_master_interface": vals["cniMasterInterface"], + "seed_k3s_cluster_cidr": vals["k3sClusterCidr"], + "seed_k3s_service_cidr": vals["k3sServiceCidr"], + "seed_k3s_flannel_backend": vals["k3sFlannelBackend"], + "seed_k3s_node_cidr_mask_size_ipv4": int(vals["k3sNodeCidrMaskSizeIpv4"]), + "seed_k3s_max_pods": int(vals["k3sMaxPods"]), + "seed_k3s_expected_ready_nodes": len(nodes), + "seed_k3s_force_reinstall": str(vals["k3sForceReinstall"]).lower() == "true", + }, + "children": { + "master": { + "hosts": { + master["name"]: ansibleHostVars(master, "server", "master") + } + }, + "workers": { + "hosts": { + node["name"]: ansibleHostVars(node, "agent", f"worker-{index}") + for index, node in enumerate(workers, start=1) + } + }, + }, + } + } + _writeYaml(args.output, payload) + print(args.output) + + +def commandWriteClusterInventory(args: argparse.Namespace) -> None: + """Write a persistent human-readable cluster inventory YAML.""" + data = loadYaml(args.config) + nodes = yamlNodes(data) + vals = configValues(data, nodes) + payload = { + "clusterName": vals["clusterName"], + "runtime": "k3s", + "k3s": { + "clusterCidr": vals["k3sClusterCidr"], + "serviceCidr": vals["k3sServiceCidr"], + "flannelBackend": vals["k3sFlannelBackend"], + "nodeCidrMaskSizeIpv4": int(vals["k3sNodeCidrMaskSizeIpv4"]), + "maxPods": int(vals["k3sMaxPods"]), + }, + "registry": {"host": vals["registryHost"], "port": int(vals["registryPort"])}, + "nodes": [ + { + "name": node["name"], + "role": node["role"], + "managementIp": node["ip"], + "connection": node["connection"], + "ssh": {"user": node["sshUser"], "key": node["sshKey"]}, + "resources": { + "vcpus": node["vcpus"], + "memoryMb": node["memoryMb"], + "diskGb": node["diskGb"], + }, + "labels": {"kubernetes.io/hostname": node["name"]}, + } + for node in nodes + ], + } + _writeYaml(vals["outputInventory"], payload) + print(vals["outputInventory"]) + + +def commandWriteRunningConfig(args: argparse.Namespace) -> None: + """Write legacy configRunning.yaml for running/Makefile. + + Args: + args.config: Source configK3s.yaml path. + args.output_dir: Optional compile output directory. + args.image_registry_prefix: Logical image prefix in k8s.yaml. + args.rollout_timeout_seconds: Rollout wait timeout for make up/wait. + """ + data = loadYaml(args.config) + output_dir = args.output_dir or str(SETUP_DIR.parent / "emulate" / "output") + output_path = expandPath(str(getNested(data, "outputs.runningConfig", SETUP_DIR / "configRunning.yaml"))) + payload = { + "setupConfig": str(Path(args.config).expanduser().resolve()), + "outputDir": expandPath(output_dir), + "imageRegistryPrefix": args.image_registry_prefix, + "rolloutTimeoutSeconds": int(args.rollout_timeout_seconds), + } + _writeYaml(output_path, payload) + print(output_path) + + +def _writeYaml(path: str, payload: dict[str, Any]) -> None: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + +def _snakeCase(value: str) -> str: + return re.sub(r"(? str: + parts = value.split("_") + return parts[0] + "".join(part[:1].upper() + part[1:] for part in parts[1:]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, help="configK3s.yaml path") + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("shell-vars").set_defaults(func=commandShellVars) + sub.add_parser("nodes-tsv").set_defaults(func=commandNodesTsv) + node_ssh = sub.add_parser("node-ssh-vars") + node_ssh.add_argument("--name", required=True) + node_ssh.set_defaults(func=commandNodeSshVars) + + sub.add_parser("fabric-shell-vars").set_defaults(func=commandFabricShellVars) + sub.add_parser("fabric-nodes-tsv").set_defaults(func=commandFabricNodesTsv) + sub.add_parser("ovn-shell-vars").set_defaults(func=commandOvnShellVars) + + ansible = sub.add_parser("write-ansible-inventory") + ansible.add_argument("--output", required=True) + ansible.set_defaults(func=commandWriteAnsibleInventory) + + cluster = sub.add_parser("write-cluster-inventory") + cluster.set_defaults(func=commandWriteClusterInventory) + + running = sub.add_parser("write-running-config") + running.add_argument("--output-dir") + running.add_argument("--image-registry-prefix", default="seedemu") + running.add_argument("--rollout-timeout-seconds", default="1800") + running.set_defaults(func=commandWriteRunningConfig) + + args = parser.parse_args() + args.func(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py new file mode 100755 index 000000000..c9b5d51e0 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/createMultiHostKvmVms.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Python entrypoint for createMultiHostKvmVms with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create KVM VMs across multiple physical hypervisors. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Outputs: +# - Host-local kvm.yaml files under setup/tmp/multiHostKvm/. +# - Remote host-local setup directories under hypervisors[].remoteWorkDir. +# - Global configK3s.yaml for applyK3sCluster.py. +# - Global multiHostKvmState.yaml for destroyMultiHostKvmVms.py. +# +# Execution context: +# Run after prepareKvmHypervisors.py. This script creates VMs on the target +# hypervisors but does not install K3s. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +LOCAL_WORK_DIR="" + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +copyVmSshKeyToHost() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key. + # + # Remote hypervisors run kvm/createKvmVms.py locally, so they need a local + # copy of the VM SSH private key to inject its public key into cloud-init + # and wait for SSH readiness. The global configK3s.yaml still keeps the + # original control-host key path for later K3s installation. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local source_key target_key source_pub target_pub + + IFS=$'\t' read -r source_key target_key source_pub target_pub < <( + python3 "${HELPER}" --config "${CONFIG_PATH}" host-vm-ssh-key-tsv --host "${name}" + ) + [ -f "${source_key}" ] || { + echo "VM SSH key not found on control host: ${source_key}" >&2 + exit 1 + } + if [ "${connection}" = "local" ]; then + return + fi + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "$(dirname "${target_key}")")" + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_key}" "${ssh_user}@${ip}:${target_key}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 600 $(printf '%q' "${target_key}")" + if [ -f "${source_pub}" ]; then + # shellcheck disable=SC2046 + scp $(sshOptions "${ssh_key}") "${source_pub}" "${ssh_user}@${ip}:${target_pub}" >/dev/null + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "chmod 644 $(printf '%q' "${target_pub}")" + fi +} + +prepareLocalWorkDir() { + mkdir -p "${outputTmpDir}" + LOCAL_WORK_DIR="${outputTmpDir}/multiHostKvm" + rm -rf "${LOCAL_WORK_DIR}" + mkdir -p "${LOCAL_WORK_DIR}" +} + +createHostVms() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local remote_work_dir="${10}" + local host_dir="${LOCAL_WORK_DIR}/${name}" + local host_kvm_yaml="${host_dir}/kvm.yaml" + + echo "[multi-host-create] ${name} (${ip})" + mkdir -p "${host_dir}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-local-kvm \ + --host "${name}" \ + --output "${host_kvm_yaml}" >/dev/null + + syncDirToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${SETUP_DIR}/kvm" "${remote_work_dir}/kvm" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${host_kvm_yaml}" "${remote_work_dir}/kvm.yaml" + copyVmSshKeyToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + cd $(printf '%q' "${remote_work_dir}") + python3 ./kvm/prepareHostAssets.py ./kvm.yaml + python3 ./kvm/createKvmVms.py ./kvm.yaml + " +} + +writeGlobalOutputs() { + python3 "${HELPER}" --config "${CONFIG_PATH}" write-global-k3s-config \ + --output "${outputConfigK3s}" >/dev/null + python3 "${HELPER}" --config "${CONFIG_PATH}" write-state \ + --output "${outputMultiHostKvmState}" >/dev/null + echo "K3s config: ${outputConfigK3s}" + echo "Multi-host KVM state: ${outputMultiHostKvmState}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand rsync + requireCommand cp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + prepareLocalWorkDir + python3 "${HELPER}" --config "${CONFIG_PATH}" validate + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + createHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + writeGlobalOutputs + echo "Multi-host KVM VMs are ready." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py new file mode 100755 index 000000000..9437d7b16 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/destroyMultiHostKvmVms.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Python entrypoint for destroyMultiHostKvmVms with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Destroy KVM VMs and routed networks created by the multi-host KVM flow. +# +# Inputs: +# $1: multiHostKvmState.yaml. Defaults to ../multiHostKvmState.yaml. +# +# Side effects: +# - Runs each host-local kvm/destroyKvmVms.py on its owning hypervisor. +# - Removes static routes between hypervisor VM subnets. +# - Removes optional point-to-point VXLAN route tunnels. +# - Destroys and undefines the generated libvirt routed networks. +# - Removes generated local configK3s/kubeconfig/inventory/state outputs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +STATE_PATH="${1:-${SETUP_DIR}/multiHostKvmState.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" &2; sleep 2; python3 ./kvm/destroyKvmVms.py ./kvmState.yaml; } + fi + virsh net-destroy $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-undefine $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -t nat -D POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + sudo -n iptables -D FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || true + fi + rm -rf $(printf '%q' "${remote_work_dir}") + " + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + echo " delete route ${remote_cidr} via ${peer_ip} (${peer_name})" + if [ -n "${route_dev}" ]; then + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") dev $(printf '%q' "${route_dev}") >/dev/null 2>&1 || sudo -n ip route del $(printf '%q' "${remote_cidr}") >/dev/null 2>&1 || true" + else + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route del $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") >/dev/null 2>&1 || true" + fi + done < <(python3 "${HELPER}" --config /dev/null state-routes-tsv --state "${STATE_PATH}" --host "${name}") + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " delete tunnel ${tunnel_name} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true" + done < <(python3 "${HELPER}" --config /dev/null state-tunnels-tsv --state "${STATE_PATH}" --host "${name}") +} + +cleanupLocalOutputs() { + eval "$(python3 "${HELPER}" --config /dev/null state-output-vars --state "${STATE_PATH}")" + rm -f "${stateConfigK3s}" + rm -f "${stateKubeconfig}" + rm -f "${stateInventory}" + rm -f "${STATE_PATH}" +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + + [ -s "${STATE_PATH}" ] || { + echo "Missing state file: ${STATE_PATH}" >&2 + exit 1 + } + + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + destroyHostVms "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config /dev/null state-hosts-tsv --state "${STATE_PATH}") + cleanupLocalOutputs + echo "Multi-host KVM cleanup completed." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py new file mode 100755 index 000000000..2bd7bb324 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/manageMultiHostKvmConfig.py @@ -0,0 +1,833 @@ +#!/usr/bin/env python3 +"""Parse and render multi-hypervisor KVM plans for k8sTools. + +The multi-host flow treats physical machines as KVM hypervisors. Each +hypervisor owns a routed libvirt subnet, while the generated VMs form one K3s +cluster. This helper converts the global kvm.yaml into host-local kvm.yaml +files and the global configK3s.yaml consumed by the K3s setup entrypoint. +""" + +from __future__ import annotations + +import argparse +import copy +import ipaddress +import os +import re +import shlex +from pathlib import Path +from typing import Any + +import yaml + + +SCRIPT_DIR = Path(__file__).resolve().parent +SETUP_DIR = SCRIPT_DIR.parent +CONFIG_DIR_KEY = "__configDir" + + +def loadYaml(path: str | Path) -> dict[str, Any]: + """Load one YAML mapping. + + Args: + path: YAML file path. + """ + with Path(path).expanduser().open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise SystemExit(f"Invalid YAML root in {path}: expected a mapping") + data[CONFIG_DIR_KEY] = str(Path(path).expanduser().resolve().parent) + normalizeConfig(data) + return data + + +def writeYaml(path: str | Path, data: dict[str, Any]) -> None: + """Write one YAML mapping. + + Args: + path: Output YAML path. + data: Mapping to serialize. + """ + output = Path(path).expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + +def normalizeConfig(data: dict[str, Any]) -> None: + """Normalize legacy snake_case keys while keeping YAML user-facing.""" + if "cluster_name" in data and "clusterName" not in data: + data["clusterName"] = data.pop("cluster_name") + for section in ("master", "workers"): + value = data.get(section) + if isinstance(value, dict): + for old, new in {"memory_mb": "memoryMb", "disk_gb": "diskGb", "name_prefix": "namePrefix"}.items(): + if old in value and new not in value: + value[new] = value.pop(old) + for host in data.get("hypervisors") or []: + if not isinstance(host, dict): + continue + if "vm_count" in host and "vmCount" not in host: + host["vmCount"] = host.pop("vm_count") + if "remote_work_dir" in host and "remoteWorkDir" not in host: + host["remoteWorkDir"] = host.pop("remote_work_dir") + routed = host.get("routedSubnet") + if not isinstance(routed, dict) and isinstance(host.get("routed_subnet"), dict): + routed = host["routed_subnet"] + host["routedSubnet"] = routed + if isinstance(routed, dict): + for old, new in { + "bridge_name": "bridgeName", + "network_name": "networkName", + "vm_ip_start": "vmIpStart", + }.items(): + if old in routed and new not in routed: + routed[new] = routed.pop(old) + + +def getNested(data: dict[str, Any], path: str, default: Any = None) -> Any: + """Read a nested mapping field using dotted path syntax. + + Args: + data: Source mapping. + path: Dotted path such as outputs.tmpDir. + default: Value returned when the field does not exist. + """ + cur: Any = data + for part in path.split("."): + if not isinstance(cur, dict): + return default + for candidate in (part, snakeCase(part), camelCase(part)): + if candidate in cur: + cur = cur[candidate] + break + else: + return default + return cur + + +def snakeCase(value: str) -> str: + """Convert one YAML key segment to snake_case.""" + return re.sub(r"(? str: + """Convert one YAML key segment to lower camelCase.""" + parts = value.split("_") + return parts[0] + "".join(part[:1].upper() + part[1:] for part in parts[1:]) + + +def expandPath(value: str) -> str: + """Expand local user/environment markers into an absolute path string.""" + return str(Path(os.path.expandvars(os.path.expanduser(value))).resolve()) + + +def clusterName(data: dict[str, Any]) -> str: + """Return the effective cluster name.""" + return str(data.get("clusterName") or "seedemu-k3s") + + +def outputPath(data: dict[str, Any], key: str, fallback: Path) -> str: + """Return a local setup output path. + + Args: + data: Parsed global kvm.yaml. + key: outputs.* key to inspect. + fallback: Default path when the key is absent. + """ + configured = getNested(data, f"outputs.{key}") + if configured: + candidate = Path(os.path.expandvars(os.path.expanduser(str(configured)))) + if not candidate.is_absolute(): + candidate = Path(str(data.get(CONFIG_DIR_KEY) or SETUP_DIR)) / candidate + return str(candidate.resolve()) + return str(fallback.resolve()) + + +def hostList(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return validated hypervisor records. + + Args: + data: Parsed global kvm.yaml. + """ + raw_hosts = data.get("hypervisors") + if not isinstance(raw_hosts, list) or not raw_hosts: + raise SystemExit("kvm.yaml requires a non-empty hypervisors list") + + names: set[str] = set() + networks: list[tuple[str, ipaddress.IPv4Network]] = [] + hosts: list[dict[str, Any]] = [] + for raw in raw_hosts: + if not isinstance(raw, dict): + raise SystemExit(f"hypervisors item must be a mapping: {raw}") + name = str(raw.get("name") or "").strip() + ip = str(raw.get("ip") or "").strip() + if not name or not ip: + raise SystemExit(f"hypervisor requires name and ip: {raw}") + if name in names: + raise SystemExit(f"duplicate hypervisor name: {name}") + names.add(name) + + routed = raw.get("routedSubnet") + if not isinstance(routed, dict): + raise SystemExit(f"hypervisor {name} requires routedSubnet") + cidr = str(routed.get("cidr") or "").strip() + gateway = str(routed.get("gateway") or "").strip() + if not cidr or not gateway: + raise SystemExit(f"hypervisor {name} routedSubnet requires cidr and gateway") + network = ipaddress.ip_network(cidr, strict=False) + gateway_ip = ipaddress.ip_address(gateway) + if gateway_ip not in network: + raise SystemExit(f"hypervisor {name} gateway {gateway} is outside {cidr}") + for other_name, other_network in networks: + if network.overlaps(other_network): + raise SystemExit(f"routedSubnet overlap: {name} {network} overlaps {other_name} {other_network}") + networks.append((name, network)) + + ssh_cfg = raw.get("ssh") if isinstance(raw.get("ssh"), dict) else {} + hosts.append( + { + "name": name, + "ip": ip, + "connection": str(raw.get("connection") or "ssh").strip().lower(), + "sshUser": str(ssh_cfg.get("user") or getNested(data, "hypervisorSsh.user", "")), + "sshKey": expandPath(str(ssh_cfg.get("key") or getNested(data, "hypervisorSsh.key", "~/.ssh/id_ed25519"))), + "vmCount": int(raw.get("vmCount") or 0), + "networkName": str(routed.get("networkName") or f"seedemu-{name}"), + "bridgeName": str(routed.get("bridgeName") or f"virbr-{name[:8]}"), + "cidr": str(network), + "gateway": str(gateway_ip), + "netmask": str(network.netmask), + "dhcpStart": str(firstUsableIp(network, gateway_ip)), + "dhcpEnd": str(lastUsableIp(network, gateway_ip)), + "vmIpStart": int(routed.get("vmIpStart") or 10), + "remoteWorkDir": str( + raw.get("remoteWorkDir") + or getNested(data, "multiHostKvm.remoteWorkDir") + or f"/tmp/seedemu-k8s-tools/{clusterName(data)}/{name}" + ), + } + ) + + if any(host["connection"] != "local" and not host["sshUser"] for host in hosts): + raise SystemExit("non-local hypervisors require ssh.user") + if sum(host["vmCount"] for host in hosts) <= 0: + raise SystemExit("at least one hypervisor vmCount must be > 0") + return hosts + + +def routingTunnelConfig(data: dict[str, Any]) -> dict[str, Any]: + """Return optional hypervisor-to-hypervisor routing tunnel settings. + + Args: + data: Parsed global multi-host KVM YAML. + + The routed libvirt subnet model needs a valid next-hop between physical + hypervisors. When the physical machines are only connected by routed + underlay networks, the peer physical IP is not a valid L2 next-hop. This + optional VXLAN tunnel creates a small point-to-point L3 next-hop per + hypervisor pair. + """ + raw = getNested(data, "multiHostKvm.routingTunnel") + if not isinstance(raw, dict): + return {"enabled": False, "type": "none"} + tunnel_type = str(raw.get("type") or "none").strip().lower() + enabled = bool(raw.get("enabled", tunnel_type in {"vxlan", "linux-vxlan", "linux_vxlan"})) + if not enabled or tunnel_type in {"none", "disabled", "false"}: + return {"enabled": False, "type": "none"} + if tunnel_type not in {"vxlan", "linux-vxlan", "linux_vxlan"}: + raise SystemExit(f"unsupported multiHostKvm.routingTunnel.type: {tunnel_type}") + cidr = str(raw.get("cidr") or "10.255.80.0/24") + network = ipaddress.ip_network(cidr, strict=False) + if network.version != 4 or network.num_addresses < 4: + raise SystemExit(f"routingTunnel.cidr must be an IPv4 network with at least four addresses: {cidr}") + return { + "enabled": True, + "type": "vxlan", + "cidr": str(network), + "namePrefix": str(raw.get("namePrefix") or "vxkvm"), + "vniBase": int(raw.get("vniBase") or 4280), + "dstPort": int(raw.get("dstPort") or 4790), + } + + +def tunnelRows(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return one VXLAN tunnel route row per ordered hypervisor pair. + + Args: + data: Parsed global multi-host KVM YAML. + """ + config = routingTunnelConfig(data) + if not config["enabled"]: + return [] + hosts = hostList(data) + rows: list[dict[str, Any]] = [] + network = ipaddress.ip_network(str(config["cidr"]), strict=False) + pair_index = 0 + for left_index, left in enumerate(hosts): + for right in hosts[left_index + 1 :]: + base_ip = int(network.network_address) + 4 * pair_index + local_a = ipaddress.ip_address(base_ip + 1) + local_b = ipaddress.ip_address(base_ip + 2) + if local_b >= network.broadcast_address: + raise SystemExit(f"routingTunnel.cidr {network} is too small for all hypervisor pairs") + iface = f"{config['namePrefix']}{pair_index}" + if len(iface) > 15: + raise SystemExit(f"routing tunnel interface name too long: {iface}") + vni = int(config["vniBase"]) + pair_index + common = {"interface": iface, "vni": vni, "dstPort": int(config["dstPort"])} + rows.append( + { + **common, + "host": left["name"], + "peerName": right["name"], + "localPhysicalIp": left["ip"], + "peerPhysicalIp": right["ip"], + "localTunnelCidr": f"{local_a}/30", + "peerTunnelIp": str(local_b), + "remoteCidr": right["cidr"], + } + ) + rows.append( + { + **common, + "host": right["name"], + "peerName": left["name"], + "localPhysicalIp": right["ip"], + "peerPhysicalIp": left["ip"], + "localTunnelCidr": f"{local_b}/30", + "peerTunnelIp": str(local_a), + "remoteCidr": left["cidr"], + } + ) + pair_index += 1 + return rows + + +def firstUsableIp(network: ipaddress.IPv4Network, gateway: ipaddress.IPv4Address) -> ipaddress.IPv4Address: + """Return the first usable DHCP range IP for a routed subnet.""" + for candidate in network.hosts(): + if candidate != gateway: + return candidate + raise SystemExit(f"network {network} has no usable host IP") + + +def lastUsableIp(network: ipaddress.IPv4Network, gateway: ipaddress.IPv4Address) -> ipaddress.IPv4Address: + """Return the last usable DHCP range IP for a routed subnet.""" + last = None + for candidate in network.hosts(): + if candidate != gateway: + last = candidate + if last is None: + raise SystemExit(f"network {network} has no usable host IP") + return last + + +def requireHost(hosts: list[dict[str, Any]], name: str) -> dict[str, Any]: + """Return a hypervisor by name.""" + for host in hosts: + if host["name"] == name: + return host + raise SystemExit(f"unknown hypervisor: {name}") + + +def allocateHostIps(host: dict[str, Any], count: int) -> list[str]: + """Allocate VM IP addresses inside one hypervisor subnet.""" + network = ipaddress.ip_network(host["cidr"], strict=False) + gateway = ipaddress.ip_address(host["gateway"]) + current = int(network.network_address) + int(host["vmIpStart"]) + end = int(network.broadcast_address) - 1 + result: list[str] = [] + while current <= end and len(result) < count: + candidate = ipaddress.ip_address(current) + current += 1 + if candidate != gateway and candidate in network: + result.append(str(candidate)) + if len(result) != count: + raise SystemExit(f"not enough VM IPs in {host['cidr']} for host {host['name']}") + return result + + +def allocateMac(index: int, data: dict[str, Any]) -> str: + """Allocate one deterministic MAC address for the global VM plan.""" + prefix = str(getNested(data, "multiHostKvm.macPrefix", "52:54:00:65:10")).lower() + start = int(getNested(data, "multiHostKvm.macStart", 0x10)) + if not re.fullmatch(r"[0-9a-f]{2}(:[0-9a-f]{2}){4}", prefix): + raise SystemExit(f"multiHostKvm.macPrefix must contain five MAC octets: {prefix}") + suffix = start + index + if suffix > 255: + raise SystemExit("multiHostKvm MAC allocation exhausted one-byte suffix space") + return f"{prefix}:{suffix:02x}" + + +def vmPlan(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return the full VM plan across all hypervisors.""" + hosts = hostList(data) + master_cfg = data.get("master") if isinstance(data.get("master"), dict) else {} + workers_cfg = data.get("workers") if isinstance(data.get("workers"), dict) else {} + master_host_name = str(master_cfg.get("placement") or hosts[0]["name"]) + requireHost(hosts, master_host_name) + master_name = str(master_cfg.get("name") or "seed-k3s-master") + worker_prefix = str(workers_cfg.get("namePrefix") or "seed-k3s-worker") + master_resources = { + "vcpus": int(master_cfg.get("vcpus") or 16), + "memory_mb": int(master_cfg.get("memoryMb") or 32768), + "disk_gb": int(master_cfg.get("diskGb") or 120), + } + worker_resources = { + "vcpus": int(workers_cfg.get("vcpus") or 8), + "memory_mb": int(workers_cfg.get("memoryMb") or 16384), + "disk_gb": int(workers_cfg.get("diskGb") or 80), + } + + nodes: list[dict[str, Any]] = [] + worker_number = 1 + master_created = False + global_index = 0 + for host in hosts: + for host_index, vm_ip in enumerate(allocateHostIps(host, host["vmCount"]), start=1): + if host["name"] == master_host_name and not master_created: + role = "master" + name = master_name + resources = master_resources + master_created = True + else: + role = "worker" + name = f"{worker_prefix}{worker_number}" + resources = worker_resources + worker_number += 1 + nodes.append( + { + "name": name, + "role": role, + "ip": vm_ip, + "mac": allocateMac(global_index, data), + "vcpus": resources["vcpus"], + "memory_mb": resources["memory_mb"], + "disk_gb": resources["disk_gb"], + "hypervisor": host["name"], + "hypervisorIp": host["ip"], + "hostIndex": host_index, + } + ) + global_index += 1 + if not master_created: + raise SystemExit(f"master placement host {master_host_name} has vmCount=0") + validateVmPlan(nodes) + return nodes + + +def validateVmPlan(nodes: list[dict[str, Any]]) -> None: + """Validate generated VM names, IPs, MACs and master cardinality.""" + masters = [node for node in nodes if node["role"] == "master"] + if len(masters) != 1: + raise SystemExit(f"expected exactly one master VM, got {len(masters)}") + for key in ("name", "ip", "mac"): + values = [str(node[key]) for node in nodes] + duplicates = sorted({value for value in values if values.count(value) > 1}) + if duplicates: + raise SystemExit(f"duplicate VM {key}: {duplicates}") + + +def passThroughSections(data: dict[str, Any]) -> dict[str, Any]: + """Return K3s-stage sections copied into global configK3s.yaml.""" + payload: dict[str, Any] = {} + for section in ("registry", "k3s", "fabric", "ovn", "cni", "tuning", "seedemu"): + value = data.get(section) + if isinstance(value, dict) and value: + payload[section] = copy.deepcopy(value) + if str(getNested(payload, "fabric.type", "")).lower() in {"ovn", "kube-ovn", "kube_ovn"}: + k3s = payload.setdefault("k3s", {}) + if isinstance(k3s, dict): + k3s.setdefault("version", "v1.29.15+k3s1") + cluster = clusterName(data) + payload["outputs"] = { + "tmpDir": outputPath(data, "tmpDir", SETUP_DIR / "tmp"), + "kubeconfig": outputPath(data, "kubeconfig", SETUP_DIR / f"{cluster}.kubeconfig.yaml"), + "inventory": outputPath(data, "inventory", SETUP_DIR / f"{cluster}.inventory.yaml"), + } + return payload + + +def hostNodes(data: dict[str, Any], host_name: str) -> list[dict[str, Any]]: + """Return planned VMs assigned to one hypervisor.""" + return [node for node in vmPlan(data) if node["hypervisor"] == host_name] + + +def printShellVars(args: argparse.Namespace) -> None: + """Print global setup output paths as shell assignments.""" + data = loadYaml(args.config) + values = { + "outputConfigK3s": outputPath(data, "k3sConfig", SETUP_DIR / "configK3s.yaml"), + "outputMultiHostKvmState": outputPath(data, "multiHostKvmState", SETUP_DIR / "multiHostKvmState.yaml"), + "outputTmpDir": outputPath(data, "tmpDir", SETUP_DIR / "tmp"), + } + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def printHostsTsv(args: argparse.Namespace) -> None: + """Print hypervisor records consumed by shell scripts.""" + for host in hostList(loadYaml(args.config)): + print( + "\t".join( + str(host[key]) + for key in ( + "name", + "ip", + "connection", + "sshUser", + "sshKey", + "networkName", + "bridgeName", + "cidr", + "gateway", + "remoteWorkDir", + ) + ) + ) + + +def printHostRoutesTsv(args: argparse.Namespace) -> None: + """Print static routes needed by one hypervisor.""" + data = loadYaml(args.config) + tunnel_by_peer = {row["peerName"]: row for row in tunnelRows(data) if row["host"] == args.host} + for host in hostList(data): + if host["name"] == args.host: + continue + tunnel = tunnel_by_peer.get(host["name"]) + if tunnel: + print("\t".join([host["cidr"], tunnel["peerTunnelIp"], host["name"], tunnel["interface"]])) + else: + print("\t".join([host["cidr"], host["ip"], host["name"], ""])) + + +def printHostTunnelsTsv(args: argparse.Namespace) -> None: + """Print VXLAN tunnel rows needed by one hypervisor.""" + for row in tunnelRows(loadYaml(args.config)): + if row["host"] != args.host: + continue + print( + "\t".join( + str(row[key]) + for key in ( + "interface", + "vni", + "dstPort", + "localPhysicalIp", + "peerPhysicalIp", + "localTunnelCidr", + "peerTunnelIp", + "peerName", + ) + ) + ) + + +def hostVmSshKeyPaths(data: dict[str, Any], host_name: str) -> tuple[str, str]: + """Return local source key and hypervisor-local target key paths. + + Args: + data: Parsed global multi-host KVM YAML. + host_name: Hypervisor name. + + The global configK3s.yaml keeps the user's local VM SSH key path so the + control host can install K3s. Remote hypervisors also need a local copy of + the same key while creating VMs and waiting for cloud-init SSH readiness. + """ + host = requireHost(hostList(data), host_name) + vm_ssh = data.get("vmSsh") if isinstance(data.get("vmSsh"), dict) else {} + source_key = expandPath(str(vm_ssh.get("key") or "~/.ssh/id_ed25519")) + if host["connection"] == "local": + return source_key, source_key + target_key = str( + vm_ssh.get("remoteHypervisorKey") + or vm_ssh.get("remoteKey") + or f"{host['remoteWorkDir']}/ssh/{Path(source_key).name}" + ) + return source_key, target_key + + +def printHostVmSshKeyTsv(args: argparse.Namespace) -> None: + """Print VM SSH key source/target paths for one hypervisor.""" + source_key, target_key = hostVmSshKeyPaths(loadYaml(args.config), args.host) + source_pub = f"{source_key}.pub" + target_pub = f"{target_key}.pub" + print("\t".join([source_key, target_key, source_pub, target_pub])) + + +def writeHostNetworkXml(args: argparse.Namespace) -> None: + """Write libvirt routed network XML for one hypervisor.""" + host = requireHost(hostList(loadYaml(args.config)), args.host) + xml = f""" + {host['networkName']} + + + + + + + + +""" + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(xml, encoding="utf-8") + print(output) + + +def writeHostLocalKvm(args: argparse.Namespace) -> None: + """Write host-local kvm.yaml consumed by kvm/createKvmVms.py.""" + data = loadYaml(args.config) + host = requireHost(hostList(data), args.host) + vm_ssh = data.get("vmSsh") if isinstance(data.get("vmSsh"), dict) else {} + ssh_user = str(vm_ssh.get("user") or "ubuntu") + _, ssh_key = hostVmSshKeyPaths(data, args.host) + seedemu = data.get("seedemu") if isinstance(data.get("seedemu"), dict) else {} + host_work_dir = str(host["remoteWorkDir"]) + payload = { + "clusterName": clusterName(data), + "nodes": [ + { + "name": node["name"], + "role": node["role"], + "ip": node["ip"], + "mac": node["mac"], + "vcpus": node["vcpus"], + "memory_mb": node["memory_mb"], + "disk_gb": node["disk_gb"], + } + for node in hostNodes(data, args.host) + ], + "ssh": {"user": ssh_user, "key": ssh_key}, + "kvm": { + "network": host["networkName"], + "storageDir": host_work_dir, + "diskDir": f"{host_work_dir}/disks", + "cloudInitDir": f"{host_work_dir}/cloud-init", + "baseImagePath": f"{host_work_dir}/base/jammy-server-cloudimg-amd64.img", + "skipK3sConfig": True, + }, + "outputs": { + "tmpDir": f"{host_work_dir}/tmp", + "kvmState": f"{host_work_dir}/kvmState.yaml", + }, + } + if seedemu: + payload["seedemu"] = copy.deepcopy(seedemu) + writeYaml(args.output, payload) + print(args.output) + + +def writeGlobalK3sConfig(args: argparse.Namespace) -> None: + """Write global configK3s.yaml for the VM cluster.""" + data = loadYaml(args.config) + vm_ssh = data.get("vmSsh") if isinstance(data.get("vmSsh"), dict) else {} + ssh_user = str(vm_ssh.get("user") or "ubuntu") + ssh_key = expandPath(str(vm_ssh.get("key") or "~/.ssh/id_ed25519")) + payload: dict[str, Any] = { + "clusterName": clusterName(data), + "nodes": [ + { + "name": node["name"], + "role": node["role"], + "ip": node["ip"], + "vcpus": int(node.get("vcpus") or 0), + "memoryMb": int(node.get("memoryMb") or node.get("memory_mb") or 0), + "diskGb": int(node.get("diskGb") or node.get("disk_gb") or 0), + "ssh": {"user": ssh_user, "key": ssh_key}, + } + for node in vmPlan(data) + ], + } + payload.update(passThroughSections(data)) + writeYaml(args.output, payload) + print(args.output) + + +def writeState(args: argparse.Namespace) -> None: + """Write multiHostKvmState.yaml for cleanup and auditing.""" + data = loadYaml(args.config) + payload = { + "clusterName": clusterName(data), + "hypervisors": hostList(data), + "nodes": vmPlan(data), + "routingTunnel": routingTunnelConfig(data), + "tunnelRows": tunnelRows(data), + "outputs": { + "configK3s": outputPath(data, "k3sConfig", SETUP_DIR / "configK3s.yaml"), + "kubeconfig": outputPath(data, "kubeconfig", SETUP_DIR / f"{clusterName(data)}.kubeconfig.yaml"), + "inventory": outputPath(data, "inventory", SETUP_DIR / f"{clusterName(data)}.inventory.yaml"), + }, + } + writeYaml(args.output, payload) + print(args.output) + + +def loadState(path: str | Path) -> dict[str, Any]: + """Load and validate multiHostKvmState.yaml.""" + data = loadYaml(path) + if not isinstance(data.get("hypervisors"), list): + raise SystemExit(f"Invalid multiHostKvmState.yaml: missing hypervisors list in {path}") + return data + + +def printStateHostsTsv(args: argparse.Namespace) -> None: + """Print hypervisor records from multiHostKvmState.yaml.""" + data = loadState(args.state) + for host in data["hypervisors"]: + print( + "\t".join( + str(host.get(key) or "") + for key in ( + "name", + "ip", + "connection", + "sshUser", + "sshKey", + "networkName", + "bridgeName", + "cidr", + "gateway", + "remoteWorkDir", + ) + ) + ) + + +def printStateRoutesTsv(args: argparse.Namespace) -> None: + """Print cleanup routes from multiHostKvmState.yaml.""" + data = loadState(args.state) + current_names = {str(host.get("name")) for host in data["hypervisors"]} + if args.host not in current_names: + raise SystemExit(f"unknown hypervisor in state: {args.host}") + tunnel_rows = data.get("tunnelRows") if isinstance(data.get("tunnelRows"), list) else [] + if tunnel_rows: + for row in tunnel_rows: + if row.get("host") == args.host: + print( + "\t".join( + [ + str(row.get("remoteCidr") or ""), + str(row.get("peerTunnelIp") or ""), + str(row.get("peerName") or ""), + str(row.get("interface") or ""), + ] + ) + ) + return + for host in data["hypervisors"]: + if host.get("name") == args.host: + continue + print("\t".join([str(host.get("cidr") or ""), str(host.get("ip") or ""), str(host.get("name") or ""), ""])) + + +def printStateTunnelsTsv(args: argparse.Namespace) -> None: + """Print VXLAN tunnel rows from multiHostKvmState.yaml for cleanup.""" + data = loadState(args.state) + tunnel_rows = data.get("tunnelRows") if isinstance(data.get("tunnelRows"), list) else [] + for row in tunnel_rows: + if row.get("host") != args.host: + continue + print( + "\t".join( + str(row.get(key) or "") + for key in ( + "interface", + "vni", + "dstPort", + "localPhysicalIp", + "peerPhysicalIp", + "localTunnelCidr", + "peerTunnelIp", + "peerName", + ) + ) + ) + + +def printStateOutputVars(args: argparse.Namespace) -> None: + """Print generated output paths from multiHostKvmState.yaml.""" + data = loadState(args.state) + outputs = data.get("outputs") if isinstance(data.get("outputs"), dict) else {} + values = { + "stateConfigK3s": outputs.get("configK3s") or str(SETUP_DIR / "configK3s.yaml"), + "stateKubeconfig": outputs.get("kubeconfig") or str(SETUP_DIR / f"{data.get('clusterName', 'seedemu-k3s')}.kubeconfig.yaml"), + "stateInventory": outputs.get("inventory") or str(SETUP_DIR / f"{data.get('clusterName', 'seedemu-k3s')}.inventory.yaml"), + } + for key, value in values.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def validate(args: argparse.Namespace) -> None: + """Validate a global multi-host KVM config.""" + data = loadYaml(args.config) + hosts = hostList(data) + nodes = vmPlan(data) + print(f"Validated {len(hosts)} hypervisors and {len(nodes)} VMs.") + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("validate").set_defaults(func=validate) + sub.add_parser("shell-vars").set_defaults(func=printShellVars) + sub.add_parser("hosts-tsv").set_defaults(func=printHostsTsv) + + routes = sub.add_parser("host-routes-tsv") + routes.add_argument("--host", required=True) + routes.set_defaults(func=printHostRoutesTsv) + + tunnels = sub.add_parser("host-tunnels-tsv") + tunnels.add_argument("--host", required=True) + tunnels.set_defaults(func=printHostTunnelsTsv) + + vm_key = sub.add_parser("host-vm-ssh-key-tsv") + vm_key.add_argument("--host", required=True) + vm_key.set_defaults(func=printHostVmSshKeyTsv) + + network_xml = sub.add_parser("write-host-network-xml") + network_xml.add_argument("--host", required=True) + network_xml.add_argument("--output", required=True) + network_xml.set_defaults(func=writeHostNetworkXml) + + local_kvm = sub.add_parser("write-host-local-kvm") + local_kvm.add_argument("--host", required=True) + local_kvm.add_argument("--output", required=True) + local_kvm.set_defaults(func=writeHostLocalKvm) + + global_config = sub.add_parser("write-global-k3s-config") + global_config.add_argument("--output", required=True) + global_config.set_defaults(func=writeGlobalK3sConfig) + + state = sub.add_parser("write-state") + state.add_argument("--output", required=True) + state.set_defaults(func=writeState) + + state_hosts = sub.add_parser("state-hosts-tsv") + state_hosts.add_argument("--state", required=True) + state_hosts.set_defaults(func=printStateHostsTsv) + + state_routes = sub.add_parser("state-routes-tsv") + state_routes.add_argument("--state", required=True) + state_routes.add_argument("--host", required=True) + state_routes.set_defaults(func=printStateRoutesTsv) + + state_tunnels = sub.add_parser("state-tunnels-tsv") + state_tunnels.add_argument("--state", required=True) + state_tunnels.add_argument("--host", required=True) + state_tunnels.set_defaults(func=printStateTunnelsTsv) + + state_outputs = sub.add_parser("state-output-vars") + state_outputs.add_argument("--state", required=True) + state_outputs.set_defaults(func=printStateOutputVars) + + args = parser.parse_args() + args.func(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py new file mode 100755 index 000000000..ac9d86ebb --- /dev/null +++ b/seedemu/k8sTools/resources/setup/multiHostKvm/prepareKvmHypervisors.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Python entrypoint for prepareKvmHypervisors with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Prepare physical KVM hypervisors for a multi-host KVM SeedEMU cluster. +# +# Inputs: +# $1: global multi-host kvm.yaml. Defaults to ../kvm.yaml. +# +# Generated/modified state: +# - Creates one routed libvirt network per hypervisor. +# - Enables IPv4 forwarding on each hypervisor. +# - Optionally creates point-to-point VXLAN route tunnels. +# - Installs static routes between the hypervisor VM subnets. +# +# Execution context: +# Run from the generated setup directory before createMultiHostKvmVms.py. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/kvm.yaml}" +HELPER="${SCRIPT_DIR}/manageMultiHostKvmConfig.py" +TMP_DIR="" + +usage() { + cat </dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +sshOptions() { + # Args: + # $1: SSH private key path. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o IdentityAgent=none \ + -i "${ssh_key}" +} + +runHostCommand() { + # Args: + # $1=name, $2=ip, $3=connection, $4=ssh_user, $5=ssh_key, $6=command. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + local libvirt_command="export LIBVIRT_DEFAULT_URI=qemu:///system; ${command}" + + if [ "${connection}" = "local" ]; then + bash -lc "${libvirt_command}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "${libvirt_command}" /dev/null +} + +cleanupTmp() { + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" || true +} + +validateConfig() { + python3 "${HELPER}" --config "${CONFIG_PATH}" validate +} + +prepareHost() { + # Args are read from one hosts-tsv row. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local network_name="$6" + local bridge_name="$7" + local cidr="$8" + local remote_work_dir="${10}" + local network_xml="${TMP_DIR}/${name}.xml" + local remote_xml="${remote_work_dir}/network.xml" + + echo "[hypervisor-prepare] ${name} (${ip}) network=${network_name} bridge=${bridge_name}" + python3 "${HELPER}" --config "${CONFIG_PATH}" write-host-network-xml \ + --host "${name}" \ + --output "${network_xml}" >/dev/null + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n true && command -v virsh >/dev/null && command -v virt-install >/dev/null && command -v qemu-img >/dev/null && command -v docker >/dev/null && command -v curl >/dev/null && command -v ssh >/dev/null" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "mkdir -p $(printf '%q' "${remote_work_dir}")" + copyFileToHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" "${network_xml}" "${remote_xml}" + + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + virsh net-info $(printf '%q' "${network_name}") >/dev/null 2>&1 || virsh net-define $(printf '%q' "${remote_xml}") >/dev/null + virsh net-start $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + virsh net-autostart $(printf '%q' "${network_name}") >/dev/null 2>&1 || true + sudo -n sysctl -w net.ipv4.ip_forward=1 >/dev/null + if command -v iptables >/dev/null 2>&1; then + sudo -n iptables -C FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -i $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -C FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT >/dev/null 2>&1 || sudo -n iptables -I FORWARD -o $(printf '%q' "${bridge_name}") -j ACCEPT + sudo -n iptables -t nat -C POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE >/dev/null 2>&1 || sudo -n iptables -t nat -A POSTROUTING -s $(printf '%q' "${cidr}") ! -d 10.0.0.0/8 -j MASQUERADE + fi + " + + while IFS=$'\t' read -r tunnel_name vni dst_port local_ip peer_ip local_tunnel_cidr peer_tunnel_ip peer_name; do + [ -n "${tunnel_name}" ] || continue + echo " tunnel ${tunnel_name}: ${local_tunnel_cidr} -> ${peer_tunnel_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" " + set -euo pipefail + underlay_dev=\$(ip route get $(printf '%q' "${peer_ip}") | awk '{for (i=1;i<=NF;i++) if (\$i==\"dev\") {print \$(i+1); exit}}') + [ -n \"\${underlay_dev}\" ] || { echo 'Cannot determine underlay dev for peer $(printf '%q' "${peer_ip}")' >&2; exit 1; } + sudo -n ip link del $(printf '%q' "${tunnel_name}") >/dev/null 2>&1 || true + sudo -n ip link add $(printf '%q' "${tunnel_name}") type vxlan id $(printf '%q' "${vni}") local $(printf '%q' "${local_ip}") remote $(printf '%q' "${peer_ip}") dstport $(printf '%q' "${dst_port}") dev \"\${underlay_dev}\" + sudo -n ip addr add $(printf '%q' "${local_tunnel_cidr}") dev $(printf '%q' "${tunnel_name}") + sudo -n ip link set $(printf '%q' "${tunnel_name}") up + " + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-tunnels-tsv --host "${name}") + + while IFS=$'\t' read -r remote_cidr peer_ip peer_name route_dev; do + [ -n "${remote_cidr}" ] || continue + if [ -n "${route_dev}" ]; then + echo " route ${remote_cidr} via ${peer_ip} dev ${route_dev} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}") dev $(printf '%q' "${route_dev}")" + else + echo " route ${remote_cidr} via ${peer_ip} (${peer_name})" + runHostCommand "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "sudo -n ip route replace $(printf '%q' "${remote_cidr}") via $(printf '%q' "${peer_ip}")" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" host-routes-tsv --host "${name}") +} + +main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 + fi + requireCommand python3 + requireCommand ssh + requireCommand scp + requireCommand cp + requireCommand mktemp + + [ -s "${CONFIG_PATH}" ] || { + echo "Missing config: ${CONFIG_PATH}" >&2 + exit 1 + } + + TMP_DIR="$(mktemp -d "${SETUP_DIR}/tmp.multi-host-kvm.XXXXXX")" + trap cleanupTmp EXIT + + validateConfig + while IFS=$'\t' read -r name ip connection ssh_user ssh_key network_name bridge_name cidr gateway remote_work_dir; do + prepareHost "${name}" "${ip}" "${connection}" "${ssh_user}" "${ssh_key}" \ + "${network_name}" "${bridge_name}" "${cidr}" "${gateway}" "${remote_work_dir}" + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" hosts-tsv) + + echo "KVM hypervisors are ready." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/setup/ovn/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py new file mode 100755 index 000000000..ddbc243b2 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/ovn/cleanKubeOvnFabric.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Python entrypoint for cleanKubeOvnFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove Kube-OVN non-primary CNI artifacts from a physical K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# Uninstalls the Kube-OVN Helm release when a live cluster is available and +# removes Kube-OVN/OVS/OVN runtime files from configured nodes. It does not +# reboot nodes; users can reboot manually if they need a fully cold network +# state as recommended by Kube-OVN upstream docs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to run as root. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + + echo "[ovn-clean] ${name} (${ip})" + if [ "${connection}" = "local" ]; then + sudo -n bash -s <<<"${script}" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "sudo -n bash -s" <<<"${script}" +} + +ensureHelm() { + # Print an existing helm binary or the private binary downloaded by + # installKubeOvnFabric.py. + if command -v helm >/dev/null 2>&1; then + command -v helm + return + fi + local helm_bin="${ovnHelmCacheDir}/bin/helm" + if [ -x "${helm_bin}" ]; then + echo "${helm_bin}" + return + fi + echo "" +} + +uninstallHelmRelease() { + # Remove Kubernetes resources while the API server is still available. + [ -s "${outputKubeconfig}" ] || return 0 + local helm_bin + helm_bin="$(ensureHelm)" + if [ -n "${helm_bin}" ]; then + "${helm_bin}" uninstall "${ovnReleaseName}" -n "${ovnNamespace}" --kubeconfig "${outputKubeconfig}" >/dev/null 2>&1 || true + else + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete deploy,ds,sts,svc,cm,sa,role,rolebinding \ + -l app.kubernetes.io/instance="${ovnReleaseName}" --ignore-not-found >/dev/null 2>&1 || true + fi +} + +clean_node_script=' +set -euo pipefail +rm -rf /var/run/openvswitch +rm -rf /var/run/ovn +rm -rf /etc/origin/openvswitch +rm -rf /etc/origin/ovn +rm -rf /etc/cni/net.d/00-kube-ovn.conflist +rm -rf /etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/00-kube-ovn.conflist +rm -rf /var/lib/rancher/k3s/agent/etc/cni/net.d/01-kube-ovn.conflist +rm -rf /var/log/openvswitch +rm -rf /var/log/ovn +rm -rf /var/log/kube-ovn +ip link del ovn0 2>/dev/null || true +' + +uninstallHelmRelease +while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + runNodeRootScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "${clean_node_script}" +done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) + +echo "Kube-OVN fabric cleaned." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py new file mode 100755 index 000000000..160806a04 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/ovn/installKubeOvnFabric.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""Python entrypoint for installKubeOvnFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Install Kube-OVN as a non-primary CNI for SeedEMU secondary networks. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml with fabric.type=ovn and optional ovn.* settings. +# +# Outputs/side effects: +# Installs the Kube-OVN Helm release into the configured K3s cluster. K3s +# flannel remains the primary CNI for eth0; Kube-OVN only serves Multus +# secondary interfaces used by SeedEMU simulated networks. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN install." + exit 0 +fi + +requireCommand() { + # Args: + # $1: Command name required by this script. + command -v "$1" >/dev/null 2>&1 || { + echo "Missing required command: $1" >&2 + exit 1 + } +} + +runWithTimeout() { + # Args: + # $1: duration accepted by timeout, remaining args: command to run. + local duration="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${duration}" "$@" + else + "$@" + fi +} + +sshOptions() { + # Args: + # $1: SSH private key used for the target node. + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeScript() { + # Args: + # $1: node name, $2: management IP, $3: connection mode, $4: SSH user, + # $5: SSH key, $6: shell script to execute on the node. + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + if [ "${connection}" = "local" ]; then + bash -s <<<"${script}" + return + fi + # shellcheck disable=SC2046 + ssh $(sshOptions "${ssh_key}") "${ssh_user}@${ip}" "bash -s" <<<"${script}" +} + +preloadKubeOvnImage() { + # Pull Kube-OVN once on the setup host, then import the image tar into each + # K3s node's containerd. This avoids repeated node-side Docker Hub pulls. + local image="docker.io/kubeovn/kube-ovn:${ovnChartVersion}" + local mirror_image="docker.m.daocloud.io/kubeovn/kube-ovn:${ovnChartVersion}" + local tar_path="${ovnHelmCacheDir}/kube-ovn_${ovnChartVersion}.tar" + mkdir -p "$(dirname "${tar_path}")" + echo "[ovn] preloading ${image} into K3s containerd nodes" + if ! runWithTimeout 180s sudo -n docker pull "${image}"; then + sudo -n docker pull "${mirror_image}" + sudo -n docker tag "${mirror_image}" "${image}" + fi + sudo -n docker save "${image}" -o "${tar_path}" + sudo -n chmod 0644 "${tar_path}" + + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " import ${image} to ${name}" + if [ "${nodeConnection}" = "local" ]; then + sudo -n k3s ctr images import "${tar_path}" + else + local remote_tar="/tmp/$(basename "${tar_path}")" + # shellcheck disable=SC2046 + scp $(sshOptions "${nodeSshKey}") "${tar_path}" "${nodeSshUser}@${ip}:${remote_tar}" + # shellcheck disable=SC2046 + ssh -n $(sshOptions "${nodeSshKey}") "${nodeSshUser}@${ip}" \ + "sudo -n k3s ctr images import '${remote_tar}' >/dev/null && rm -f '${remote_tar}'" + fi + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +prepareKubeOvnCniBinDir() { + # Prepare each node's K3s CNI binary directory for Kube-OVN install-cni. + # Args: none. Reads nodes and ovnCniBinDir from configK3s.yaml. + # + # K3s often keeps CNI plugins as symlinks under /var/lib/rancher/k3s/data/cni. + # Kube-OVN's install-cni init container copies these exact binaries into + # that same mounted path and fails if a target is a dangling symlink inside + # the container mount. Remove only the known overwrite targets. + echo "[ovn] preparing K3s CNI binary directory on all nodes: ${ovnCniBinDir}" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" node-ssh-vars --name "${name}")" + echo " prepare ${name}" + runNodeScript "${name}" "${ip}" "${nodeConnection}" "${nodeSshUser}" "${nodeSshKey}" "$(cat </dev/null 2>&1 + } + + if command -v helm >/dev/null 2>&1 && helmUsable "$(command -v helm)"; then + command -v helm + return + fi + + local helm_dir="${ovnHelmCacheDir}/bin" + local helm_bin="${helm_dir}/helm" + if helmUsable "${helm_bin}"; then + echo "${helm_bin}" + return + fi + + mkdir -p "${helm_dir}" "${ovnHelmCacheDir}/download" + rm -f "${helm_bin}" + local archive="${ovnHelmCacheDir}/download/helm-v3.15.4-linux-amd64.tar.gz" + local archive_tmp="${archive}.tmp" + echo "[ovn] downloading private helm binary to ${helm_bin}" >&2 + rm -f "${archive_tmp}" + curl --fail --location --show-error --retry 5 --retry-delay 2 --retry-all-errors \ + https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o "${archive_tmp}" + tar -tzf "${archive_tmp}" >/dev/null + mv "${archive_tmp}" "${archive}" + rm -rf "${ovnHelmCacheDir}/download/linux-amd64" + tar -xzf "${archive}" -C "${ovnHelmCacheDir}/download" + cp "${ovnHelmCacheDir}/download/linux-amd64/helm" "${helm_bin}" + chmod +x "${helm_bin}" + if ! helmUsable "${helm_bin}"; then + rm -f "${helm_bin}" + echo "Downloaded helm binary is not usable: ${helm_bin}" >&2 + exit 1 + fi + echo "${helm_bin}" +} + +installKubeOvn() { + # Install Kube-OVN in non-primary mode. Kube-OVN's CNI binary is placed into + # the K3s CNI bin dir so Multus can invoke type=kube-ovn delegates. + local helm_bin="$1" + + "${helm_bin}" repo add "${ovnHelmRepoName}" "${ovnHelmRepoUrl}" >/dev/null 2>&1 || true + "${helm_bin}" repo update >/dev/null + + kubectl --kubeconfig "${outputKubeconfig}" label node "${k3sMasterName}" kube-ovn/role=master --overwrite + + echo "[ovn] installing Kube-OVN ${ovnChartVersion} in non-primary CNI mode" + "${helm_bin}" upgrade --install "${ovnReleaseName}" "${ovnHelmRepoName}/kube-ovn" \ + --kubeconfig "${outputKubeconfig}" \ + --namespace "${ovnNamespace}" \ + --version "${ovnChartVersion}" \ + --set cni_conf.NON_PRIMARY_CNI=true \ + --set cni_conf.CNI_CONF_DIR="${ovnCniConfDir}" \ + --set cni_conf.MOUNT_CNI_CONF_DIR="${ovnMountCniConfDir}" \ + --set cni_conf.CNI_BIN_DIR="${ovnCniBinDir}" \ + --set networking.TUNNEL_TYPE="${ovnTunnelType}" \ + --set networking.IFACE="${ovnIface}" \ + --set ipv4.POD_CIDR="${ovnPodCidr}" \ + --set ipv4.POD_GATEWAY="${ovnPodGateway}" \ + --set ipv4.SVC_CIDR="${ovnServiceCidr}" \ + --set ipv4.JOIN_CIDR="${ovnJoinCidr}" \ + --set MASTER_NODES="${ovnMasterNodes}" \ + --wait \ + --timeout 10m +} + +waitKubeOvnReady() { + # Confirm the controller, OVS/OVN daemonset, and CNI daemonset are rolled + # out before the running stage creates Kube-OVN NAD/Subnet resources. + local resources=( + "deployment/kube-ovn-controller" + "deployment/ovn-central" + "daemonset/ovs-ovn" + "daemonset/kube-ovn-cni" + ) + for resource in "${resources[@]}"; do + if kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get "${resource}" >/dev/null 2>&1; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status "${resource}" --timeout=600s + fi + done + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=300s + kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=300s +} + +repairK3sCniBinDirAfterKubeOvnInstall() { + # Kube-OVN's install-cni init container runs inside a container mount of + # the K3s CNI bin dir. On K3s this can leave the host dir without the + # primary loopback/portmap links or without the kube-ovn delegate binary. + # Use a short-lived hostNetwork pod per node so this repair does not depend + # on the CNI plugin path that it is fixing. + echo "[ovn] verifying K3s CNI binaries after Kube-OVN install" + while IFS=$'\t' read -r name role ip mac vcpus memory_mb disk_gb; do + local pod_name + pod_name="seedemu-ovn-cni-repair-$(printf '%s' "${name}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')" + pod_name="${pod_name%-}" + echo " repair ${name} with pod/${pod_name}" + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" \ + --ignore-not-found=true --wait=false >/dev/null 2>&1 || true + cat </dev/null || true)" + if [ "${phase}" = "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=80 || true + break + fi + if [ "${phase}" = "Failed" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" logs "${pod_name}" --tail=120 || true + return 1 + fi + sleep 2 + done + local final_phase + final_phase="$(kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + if [ "${final_phase}" != "Succeeded" ]; then + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" describe pod "${pod_name}" || true + return 1 + fi + kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" delete pod "${pod_name}" --wait=false >/dev/null 2>&1 || true + done < <(python3 "${HELPER}" --config "${CONFIG_PATH}" nodes-tsv) +} + +main() { + requireCommand curl + requireCommand kubectl + requireCommand python3 + requireCommand scp + requireCommand ssh + requireCommand docker + requireCommand tar + + if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + echo "Run applyK3sCluster.py or k8sTools.py build before installing Kube-OVN." >&2 + exit 1 + fi + + local helm_bin + helm_bin="$(ensureHelm)" + preloadKubeOvnImage + prepareKubeOvnCniBinDir + installKubeOvn "${helm_bin}" + waitKubeOvnReady + repairK3sCniBinDirAfterKubeOvnInstall + echo "Kube-OVN non-primary CNI is ready." +} + +main "$@" +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py new file mode 100755 index 000000000..5b6ea0346 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/ovn/validateKubeOvnFabric.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Python entrypoint for validateKubeOvnFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate that Kube-OVN non-primary CNI is ready for SeedEMU NADs. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ../configK3s.yaml. +# +# Inputs: +# configK3s.yaml and the kubeconfig generated by applyK3sCluster.py. +# +# Side effects: +# None. This script only reads cluster state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "${CONFIG_PATH}" ]; then + echo "Missing config file: ${CONFIG_PATH}" >&2 + exit 1 +fi + +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" shell-vars)" +eval "$(python3 "${HELPER}" --config "${CONFIG_PATH}" ovn-shell-vars)" + +if [ "${fabricType}" != "ovn" ] && [ "${fabricType}" != "kube-ovn" ]; then + echo "No Kube-OVN fabric configured; skipping OVN validation." + exit 0 +fi + +if [ ! -s "${outputKubeconfig}" ]; then + echo "Kubeconfig not found: ${outputKubeconfig}" >&2 + exit 1 +fi + +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" get pods -o wide +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/subnets.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" wait --for=condition=Established crd/vpcs.kubeovn.io --timeout=120s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status deployment/kube-ovn-controller --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/kube-ovn-cni --timeout=300s +kubectl --kubeconfig "${outputKubeconfig}" -n "${ovnNamespace}" rollout status daemonset/ovs-ovn --timeout=300s +echo "Kube-OVN fabric validation passed." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py new file mode 100755 index 000000000..99981ac62 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/preparePhysicalNodes.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Python entrypoint for preparePhysicalNodes with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate physical nodes before building a SeedEMU K3s cluster. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Checks: +# - Each node can be reached with the configured connection method. +# - sudo -n works for the configured user. +# - iproute2 and curl are available. +# - linux-vxlan fabric nodes expose their configured underlay interface. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +CONFIG_PATH="${1:-./configK3s.yaml}" +HELPER="${SCRIPT_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + if [ "$connection" = "local" ]; then + bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "$command" /dev/null && command -v curl >/dev/null" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" nodes-tsv) + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" +if [ "${fabricType}" = "linux-vxlan" ]; then + echo "Validating linux-vxlan underlay interfaces..." + while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + echo " ${name}: underlay=${underlay}" + runNodeCommand "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "ip link show $(printf '%q' "$underlay") >/dev/null" + done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +fi + +echo "Physical node preflight passed." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py new file mode 100644 index 000000000..0e0b17d8f --- /dev/null +++ b/seedemu/k8sTools/resources/setup/vxlan/_embeddedShell.py @@ -0,0 +1,21 @@ +"""Run embedded shell bodies for k8sTools Python entrypoints.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def runEmbeddedShell(script_path: Path, argv: list[str], shell_body: str) -> int: + """Execute shell_body as the implementation of a Python entrypoint. + + Args: + script_path: Path to the Python entrypoint being executed. + argv: Arguments passed by the caller. + shell_body: Embedded bash program. + """ + entrypoint = script_path.resolve() + env = os.environ.copy() + env["SEED_K8S_ENTRYPOINT"] = str(entrypoint) + completed = subprocess.run(["bash", "-c", shell_body, str(entrypoint), *argv], env=env) + return int(completed.returncode) diff --git a/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py new file mode 100755 index 000000000..9288effe6 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/vxlan/cleanLinuxVxlanFabric.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Python entrypoint for cleanLinuxVxlanFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Remove the Linux VXLAN bridge fabric described by configK3s.yaml. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Side effects: +# Deletes the configured test macvlan interface, VXLAN interface, and bridge +# on every configured node. It does not uninstall K3s or delete workload pods. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; nothing to clean." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-clean] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +cleanup_script=' +set -euo pipefail +macvlan_name="$1" +vxlan_name="$2" +bridge_name="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link del "$vxlan_name" 2>/dev/null || true +ip link del "$bridge_name" 2>/dev/null || true +' + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$cleanup_script" \ + "$fabricMacvlanTestName" "$fabricVxlanName" "$fabricBridgeName" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Linux VXLAN fabric cleaned." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py new file mode 100755 index 000000000..040f6d024 --- /dev/null +++ b/seedemu/k8sTools/resources/setup/vxlan/configureLinuxVxlanFabric.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Python entrypoint for configureLinuxVxlanFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Create a two-node Linux VXLAN bridge fabric for SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Expected config: +# fabric.type: linux-vxlan +# fabric.nodes..underlayInterface: management NIC used as VXLAN underlay. +# +# Side effects: +# Creates a bridge such as br-seedemu and a VXLAN device such as vxseed0 on +# each node. On failure, it calls cleanLinuxVxlanFabric.py to roll back all +# configured fabric interfaces. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric setup." + exit 0 +fi + +rollbackOnError() { + local rc=$? + if [ "$rc" -ne 0 ]; then + echo "[fabric] setup failed; rolling back configured fabric interfaces" >&2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap rollbackOnError EXIT + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-configure] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +configure_script=' +set -euo pipefail +bridge_name="$1" +vxlan_name="$2" +vni="$3" +local_ip="$4" +peer_ip="$5" +underlay="$6" +dst_port="$7" +mtu="$8" + +if [ "${#bridge_name}" -gt 15 ] || [ "${#vxlan_name}" -gt 15 ]; then + echo "Linux interface names must be at most 15 characters" >&2 + exit 1 +fi + +ip link show "$underlay" >/dev/null + +if ip link show "$vxlan_name" >/dev/null 2>&1; then + ip link del "$vxlan_name" +fi + +if ! ip link show "$bridge_name" >/dev/null 2>&1; then + ip link add "$bridge_name" type bridge +fi + +ip link set "$bridge_name" up +ip link add "$vxlan_name" type vxlan id "$vni" local "$local_ip" remote "$peer_ip" dev "$underlay" dstport "$dst_port" +ip link set "$vxlan_name" mtu "$mtu" || true +ip link set "$vxlan_name" master "$bridge_name" +ip link set "$vxlan_name" up +' + +echo "Configuring Linux VXLAN fabric:" +echo " bridge=${fabricBridgeName}" +echo " vxlan=${fabricVxlanName}" +echo " vni=${fabricVni}" +echo " dstPort=${fabricDstPort}" + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$configure_script" \ + "$fabricBridgeName" "$fabricVxlanName" "$fabricVni" "$ip" "$peer_ip" \ + "$underlay" "$fabricDstPort" "$fabricMtu" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +trap - EXIT +echo "Linux VXLAN fabric configured." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py new file mode 100755 index 000000000..889d212ce --- /dev/null +++ b/seedemu/k8sTools/resources/setup/vxlan/validateLinuxVxlanFabric.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Python entrypoint for validateLinuxVxlanFabric with its shell body embedded.""" +from __future__ import annotations + +import sys +from pathlib import Path + +from _embeddedShell import runEmbeddedShell + + +SHELL_BODY = r'''#!/usr/bin/env bash +# Validate the Linux VXLAN bridge fabric used by SeedEMU Multus/macvlan. +# +# Args: +# $1: Optional configK3s.yaml path. Defaults to ./configK3s.yaml. +# +# Validation: +# 1. Adds temporary /30 IPs to the configured bridge and pings both ways. +# 2. Creates temporary macvlan interfaces on the bridge and pings both ways. +# +# Side effects: +# Temporary test IPs and macvlan interfaces are always removed. If validation +# fails, the configured VXLAN fabric is also removed to avoid stale state. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${SEED_K8S_ENTRYPOINT}")" && pwd)" +SETUP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_PATH="${1:-${SETUP_DIR}/configK3s.yaml}" +HELPER="${SETUP_DIR}/manageK3sConfig.py" + +if [ ! -s "$CONFIG_PATH" ]; then + echo "Missing config file: $CONFIG_PATH" >&2 + exit 1 +fi + +eval "$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-shell-vars)" + +if [ "${fabricType}" != "linux-vxlan" ]; then + echo "No linux-vxlan fabric configured; skipping fabric validation." + exit 0 +fi + +sshOptions() { + local ssh_key="$1" + printf '%s\n' \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + -i "$ssh_key" +} + +runNodeRootScript() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local script="$6" + shift 6 + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -s -- "$@" <<<"$script" + return + fi + + local quoted_args="" + local arg + for arg in "$@"; do + quoted_args+=" $(printf '%q' "$arg")" + done + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -s --${quoted_args}" <<<"$script" +} + +runNodeRootCommand() { + local name="$1" + local ip="$2" + local connection="$3" + local ssh_user="$4" + local ssh_key="$5" + local command="$6" + + echo "[fabric-validate] ${name} (${ip})" + if [ "$connection" = "local" ]; then + sudo -n bash -lc "$command" + return + fi + + # shellcheck disable=SC2046 + ssh $(sshOptions "$ssh_key") "${ssh_user}@${ip}" "sudo -n bash -lc $(printf '%q' "$command")" &2 + python3 "${SCRIPT_DIR}/cleanLinuxVxlanFabric.py" "$CONFIG_PATH" || true + fi + exit "$rc" +} +trap cleanupOnExit EXIT + +setup_bridge_ip=' +set -euo pipefail +bridge_name="$1" +bridge_test_ip="$2" + +ip link show "$bridge_name" >/dev/null +ip addr del "$bridge_test_ip" dev "$bridge_name" 2>/dev/null || true +ip addr add "$bridge_test_ip" dev "$bridge_name" +ip link set "$bridge_name" up +' + +setup_macvlan=' +set -euo pipefail +bridge_name="$1" +macvlan_name="$2" +macvlan_test_ip="$3" + +ip link del "$macvlan_name" 2>/dev/null || true +ip link add "$macvlan_name" link "$bridge_name" type macvlan mode bridge +ip addr add "$macvlan_test_ip" dev "$macvlan_name" +ip link set "$macvlan_name" up +' + +echo "Validating Linux VXLAN bridge reachability..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_bridge_ip" \ + "$fabricBridgeName" "$bridge_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 1 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_bridge_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $8; exit}')" + peer_bridge_ip="${peer_bridge_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -c 3 -W 2 $(printf '%q' "$peer_bridge_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +echo "Removing bridge test IPs before macvlan validation..." +cleanupTestArtifacts +sleep 1 + +echo "Validating macvlan reachability over Linux VXLAN bridge..." +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + runNodeRootScript \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" "$setup_macvlan" \ + "$fabricBridgeName" "$fabricMacvlanTestName" "$macvlan_test_ip" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) +sleep 2 + +while IFS=$'\t' read -r name role ip connection ssh_user ssh_key underlay bridge_test_ip macvlan_test_ip peer_name peer_ip; do + peer_macvlan_test_ip="$(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv | awk -F '\t' -v peer="$peer_name" '$1 == peer {print $9; exit}')" + peer_macvlan_ip="${peer_macvlan_test_ip%/*}" + runNodeRootCommand \ + "$name" "$ip" "$connection" "$ssh_user" "$ssh_key" \ + "ping -I $(printf '%q' "$fabricMacvlanTestName") -c 3 -W 2 $(printf '%q' "$peer_macvlan_ip")" +done < <(python3 "$HELPER" --config "$CONFIG_PATH" fabric-nodes-tsv) + +cleanupTestArtifacts +trap - EXIT +echo "Linux VXLAN fabric validation passed." +''' + + +def main(argv: list[str] | None = None) -> int: + """Run this entrypoint with optional argv override for tests.""" + return runEmbeddedShell(Path(__file__), list(sys.argv[1:] if argv is None else argv), SHELL_BODY) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/k8sTools/runner.py b/seedemu/k8sTools/runner.py new file mode 100644 index 000000000..7f5891b19 --- /dev/null +++ b/seedemu/k8sTools/runner.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +from contextlib import contextmanager +import shlex +import subprocess +import tempfile +import shutil +from pathlib import Path +from typing import Any, Iterator + +from .config import ( + addK8sToolsMetadata, + detectKind, + loadYaml, + makeKvmConfig, + makeMultiHostKvmConfig, + resolvePath, + setOutputPaths, + writeYaml, +) +from .utils import chmodScripts, copyTree + + +def runCommand(args: list[str], *, cwd: str | Path | None = None) -> subprocess.CompletedProcess: + """Run one external command and fail immediately on errors. + + Args: + args: Command argv. Shell expansion is intentionally avoided. + cwd: Optional working directory. + """ + print("+ " + shlex.join(args)) + return subprocess.run(args, cwd=str(cwd) if cwd is not None else None, check=True) + + +@contextmanager +def temporaryWorkDir(prefix: str, keep_temp: bool = False) -> Iterator[Path]: + """Create a temporary k8sTools work directory. + + Args: + prefix: Prefix passed to tempfile. + keep_temp: Leave the directory on disk for debugging when true. + """ + if keep_temp: + root = Path(tempfile.mkdtemp(prefix=prefix)) + print(f"[k8sTools] keeping temporary work directory: {root}") + yield root + return + with tempfile.TemporaryDirectory(prefix=prefix) as tmp: + yield Path(tmp) + + +def inferImageRegistryPrefix(output_dir: Path) -> str: + """Infer the compiler image prefix from generated image metadata. + + Args: + output_dir: Compile output directory containing images.yaml or images.txt. + """ + images_path = output_dir / "images.yaml" + if images_path.exists(): + data = loadYaml(images_path) + images = data.get("images") if isinstance(data.get("images"), list) else [] + for item in images: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + if "/" in name: + return name.rsplit("/", 1)[0] + return "seedemu" + + images_txt = output_dir / "images.txt" + if images_txt.exists(): + for line in images_txt.read_text(encoding="utf-8").splitlines(): + name = line.strip() + if not name or name.startswith("#"): + continue + if "/" in name: + return name.rsplit("/", 1)[0] + return "seedemu" + return "seedemu" + + +def buildCluster( + input_config: str | Path, + config_k3s: str | Path, + kubeconfig: str | Path, + *, + inventory: str | Path | None = None, + keep_temp: bool = False, +) -> None: + """Build KVM/physical infrastructure and write final config outputs. + + Args: + input_config: User YAML with explicit `kind`. + config_k3s: Final configK3s.yaml path requested by the user. + kubeconfig: Final kubeconfig path requested by the user. + inventory: Optional final cluster inventory YAML path requested by the user. + keep_temp: Keep temporary setup resources for debugging. + + The first-phase implementation expands bundled setup resources into a + temporary directory, reuses the existing resource scripts there, copies the + resulting config back to the requested paths, and removes the temporary + directory. No persistent setup/running directory is created. + """ + input_path = resolvePath(input_config) + config_path = resolvePath(config_k3s) + kubeconfig_path = resolvePath(kubeconfig) + inventory_path = resolvePath(inventory) if inventory is not None else None + source = loadYaml(input_path) + kind = detectKind(source, input_path) + + with temporaryWorkDir("seedemu-k8s-tools-build-", keep_temp) as root: + setup_dir = copyTree("setup", root / "setup", overwrite=True) + chmodScripts(setup_dir) + if kind == "kvmOvn": + _buildKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) + elif kind == "multiHostKvmOvn": + _buildMultiHostKvmOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) + elif kind == "physicalOvn": + _buildPhysicalOvn(setup_dir, input_path, config_path, kubeconfig_path, inventory_path) + else: + raise ValueError(f"unsupported kind: {kind}") + + +def deployWorkload( + output_dir: str | Path, + kubeconfig: str | Path, + config_k3s: str | Path, + *, + image_registry_prefix: str | None = None, + keep_temp: bool = False, +) -> None: + """Run preflight, image build/push, deploy, and wait for workload readiness. + + Args: + output_dir: Compile output directory containing images.yaml and k8s.yaml. + kubeconfig: Kubeconfig used by kubectl. + config_k3s: configK3s.yaml used to resolve registry and network backend. + image_registry_prefix: Logical compiler image prefix. If omitted, it + is inferred from images.yaml. + keep_temp: Keep temporary running resources for debugging. + """ + output_path = resolvePath(output_dir) + kubeconfig_path = resolvePath(kubeconfig) + config_path = resolvePath(config_k3s) + logical_image_prefix = image_registry_prefix or inferImageRegistryPrefix(output_path) + with temporaryWorkDir("seedemu-k8s-tools-running-", keep_temp) as root: + running_dir = copyTree("running", root / "running", overwrite=True) + chmodScripts(running_dir) + config_copy = root / "configK3s.yaml" + setup_config = loadYaml(config_path) + setup_config.setdefault("outputs", {})["kubeconfig"] = str(kubeconfig_path) + writeYaml(config_copy, setup_config) + writeYaml( + running_dir / "configRunning.yaml", + { + "setupConfig": str(config_copy), + "outputDir": str(output_path), + "imageRegistryPrefix": logical_image_prefix, + "rolloutTimeoutSeconds": 1800, + }, + ) + stage = running_dir / "manageRunningStage.py" + running_config = running_dir / "configRunning.yaml" + runCommand(["python3", str(stage), "--config", str(running_config), "preflight"], cwd=running_dir) + runCommand(["python3", str(stage), "--config", str(running_config), "build"], cwd=running_dir) + runCommand(["python3", str(stage), "--config", str(running_config), "up"], cwd=running_dir) + + +def cleanWorkload(output_dir: str | Path, kubeconfig: str | Path, *, keep_temp: bool = False) -> None: + """Delete workload resources without requiring configK3s.yaml. + + Args: + output_dir: Compile output directory. + kubeconfig: Kubeconfig used by kubectl. + keep_temp: Keep temporary cleanup resources for debugging. + """ + output_path = resolvePath(output_dir) + kubeconfig_path = resolvePath(kubeconfig) + with temporaryWorkDir("seedemu-k8s-tools-clean-", keep_temp) as root: + running_dir = copyTree("running", root / "running", overwrite=True) + chmodScripts(running_dir) + helper = running_dir / "manageK8sManifest.py" + manifest = output_path / "k8s.kube-ovn.yaml" + if not manifest.exists(): + manifest = output_path / "k8s.yaml" + namespace = subprocess.check_output( + ["python3", str(helper), "namespace", "--manifest", str(manifest)], + text=True, + ).strip() + if (output_path / "kustomization.yaml").exists(): + runCommand( + [ + "kubectl", + "--kubeconfig", + str(kubeconfig_path), + "delete", + "-k", + str(output_path), + "--ignore-not-found=true", + "--wait=false", + ] + ) + runCommand( + [ + "kubectl", + "--kubeconfig", + str(kubeconfig_path), + "delete", + "namespace", + namespace, + "--ignore-not-found=true", + "--wait=false", + ] + ) + + +def destroyCluster(config_k3s: str | Path, *, keep_temp: bool = False) -> None: + """Destroy K3s/OVN and optional KVM resources recorded in configK3s.yaml. + + Args: + config_k3s: configK3s.yaml produced by buildCluster(). + keep_temp: Keep temporary destroy resources for debugging. + """ + config_path = resolvePath(config_k3s) + config = loadYaml(config_path) + destroy = config.get("k8sTools", {}).get("destroy", {}) + destroy_type = str(destroy.get("type") or config.get("kind") or "") + if not destroy_type: + raise ValueError(f"{config_path} does not contain k8sTools.destroy metadata") + with temporaryWorkDir("seedemu-k8s-tools-destroy-", keep_temp) as root: + setup_dir = copyTree("setup", root / "setup", overwrite=True) + chmodScripts(setup_dir) + temp_config = setup_dir / "configK3s.yaml" + writeYaml(temp_config, config) + try: + runCommand(["python3", str(setup_dir / "destroyPhysicalCluster.py"), str(temp_config)], cwd=setup_dir) + except subprocess.CalledProcessError: + if destroy_type == "physicalOvn": + raise + print("[k8sTools] warning: K3s/OVN cleanup failed; continuing with KVM cleanup") + if destroy_type == "kvmOvn": + state = destroy.get("state") + setup_config = destroy.get("config") + if not isinstance(state, dict) or not isinstance(setup_config, dict): + raise ValueError("kvmOvn destroy requires embedded kvm state and config") + writeYaml(setup_dir / "kvm.yaml", setup_config) + writeYaml(setup_dir / "kvmState.yaml", state) + runCommand(["python3", str(setup_dir / "kvm" / "destroyKvmVms.py"), str(setup_dir / "kvmState.yaml")], cwd=setup_dir) + _removeManagedKvmDataDir(state) + elif destroy_type == "multiHostKvmOvn": + state = destroy.get("state") + if not isinstance(state, dict): + raise ValueError("multiHostKvmOvn destroy requires embedded multi-host state") + writeYaml(setup_dir / "multiHostKvmState.yaml", state) + runCommand( + [ + "python3", + str(setup_dir / "multiHostKvm" / "destroyMultiHostKvmVms.py"), + str(setup_dir / "multiHostKvmState.yaml"), + ], + cwd=setup_dir, + ) + elif destroy_type == "physicalOvn": + return + else: + raise ValueError(f"unsupported destroy type: {destroy_type}") + + +def _buildKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: + """Build a single-hypervisor KVM cluster with Kube-OVN.""" + kvm_config = makeKvmConfig( + config=input_path, + setup_dir=setup_dir, + k3s_config_path=setup_dir / "configK3s.yaml", + kubeconfig_path=kubeconfig_path, + inventory_path=inventory_path or setup_dir / "cluster.inventory.yaml", + tmp_dir=setup_dir / "tmp", + ) + kvm_config.setdefault("fabric", {})["type"] = "ovn" + _applyOvnK3sDefaults(kvm_config) + writeYaml(setup_dir / "kvm.yaml", kvm_config) + runCommand(["python3", str(setup_dir / "kvm" / "prepareHostAssets.py"), str(setup_dir / "kvm.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "kvm" / "createKvmVms.py"), str(setup_dir / "kvm.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "kvm" / "tuneVmLimits.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "applyK3sCluster.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + final_config = loadYaml(setup_dir / "configK3s.yaml") + addK8sToolsMetadata( + final_config, + kind="kvmOvn", + source_config=input_path, + kubeconfig=kubeconfig_path, + destroy_type="kvmOvn", + destroy_state=loadYaml(setup_dir / "kvmState.yaml"), + destroy_config=kvm_config, + ) + writeYaml(config_path, final_config) + + +def _buildMultiHostKvmOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: + """Build a multi-hypervisor KVM cluster with routed VM subnets and Kube-OVN.""" + source = loadYaml(input_path) + source.setdefault("fabric", {})["type"] = "ovn" + _applyOvnK3sDefaults(source) + outputs = source.setdefault("outputs", {}) + outputs["k3sConfig"] = str(setup_dir / "configK3s.yaml") + outputs["multiHostKvmState"] = str(setup_dir / "multiHostKvmState.yaml") + setOutputPaths( + source, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) + temp_input = setup_dir / "multiHostInput.yaml" + writeYaml(temp_input, source) + kvm_config = makeMultiHostKvmConfig(config=temp_input, setup_dir=setup_dir) + writeYaml(setup_dir / "kvm.yaml", kvm_config) + runCommand(["python3", str(setup_dir / "multiHostKvm" / "prepareKvmHypervisors.py"), str(setup_dir / "kvm.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "multiHostKvm" / "createMultiHostKvmVms.py"), str(setup_dir / "kvm.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "kvm" / "tuneVmLimits.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "applyK3sCluster.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + final_config = loadYaml(setup_dir / "configK3s.yaml") + addK8sToolsMetadata( + final_config, + kind="multiHostKvmOvn", + source_config=input_path, + kubeconfig=kubeconfig_path, + destroy_type="multiHostKvmOvn", + destroy_state=loadYaml(setup_dir / "multiHostKvmState.yaml"), + destroy_config=kvm_config, + ) + writeYaml(config_path, final_config) + + +def _buildPhysicalOvn( + setup_dir: Path, + input_path: Path, + config_path: Path, + kubeconfig_path: Path, + inventory_path: Path | None, +) -> None: + """Build a K3s + Kube-OVN cluster on existing physical nodes.""" + config = loadYaml(input_path) + config["kind"] = "physicalOvn" + config.setdefault("fabric", {})["type"] = "ovn" + _applyOvnK3sDefaults(config) + setOutputPaths( + config, + kubeconfig=kubeconfig_path, + tmp_dir=setup_dir / "tmp", + inventory=inventory_path or setup_dir / "cluster.inventory.yaml", + ) + writeYaml(setup_dir / "configK3s.yaml", config) + runCommand(["python3", str(setup_dir / "preparePhysicalNodes.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + runCommand(["python3", str(setup_dir / "applyK3sCluster.py"), str(setup_dir / "configK3s.yaml")], cwd=setup_dir) + final_config = loadYaml(setup_dir / "configK3s.yaml") + addK8sToolsMetadata( + final_config, + kind="physicalOvn", + source_config=input_path, + kubeconfig=kubeconfig_path, + destroy_type="physicalOvn", + ) + writeYaml(config_path, final_config) + + +def _applyOvnK3sDefaults(config: dict[str, Any]) -> None: + """Set K3s defaults required by the bundled Kube-OVN chart. + + Args: + config: User input or generated setup config mapping. + """ + fabric_type = str(config.get("fabric", {}).get("type") if isinstance(config.get("fabric"), dict) else "").lower() + if fabric_type not in {"ovn", "kube-ovn", "kube_ovn"}: + return + k3s = config.setdefault("k3s", {}) + if isinstance(k3s, dict): + k3s.setdefault("version", "v1.29.15+k3s1") + + +def _removeManagedKvmDataDir(state: dict[str, Any]) -> None: + """Remove the k8sTools-managed /data work directory after VM destruction. + + Args: + state: Embedded kvmState.yaml mapping produced by createKvmVms.py. + + Only directories created from the k8sTools temporary-build naming pattern + are removed. User-provided KVM storage paths are intentionally preserved. + """ + kvm = state.get("kvm") if isinstance(state.get("kvm"), dict) else {} + disk_dir = kvm.get("diskDir") + if not disk_dir: + return + data_dir = Path(str(disk_dir)).expanduser().resolve().parent + parts = data_dir.parts + if len(parts) < 4 or parts[1] != "data" or parts[-1].startswith("seedemu-k8s-tools-build-") is False: + return + print(f"[k8sTools] removing managed KVM data dir: {data_dir}") + shutil.rmtree(data_dir, ignore_errors=True) diff --git a/seedemu/k8sTools/utils.py b/seedemu/k8sTools/utils.py new file mode 100644 index 000000000..6b30f3d2b --- /dev/null +++ b/seedemu/k8sTools/utils.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import shutil +import stat +from importlib.resources import as_file, files +from pathlib import Path + + +def resourcePath(resource_name: str): + """Return the package resource path for a bundled setup/running tree.""" + return files("seedemu.k8sTools.resources").joinpath(resource_name) + + +def copyTree(resource_name: str, target_dir: str | Path, overwrite: bool = False) -> Path: + """Copy one bundled resource tree to a user-visible directory. + + Args: + resource_name: Resource tree name, for example "setup" or "running". + target_dir: Destination directory. + overwrite: Remove the destination first when true. + """ + target = Path(target_dir).expanduser() + if target.exists(): + if not overwrite: + raise FileExistsError(f"{target} already exists; use overwrite=True to replace it") + shutil.rmtree(target) + + with as_file(resourcePath(resource_name)) as source: + shutil.copytree(str(source), str(target)) + return target + + +def copyResourceItems( + resource_name: str, + item_names: list[str], + target_dir: str | Path, + overwrite: bool = False, +) -> Path: + """Copy selected files/directories from one bundled resource tree. + + Args: + resource_name: Resource tree name, for example "setup". + item_names: Relative file or directory names to copy from the resource. + target_dir: Destination directory. + overwrite: Replace destination items with the same relative name. + """ + target = Path(target_dir).expanduser() + target.mkdir(parents=True, exist_ok=True) + with as_file(resourcePath(resource_name)) as source: + source_root = Path(source) + for item_name in item_names: + source_item = source_root / item_name + target_item = target / item_name + if not source_item.exists(): + raise FileNotFoundError(f"Missing package resource: {resource_name}/{item_name}") + if target_item.exists(): + if not overwrite: + continue + if target_item.is_dir(): + shutil.rmtree(target_item) + else: + target_item.unlink() + target_item.parent.mkdir(parents=True, exist_ok=True) + if source_item.is_dir(): + shutil.copytree(str(source_item), str(target_item)) + else: + shutil.copy2(str(source_item), str(target_item)) + return target + + +def chmodScripts(path: str | Path) -> None: + """Make every bundled Python script entrypoint below path executable. + + Args: + path: Root directory copied from package resources. + """ + root = Path(path).expanduser() + for script in root.rglob("*.py"): + mode = script.stat().st_mode + script.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def chmodSingleScript(path: str | Path) -> None: + """Make one generated script executable. + + Args: + path: Generated script path. + """ + script = Path(path).expanduser() + mode = script.stat().st_mode + script.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def writeExecutableScript(path: str | Path, content: str) -> Path: + """Write one executable shell script. + + Args: + path: Script path. + content: Full script content. + """ + script = Path(path).expanduser() + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text(content, encoding="utf-8") + chmodSingleScript(script) + return script diff --git a/seedemu/layers/Ebgp.py b/seedemu/layers/Ebgp.py index 66cad84b6..b63425936 100644 --- a/seedemu/layers/Ebgp.py +++ b/seedemu/layers/Ebgp.py @@ -1,44 +1,19 @@ from __future__ import annotations -from .Routing import Router -from seedemu.core import Registry, ScopedRegistry, Network, Interface, Graphable, Emulator, Layer +from seedemu.core import Registry, ScopedRegistry, Network, Interface, Graphable, Emulator, Layer, Router from seedemu.core.enums import NodeRole -from typing import Tuple, List, Dict +from typing import Tuple, List, Dict, Optional from enum import Enum +from ._bgp_metadata import ( + BGP_COMMUNITY_CUSTOMER, + BGP_COMMUNITY_PEER, + BGP_COMMUNITY_PROVIDER, + BGP_EXPORT_ALL, + BGP_EXPORT_LOCAL_AND_CUSTOMER, + install_router_bgp_session, +) EbgpFileTemplates: Dict[str, str] = {} -EbgpFileTemplates["bgp_commons"] = """\ -define LOCAL_COMM = ({localAsn}, 0, 0); -define CUSTOMER_COMM = ({localAsn}, 1, 0); -define PEER_COMM = ({localAsn}, 2, 0); -define PROVIDER_COMM = ({localAsn}, 3, 0); -""" - -EbgpFileTemplates["rs_bird_peer"] = """ - ipv4 {{ - import all; - export all; - }}; - rs client; - local {localAddress} as {localAsn}; - neighbor {peerAddress} as {peerAsn}; -""" - -EbgpFileTemplates["rnode_bird_peer"] = """ - ipv4 {{ - table t_bgp; - import filter {{ - bgp_large_community.add({importCommunity}); - bgp_local_pref = {bgpPref}; - accept; - }}; - export {exportFilter}; - next hop self; - }}; - local {localAddress} as {localAsn}; - neighbor {peerAddress} as {peerAsn}; -""" - class PeerRelationship(Enum): """! @brief Relationship between peers. @@ -62,7 +37,9 @@ class Ebgp(Layer, Graphable): """ __peerings: Dict[Tuple[int, int, int], PeerRelationship] + __peering_routers: Dict[Tuple[int, int, int], Tuple[Optional[str], Optional[str]]] __rs_peers: List[Tuple[int, int]] + __rs_peer_routers: Dict[Tuple[int, int], str] __xc_peerings: Dict[Tuple[int, int], PeerRelationship] def __init__(self): @@ -71,17 +48,51 @@ def __init__(self): """ super().__init__() self.__peerings = {} + self.__peering_routers = {} self.__xc_peerings = {} self.__rs_peers = [] + self.__rs_peer_routers = {} self.addDependency('Routing', False, False) + def __recordPeer( + self, + node: Router, + name: str, + localAddress: str, + localAsn: int, + peerAddress: str, + peerAsn: int, + importCommunity: str = None, + bgpPref: int = None, + exportPolicy: str = BGP_EXPORT_ALL, + nextHopSelf: bool = True, + routeServerClient: bool = False, + ) -> None: + install_router_bgp_session( + node, + { + "name": name, + "kind": "ebgp", + "local_address": str(localAddress), + "local_asn": localAsn, + "peer_address": str(peerAddress), + "peer_asn": peerAsn, + "import_community": importCommunity, + "local_pref": bgpPref, + "export_policy": exportPolicy, + "next_hop_self": nextHopSelf, + "route_server_client": routeServerClient, + }, + ) + def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel: PeerRelationship) -> None: rsNode: Router = None routerA: Router = None routerB: Router = None - # for both nodes + # The Ebgp layer records BGP intent only. Routing decides later whether + # those sessions become BIRD protocol blocks or FRR configuration. for node in [nodeA, nodeB]: if node.getRegistryInfo()[1] == 'rs': # getRole() would be BorderRouter not RouteServer here @@ -91,113 +102,85 @@ def __createPeer(self, nodeA: Router, nodeB: Router, addrA: str, addrB: str, rel if routerA == None: routerA = node elif routerB == None: routerB = node - if not node.getAttribute('__bgp_bootstrapped', False): - self._log('Bootstrapping as{}/{} for BGP...'.format(node.getAsn(), node.getName())) - - node.setAttribute('__bgp_bootstrapped', True) - node.appendFile('/etc/bird/bird.conf', EbgpFileTemplates['bgp_commons'].format(localAsn = node.getAsn())) - - # create table for bgp - node.addTable('t_bgp') - - # pipe all routes in bgp table to main table - node.addTablePipe('t_bgp') - - # pipe direct routes to bgp, set LOCAL community, set pref 40 - node.addTablePipe('t_direct', 't_bgp', exportFilter = 'filter { bgp_large_community.add(LOCAL_COMM); bgp_local_pref = 40; accept; }') - - - assert routerA != None, 'both nodes are RS node. cannot setup peering.' assert routerA != routerB, 'cannot peer with oneself.' if rsNode != None: - rsNode.addProtocol('bgp', 'p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rs_bird_peer"].format( - localAddress = addrA, - localAsn = rsNode.getAsn(), - peerAddress = addrB, - peerAsn = routerA.getAsn() - )) - - routerA.addProtocol('bgp', 'p_rs{}'.format(rsNode.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrB, - localAsn = routerA.getAsn(), - peerAddress = addrA, - peerAsn = rsNode.getAsn(), - exportFilter = "where bgp_large_community ~ [LOCAL_COMM, CUSTOMER_COMM]", - importCommunity = "PEER_COMM", - bgpPref = 20 - )) + self.__recordPeer( + rsNode, + 'p_as{}'.format(routerA.getAsn()), + addrA, + rsNode.getAsn(), + addrB, + routerA.getAsn(), + nextHopSelf=False, + routeServerClient=True, + ) + + self.__recordPeer( + routerA, + 'p_rs{}'.format(rsNode.getAsn()), + addrB, + routerA.getAsn(), + addrA, + rsNode.getAsn(), + importCommunity=BGP_COMMUNITY_PEER, + bgpPref=20, + exportPolicy=BGP_EXPORT_LOCAL_AND_CUSTOMER, + ) return if rel == PeerRelationship.Peer: - routerA.addProtocol('bgp', 'p_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrA, - localAsn = routerA.getAsn(), - peerAddress = addrB, - peerAsn = routerB.getAsn(), - exportFilter = "where bgp_large_community ~ [LOCAL_COMM, CUSTOMER_COMM]", - importCommunity = "PEER_COMM", - bgpPref = 20 - )) - - routerB.addProtocol('bgp', 'p_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrB, - localAsn = routerB.getAsn(), - peerAddress = addrA, - peerAsn = routerA.getAsn(), - exportFilter = "where bgp_large_community ~ [LOCAL_COMM, CUSTOMER_COMM]", - importCommunity = "PEER_COMM", - bgpPref = 20 - )) + self.__recordPeer(routerA, 'p_as{}'.format(routerB.getAsn()), addrA, routerA.getAsn(), addrB, routerB.getAsn(), BGP_COMMUNITY_PEER, 20, BGP_EXPORT_LOCAL_AND_CUSTOMER) + self.__recordPeer(routerB, 'p_as{}'.format(routerA.getAsn()), addrB, routerB.getAsn(), addrA, routerA.getAsn(), BGP_COMMUNITY_PEER, 20, BGP_EXPORT_LOCAL_AND_CUSTOMER) if rel == PeerRelationship.Provider: - routerA.addProtocol('bgp', 'c_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrA, - localAsn = routerA.getAsn(), - peerAddress = addrB, - peerAsn = routerB.getAsn(), - exportFilter = "all", - importCommunity = "CUSTOMER_COMM", - bgpPref = 30 - )) - - routerB.addProtocol('bgp', 'u_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrB, - localAsn = routerB.getAsn(), - peerAddress = addrA, - peerAsn = routerA.getAsn(), - exportFilter = "where bgp_large_community ~ [LOCAL_COMM, CUSTOMER_COMM]", - importCommunity = "PROVIDER_COMM", - bgpPref = 10 - )) + self.__recordPeer(routerA, 'c_as{}'.format(routerB.getAsn()), addrA, routerA.getAsn(), addrB, routerB.getAsn(), BGP_COMMUNITY_CUSTOMER, 30, BGP_EXPORT_ALL) + self.__recordPeer(routerB, 'u_as{}'.format(routerA.getAsn()), addrB, routerB.getAsn(), addrA, routerA.getAsn(), BGP_COMMUNITY_PROVIDER, 10, BGP_EXPORT_LOCAL_AND_CUSTOMER) if rel == PeerRelationship.Unfiltered: - routerA.addProtocol('bgp', 'x_as{}'.format(routerB.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrA, - localAsn = routerA.getAsn(), - peerAddress = addrB, - peerAsn = routerB.getAsn(), - exportFilter = "all", - importCommunity = "CUSTOMER_COMM", - bgpPref = 30 - )) - - routerB.addProtocol('bgp', 'x_as{}'.format(routerA.getAsn()), EbgpFileTemplates["rnode_bird_peer"].format( - localAddress = addrB, - localAsn = routerB.getAsn(), - peerAddress = addrA, - peerAsn = routerA.getAsn(), - exportFilter = "all", - importCommunity = "PROVIDER_COMM", - bgpPref = 10 - )) + self.__recordPeer(routerA, 'x_as{}'.format(routerB.getAsn()), addrA, routerA.getAsn(), addrB, routerB.getAsn(), BGP_COMMUNITY_CUSTOMER, 30, BGP_EXPORT_ALL) + self.__recordPeer(routerB, 'x_as{}'.format(routerA.getAsn()), addrB, routerB.getAsn(), addrA, routerA.getAsn(), BGP_COMMUNITY_PROVIDER, 10, BGP_EXPORT_ALL) + + def __selectIxRouter( + self, + candidates: List[Router], + ixNet: Network, + ix: int, + asn: int, + routerName: Optional[str] = None, + ) -> Tuple[Router, Interface]: + """Resolve the router/interface used by an IX peering intent.""" + if routerName is not None: + for node in candidates: + if node.getName() != routerName: + continue + for iface in node.getInterfaces(): + if iface.getNet() == ixNet: + return node, iface + assert False, 'explicit peering router as{}/{} is not connected to ix{}'.format(asn, routerName, ix) + assert False, 'explicit peering router as{}/{} does not exist for ix{}'.format(asn, routerName, ix) + + for node in candidates: + for iface in node.getInterfaces(): + if iface.getNet() == ixNet: + return node, iface + + assert False, 'cannot resolve peering: as{} not in ix{}'.format(asn, ix) def getName(self) -> str: return "Ebgp" - def addPrivatePeering(self, ix: int, a: int, b: int, abRelationship: PeerRelationship = PeerRelationship.Peer) -> Ebgp: + def addPrivatePeering( + self, + ix: int, + a: int, + b: int, + abRelationship: PeerRelationship = PeerRelationship.Peer, + aRouter: str = None, + bRouter: str = None, + ) -> Ebgp: """! @brief Setup private peering between two ASes in IX. @@ -218,9 +201,37 @@ def addPrivatePeering(self, ix: int, a: int, b: int, abRelationship: PeerRelatio assert abRelationship == PeerRelationship.Peer or abRelationship == PeerRelationship.Provider or abRelationship == PeerRelationship.Unfiltered, 'unknown peering relationship {}'.format(abRelationship) self.__peerings[(ix, a, b)] = abRelationship + if aRouter is not None or bRouter is not None: + self.__peering_routers[(ix, a, b)] = (aRouter, bRouter) return self + def addPrivatePeeringByRouters( + self, + ix: int, + a: int, + aRouter: str, + b: int, + bRouter: str, + abRelationship: PeerRelationship = PeerRelationship.Peer, + ) -> Ebgp: + """! + @brief Setup private peering using explicit router names on the IX. + + This is the preferred API when one AS has multiple routers attached to + the same IX and the peering must bind to a specific edge router. + + @param ix IXP id. + @param a First ASN. + @param aRouter router in AS a connected to the IX. + @param b Second ASN. + @param bRouter router in AS b connected to the IX. + @param abRelationship peering relationship. + + @returns self, for chaining API calls. + """ + return self.addPrivatePeering(ix, a, b, abRelationship, aRouter, bRouter) + def addPrivatePeerings(self, ix: int, a_asns: List[int], b_asns: List[int], abRelationship: PeerRelationship = PeerRelationship.Peer) -> Ebgp: """! @brief Setup private peering between two sets of ASes in IX. @@ -251,6 +262,12 @@ def getPrivatePeerings(self) -> Dict[Tuple[int, int, int], PeerRelationship]: """ return self.__peerings + def getPrivatePeeringRouters(self) -> Dict[Tuple[int, int, int], Tuple[Optional[str], Optional[str]]]: + """! + @brief Get explicit router selections for private IX peerings. + """ + return dict(self.__peering_routers) + def addCrossConnectPeering(self, a: int, b: int, abRelationship: PeerRelationship = PeerRelationship.Peer) -> Ebgp: """! @brief add cross-connect peering. @@ -282,12 +299,13 @@ def getCrossConnectPeerings(self) -> Dict[Tuple[int, int], PeerRelationship]: """ return self.__xc_peerings - def addRsPeer(self, ix: int, peer: int) -> Ebgp: + def addRsPeer(self, ix: int, peer: int, routerName: str = None) -> Ebgp: """! @brief Setup RS peering for an AS. @param ix IXP id. @param peer Participant ASN. + @param routerName optional router in the participant AS connected to the IX. @throws AssertionError if peering already exist. @@ -296,9 +314,23 @@ def addRsPeer(self, ix: int, peer: int) -> Ebgp: assert (ix, peer) not in self.__rs_peers, '{} already peered with RS at IX{}'.format(peer, ix) self.__rs_peers.append((ix, peer)) + if routerName is not None: + self.__rs_peer_routers[(ix, peer)] = routerName return self + def addRsPeerByRouter(self, ix: int, peer: int, routerName: str) -> Ebgp: + """! + @brief Setup RS peering using an explicit participant router name. + + @param ix IXP id. + @param peer Participant ASN. + @param routerName router in the participant AS connected to the IX. + + @returns self, for chaining API calls. + """ + return self.addRsPeer(ix, peer, routerName) + def addRsPeers(self, ix: int, peers: List[int]): """! @brief Setup RS peering for list of ASes. @@ -323,6 +355,12 @@ def getRsPeers(self) -> List[Tuple[int, int]]: """ return self.__rs_peers + def getRsPeerRouters(self) -> Dict[Tuple[int, int], str]: + """! + @brief Get explicit router selections for RS peerings. + """ + return dict(self.__rs_peer_routers) + def configure(self, emulator: Emulator) -> None: reg = emulator.getRegistry() @@ -337,17 +375,13 @@ def configure(self, emulator: Emulator) -> None: rs_if = rs_ifs[0] p_rnodes: List[Router] = p_reg.getByType('brdnode') - p_ixnode: Router = None - p_ixif: Interface = None - for node in p_rnodes: - if p_ixnode != None: break - for iface in node.getInterfaces(): - if iface.getNet() == ix_net: - p_ixnode = node - p_ixif = iface - break - - assert p_ixnode != None, 'cannot resolve peering: as{} not in ix{}'.format(peer, ix) + p_ixnode, p_ixif = self.__selectIxRouter( + p_rnodes, + ix_net, + ix, + peer, + self.__rs_peer_routers.get((ix, peer)), + ) self._log("adding peering: {} as {} (RS) <-> {} as {}".format(rs_if.getAddress(), ix, p_ixif.getAddress(), peer)) self.__createPeer(ix_rs, p_ixnode, rs_if.getAddress(), p_ixif.getAddress(), PeerRelationship.Peer) @@ -395,30 +429,9 @@ def configure(self, emulator: Emulator) -> None: ix_net: Network = ix_reg.get('net', 'ix{}'.format(ix)) a_rnodes: List[Router] = a_reg.getByType('rnode') b_rnodes: List[Router] = b_reg.getByType('rnode') - - a_ixnode: Router = None - a_ixif: Interface = None - for node in a_rnodes: - if a_ixnode != None: break - for iface in node.getInterfaces(): - if iface.getNet() == ix_net: - a_ixnode = node - a_ixif = iface - break - - assert a_ixnode != None, 'cannot resolve peering: as{} not in ix{}'.format(a, ix) - - b_ixnode: Router = None - b_ixif: Interface = None - for node in b_rnodes: - if b_ixnode != None: break - for iface in node.getInterfaces(): - if iface.getNet() == ix_net: - b_ixnode = node - b_ixif = iface - break - - assert b_ixnode != None, 'cannot resolve peering: as{} not in ix{}'.format(b, ix) + a_router, b_router = self.__peering_routers.get((ix, a, b), (None, None)) + a_ixnode, a_ixif = self.__selectIxRouter(a_rnodes, ix_net, ix, a, a_router) + b_ixnode, b_ixif = self.__selectIxRouter(b_rnodes, ix_net, ix, b, b_router) self._log("adding IX peering: {} as {} <-({})-> {} as {}".format(a_ixif.getAddress(), a, rel, b_ixif.getAddress(), b)) diff --git a/seedemu/layers/Ibgp.py b/seedemu/layers/Ibgp.py index 2f8b9d438..ef75ac14a 100644 --- a/seedemu/layers/Ibgp.py +++ b/seedemu/layers/Ibgp.py @@ -2,26 +2,26 @@ from seedemu.core.enums import NetworkType, NodeRole from .Base import Base from seedemu.core import ScopedRegistry, Node, Graphable, Emulator, Layer -from typing import List, Set, Dict +from typing import Dict, List, Set, Tuple +from ._bgp_metadata import install_router_bgp_session -IbgpFileTemplates: Dict[str, str] = {} +IBGP_MODE_FULL_MESH = "full-mesh" +IBGP_MODE_ROUTE_REFLECTOR = "route-reflector" +IBGP_MODE_DISABLED = "disabled" + +IBGP_MODES = { + IBGP_MODE_FULL_MESH, + IBGP_MODE_ROUTE_REFLECTOR, + IBGP_MODE_DISABLED, +} -IbgpFileTemplates['ibgp_peer'] = ''' - ipv4 {{ - table t_bgp; - import all; - export all; - igp table t_ospf; - }}; - local {localAddress} as {asn}; - neighbor {peerAddress} as {asn}; -''' class Ibgp(Layer, Graphable): """! @brief The Ibgp (iBGP) layer. - This layer automatically setup full mesh peering between routers within AS. + This layer automatically sets up full-mesh iBGP or Route Reflector based + iBGP sessions between routers within each AS. """ __masked: Set[int] @@ -89,45 +89,219 @@ def getMaskedAsns(self) -> Set[int]: """ return self.__masked - def render(self, emulator: Emulator): + def __is_ibgp_disabled(self, router: Node) -> bool: + return hasattr(router, "isControlPlaneDisabled") and router.isControlPlaneDisabled("ibgp") + + def __get_igp_table(self, emulator: Emulator, asn: int) -> str: + reg = emulator.getRegistry() + if reg.has('seedemu', 'layer', 'Mpls'): + mpls = reg.get('seedemu', 'layer', 'Mpls') + if hasattr(mpls, "getEnabled") and asn in mpls.getEnabled(): + return "master4" + return "t_ospf" + + def configure(self, emulator: Emulator): reg = emulator.getRegistry() base: Base = reg.get('seedemu', 'layer', 'Base') for asn in base.getAsns(): - if asn in self.__masked: continue + asobj = base.getAutonomousSystem(asn) + if int(asn) in self.__masked: + asobj.setIbgpMode(IBGP_MODE_DISABLED) + + asobj.completeIbgpSetup() + mode = asobj.getIbgpMode() + if mode == IBGP_MODE_DISABLED: continue self._log('setting up IBGP peering for as{}...'.format(asn)) routers: List[Node] = ScopedRegistry(str(asn), reg).getByType('rnode') + routers_map: Dict[str, Node] = {router.getName(): router for router in routers} + participant_names = asobj.getIbgpParticipants() + disabled = {name for name, router in routers_map.items() if self.__is_ibgp_disabled(router)} + active_names = participant_names - disabled + active_routers_map = { + name: router for name, router in routers_map.items() + if name in active_names + } + igp_table = self.__get_igp_table(emulator, asn) + + if mode == IBGP_MODE_ROUTE_REFLECTOR: + clusters = asobj._aggregateBgpClusters() + self._render_rr_mode(asn, clusters, active_routers_map, igp_table) + continue + + self._render_full_mesh_mode( + asn, + [active_routers_map[name] for name in sorted(active_routers_map.keys())], + igp_table, + ) + + def _render_rr_mode(self, asn: int, clusters: Dict[str, Tuple[Set[str], Set[str]]], routers_map: Dict[str, Node], igp_table: str = "t_ospf"): + """! + @brief Render Route Reflector based iBGP sessions for one AS. + + @param asn AS number being rendered. + @param clusters mapping from cluster ID to RR names and client names. + @param routers_map mapping from router name to router node. + """ + self._log('setting up IBGP (Route Reflector) for as{}...'.format(asn)) - for local in routers: - self._log('setting up IBGP peering on as{}/{}...'.format(asn, local.getName())) + all_rr_names: Set[str] = set() - remotes = [] - self.__dfs(local, remotes) + for cluster_id, (rr_names, client_names) in clusters.items(): + all_rr_names.update(rr_names) - n = 1 - for remote in remotes: - if local == remote: continue + for rr_name in sorted(rr_names): + if rr_name not in routers_map: + continue + + rr_node = routers_map[rr_name] + rr_address = rr_node.getLoopbackAddress() + + for client_name in sorted(client_names): + if client_name not in routers_map: + continue + + client_node = routers_map[client_name] + client_address = client_node.getLoopbackAddress() + install_router_bgp_session( + rr_node, + { + "name": "Ibgp_rr_client_{}".format(client_name), + "kind": "ibgp", + "local_address": str(rr_address), + "local_asn": asn, + "peer_address": str(client_address), + "peer_asn": asn, + "export_policy": "all", + "next_hop_self": False, + "igp_table": igp_table, + "passive": True, + "route_reflector_client": True, + "route_reflector_cluster_id": cluster_id, + }, + ) + install_router_bgp_session( + client_node, + { + "name": "Ibgp_rr_{}".format(rr_name), + "kind": "ibgp", + "local_address": str(client_address), + "local_asn": asn, + "peer_address": str(rr_address), + "peer_asn": asn, + "export_policy": "all", + "next_hop_self": True, + "igp_table": igp_table, + }, + ) + + self._log( + 'adding RR peering: {}(RR) <-> {}(client) cluster {} (as{})'.format( + rr_name, client_name, cluster_id, asn + ) + ) + + sorted_rrs = sorted( + [routers_map[name] for name in all_rr_names if name in routers_map], + key = lambda router: router.getName() + ) + + for i in range(len(sorted_rrs)): + for j in range(i + 1, len(sorted_rrs)): + node_a = sorted_rrs[i] + node_b = sorted_rrs[j] + + install_router_bgp_session( + node_a, + { + "name": "Ibgp_rr_mesh_{}".format(node_b.getName()), + "kind": "ibgp", + "local_address": str(node_a.getLoopbackAddress()), + "local_asn": asn, + "peer_address": str(node_b.getLoopbackAddress()), + "peer_asn": asn, + "export_policy": "all", + "next_hop_self": False, + "igp_table": igp_table, + }, + ) + install_router_bgp_session( + node_b, + { + "name": "Ibgp_rr_mesh_{}".format(node_a.getName()), + "kind": "ibgp", + "local_address": str(node_b.getLoopbackAddress()), + "local_asn": asn, + "peer_address": str(node_a.getLoopbackAddress()), + "peer_asn": asn, + "export_policy": "all", + "next_hop_self": False, + "igp_table": igp_table, + }, + ) + + self._log( + 'adding RR mesh peering: {} <-> {} (as{})'.format( + node_a.getName(), node_b.getName(), asn + ) + ) + + def _render_full_mesh_mode(self, asn: int, routers: List[Node], igp_table: str = "t_ospf"): + """! + @brief Render the legacy full-mesh iBGP sessions for one AS. - laddr = local.getLoopbackAddress() - raddr = remote.getLoopbackAddress() - local.addTable('t_bgp') - local.addTablePipe('t_bgp') - local.addTablePipe('t_direct', 't_bgp') - local.addProtocol('bgp', 'ibgp{}'.format(n), IbgpFileTemplates['ibgp_peer'].format( - localAddress = laddr, - peerAddress = raddr, - asn = asn - )) + @param asn AS number being rendered. + @param routers routers participating in the legacy iBGP mesh. + """ + self._log('setting up IBGP (Full Mesh) for as{}...'.format(asn)) + allowed_names = {router.getName() for router in routers} - n += 1 + for local in routers: + self._log('setting up IBGP peering on as{}/{}...'.format(asn, local.getName())) - self._log('adding peering: {} <-> {} (ibgp, as{})'.format(laddr, raddr, asn)) + remotes = [] + self.__dfs(local, remotes) + + n = 1 + for remote in remotes: + if local == remote: + continue + if remote.getName() not in allowed_names: + continue + + laddr = local.getLoopbackAddress() + raddr = remote.getLoopbackAddress() + install_router_bgp_session( + local, + { + "name": "ibgp{}".format(n), + "kind": "ibgp", + "local_address": str(laddr), + "local_asn": asn, + "peer_address": str(raddr), + "peer_asn": asn, + "export_policy": "all", + "next_hop_self": False, + "igp_table": igp_table, + }, + ) + + n += 1 + + self._log('adding peering: {} <-> {} (ibgp, as{})'.format(laddr, raddr, asn)) + + def render(self, emulator: Emulator): + pass def _doCreateGraphs(self, emulator: Emulator): base: Base = emulator.getRegistry().get('seedemu', 'layer', 'Base') for asn in base.getAsns(): - if asn in self.__masked: continue asobj = base.getAutonomousSystem(asn) + if int(asn) in self.__masked: + asobj.setIbgpMode(IBGP_MODE_DISABLED) + asobj.completeIbgpSetup() + mode = asobj.getIbgpMode() + if mode == IBGP_MODE_DISABLED: continue asobj.createGraphs(emulator) l2graph = asobj.getGraph('AS{}: Layer 2 Connections'.format(asn)) ibgpgraph = self._addGraph('AS{}: iBGP sessions'.format(asn), False) @@ -135,8 +309,33 @@ def _doCreateGraphs(self, emulator: Emulator): for edge in ibgpgraph.edges: edge.style = 'dotted' - rtrs = ScopedRegistry(str(asn), emulator.getRegistry()).getByType('rnode').copy() - + scope = ScopedRegistry(str(asn), emulator.getRegistry()) + rtrs = scope.getByType('rnode').copy() + routers_map: Dict[str, Node] = {router.getName(): router for router in rtrs} + disabled = {name for name, router in routers_map.items() if self.__is_ibgp_disabled(router)} + active_names = asobj.getIbgpParticipants() - disabled + + if mode == IBGP_MODE_ROUTE_REFLECTOR: + clusters = asobj._aggregateBgpClusters() + all_rrs: Set[str] = set() + for _cluster_id, (rr_names, client_names) in clusters.items(): + all_rrs.update(rr_names) + for rr_name in rr_names: + for client_name in client_names: + if rr_name in active_names and client_name in active_names: + ibgpgraph.addEdge( + 'Router: {}'.format(rr_name), + 'Router: {}'.format(client_name), + style = 'solid' + ) + rtrs = [ + routers_map[name] + for name in sorted(all_rrs) + if name in routers_map and name in active_names + ] + else: + rtrs = [routers_map[name] for name in sorted(active_names) if name in routers_map] + while len(rtrs) > 0: a = rtrs.pop() for b in rtrs: @@ -157,4 +356,3 @@ def print(self, indent: int) -> str: out += '{}\n'.format(asn) return out - diff --git a/seedemu/layers/Mpls.py b/seedemu/layers/Mpls.py index 8e4723489..e1395537e 100644 --- a/seedemu/layers/Mpls.py +++ b/seedemu/layers/Mpls.py @@ -1,13 +1,19 @@ from __future__ import annotations +from .Base import Base from .Ospf import Ospf from .Ibgp import Ibgp -from .Routing import Router -from seedemu.core import Node, ScopedRegistry, Graphable, Emulator, Layer +from seedemu.core import Node, ScopedRegistry, Graphable, Emulator, Layer, Router from seedemu.core.enums import NetworkType, NodeRole from typing import List, Tuple, Dict, Set +from ._bgp_metadata import install_router_bgp_session MplsFileTemplates: Dict[str, str] = {} +MPLS_PRESERVED_IBGP_MODES = {"route-reflector"} +MPLS_DISABLED_IBGP_MODES = {"disabled"} +MPLS_CORE_FORWARDING = "mpls" +MPLS_EDGE_BGP_SCOPE = "edge-only" + MplsFileTemplates['frr_start_script'] = """\ #!/bin/bash mount -o remount rw /proc/sys 2> /dev/null @@ -41,18 +47,6 @@ ip ospf dead-interval minimal hello-multiplier 2 """ -MplsFileTemplates['bird_ibgp_peer'] = ''' - ipv4 {{ - table t_bgp; - import all; - export all; - igp table master4; - }}; - local {localAddress} as {asn}; - neighbor {peerAddress} as {asn}; -''' - - class Mpls(Layer, Graphable): """! @brief The Mpls (MPLS) layer. @@ -152,6 +146,62 @@ def getEnabled(self) -> Set[int]: """ return self.__enabled + def __syncAsLevelEnabled(self, emulator: Emulator) -> Set[int]: + reg = emulator.getRegistry() + if reg.has('seedemu', 'layer', 'Base'): + base: Base = reg.get('seedemu', 'layer', 'Base') + for asn in base.getAsns(): + asobj = base.getAutonomousSystem(asn) + if hasattr(asobj, "getCoreForwarding") and asobj.getCoreForwarding() == MPLS_CORE_FORWARDING: + self.__enabled.add(asn) + return set(self.__enabled) + + def __getAsObject(self, emulator: Emulator, asn: int): + reg = emulator.getRegistry() + if reg.has('seedemu', 'layer', 'Base'): + base: Base = reg.get('seedemu', 'layer', 'Base') + return base.getAutonomousSystem(asn) + return None + + def __getExplicitIbgpMode(self, emulator: Emulator, asn: int): + asobj = self.__getAsObject(emulator, asn) + if asobj is not None: + if hasattr(asobj, "hasIbgpMode") and asobj.hasIbgpMode(): + return asobj.getIbgpMode() + reg = emulator.getRegistry() + if reg.has('seedemu', 'layer', 'Ibgp'): + ibgp: Ibgp = reg.get('seedemu', 'layer', 'Ibgp') + if asn in ibgp.getMaskedAsns(): + return "disabled" + return None + + def __preserveExistingIbgp(self, emulator: Emulator, asn: int) -> bool: + asobj = self.__getAsObject(emulator, asn) + if asobj is not None: + if asobj.getIbgpMode() in MPLS_PRESERVED_IBGP_MODES: + return True + if asobj.getIbgpMode() == "full-mesh" and asobj.getBgpScope() == MPLS_EDGE_BGP_SCOPE: + return True + return self.__getExplicitIbgpMode(emulator, asn) in MPLS_PRESERVED_IBGP_MODES + + def __installMplsIbgp(self, emulator: Emulator, asn: int) -> bool: + mode = self.__getExplicitIbgpMode(emulator, asn) + return mode not in MPLS_PRESERVED_IBGP_MODES and mode not in MPLS_DISABLED_IBGP_MODES + + def __maskExistingControlPlaneLayers(self, emulator: Emulator) -> None: + """Mask OSPF/iBGP before those layers record non-MPLS intent.""" + reg = emulator.getRegistry() + for asn in self.__syncAsLevelEnabled(emulator): + if reg.has('seedemu', 'layer', 'Ospf'): + self._log('Ospf layer exists, masking as{}'.format(asn)) + ospf: Ospf = reg.get('seedemu', 'layer', 'Ospf') + ospf.maskAsn(asn) + + if reg.has('seedemu', 'layer', 'Ibgp') and not self.__preserveExistingIbgp(emulator, asn): + self._log('Ibgp layer exists, masking as{}'.format(asn)) + ibgp: Ibgp = reg.get('seedemu', 'layer', 'Ibgp') + ibgp.maskAsn(asn) + def __getEdgeNodes(self, scope: ScopedRegistry) -> Tuple[List[Node], List[Node]]: """! @brief Helper tool - get list of routers (edge, non-edge) of an AS. @@ -179,6 +229,17 @@ def __getEdgeNodes(self, scope: ScopedRegistry) -> Tuple[List[Node], List[Node]] return (enodes, nodes) + def __addAdditionalEdges(self, scope: ScopedRegistry, asn: int, enodes: List[Node]) -> List[Node]: + names = {node.getName() for node in enodes} + for (asn_, nodename) in self.__additional_edges: + if asn_ != asn: + continue + if scope.has('rnode', nodename) and nodename not in names: + node = scope.get('rnode', nodename) + enodes.append(node) + names.add(nodename) + return enodes + def __setUpLdpOspf(self, node: Router): """! @brief Setup LDP and OSPF on router. @@ -229,46 +290,56 @@ def __setUpIbgpMesh(self, nodes: List[Router]): for remote in nodes: if local == remote: continue - local.addTable('t_bgp') - local.addTablePipe('t_bgp') - local.addTablePipe('t_direct', 't_bgp') - local.addProtocol('bgp', 'ibgp{}'.format(n), MplsFileTemplates['bird_ibgp_peer'].format( - localAddress = local.getLoopbackAddress(), - peerAddress = remote.getLoopbackAddress(), - asn = local.getAsn() - )) + install_router_bgp_session( + local, + { + "name": "mpls_ibgp{}".format(n), + "kind": "ibgp", + "local_address": str(local.getLoopbackAddress()), + "local_asn": local.getAsn(), + "peer_address": str(remote.getLoopbackAddress()), + "peer_asn": local.getAsn(), + "export_policy": "all", + "next_hop_self": False, + "igp_table": "master4", + }, + ) n += 1 - def render(self, emulator: Emulator): + def configure(self, emulator: Emulator): + super().configure(emulator) + enabled = self.__syncAsLevelEnabled(emulator) + install_mpls_ibgp = { + asn: self.__installMplsIbgp(emulator, asn) + for asn in enabled + } + self.__maskExistingControlPlaneLayers(emulator) reg = emulator.getRegistry() - for asn in self.__enabled: - if reg.has('seedemu', 'layer', 'Ospf'): - self._log('Ospf layer exists, masking as{}'.format(asn)) - ospf: Ospf = reg.get('seedemu', 'layer', 'Ospf') - ospf.maskAsn(asn) - - if reg.has('seedemu', 'layer', 'Ibgp'): - self._log('Ibgp layer exists, masking as{}'.format(asn)) - ibgp: Ibgp = reg.get('seedemu', 'layer', 'Ibgp') - ibgp.maskAsn(asn) + for asn in enabled: + scope = ScopedRegistry(str(asn), reg) + (enodes, _) = self.__getEdgeNodes(scope) + enodes = self.__addAdditionalEdges(scope, asn, enodes) + if install_mpls_ibgp[asn]: + self.__setUpIbgpMesh(enodes) + def render(self, emulator: Emulator): + reg = emulator.getRegistry() + enabled = self.__syncAsLevelEnabled(emulator) + self.__maskExistingControlPlaneLayers(emulator) + for asn in enabled: scope = ScopedRegistry(str(asn), reg) (enodes, nodes) = self.__getEdgeNodes(scope) - - for (asn_, nodename) in self.__additional_edges: - if asn_ != asn: continue - if scope.has('rnode', nodename): - enodes.append(scope.get('rnode', nodename)) + enodes = self.__addAdditionalEdges(scope, asn, enodes) for n in enodes: self.__setUpLdpOspf(n) for n in nodes: self.__setUpLdpOspf(n) - self.__setUpIbgpMesh(enodes) def _doCreateGraphs(self, emulator: Emulator): base = emulator.getRegistry().get('seedemu', 'layer', 'Base') + enabled = self.__syncAsLevelEnabled(emulator) for asn in base.getAsns(): - if asn not in self.__enabled: continue + if asn not in enabled: continue asobj = base.getAutonomousSystem(asn) asobj.createGraphs(emulator) l2graph = asobj.getGraph('AS{}: Layer 2 Connections'.format(asn)) @@ -299,4 +370,3 @@ def print(self, indent: int) -> str: out += 'as{}\n'.format(asn) return out - \ No newline at end of file diff --git a/seedemu/layers/Ospf.py b/seedemu/layers/Ospf.py index b25de7e74..50051096c 100644 --- a/seedemu/layers/Ospf.py +++ b/seedemu/layers/Ospf.py @@ -1,28 +1,15 @@ from __future__ import annotations from seedemu.core import Node, Emulator, Layer -from seedemu.core.enums import NetworkType +from seedemu.core.enums import NetworkType, NodeRole from typing import Set, Dict, List, Tuple +from .Base import Base +from ._bgp_metadata import classify_ospf_interfaces, set_ospf_interface_intents OspfFileTemplates: Dict[str, str] = {} -OspfFileTemplates['ospf_body'] = """ - ipv4 {{ - table t_ospf; - import all; - export all; - }}; - area 0 {{ -{interfaces} - }}; -""" - -OspfFileTemplates['ospf_interface'] = """\ - interface "{interfaceName}" {{ hello 1; dead count 2; }}; -""" - -OspfFileTemplates['ospf_stub_interface'] = """\ - interface "{interfaceName}" {{ stub; }}; -""" +OSPF_MODE_LEGACY = "legacy" +OSPF_MODE_ROUTER_TRANSIT_ONLY = "router-transit-only" +OSPF_MODES = {OSPF_MODE_LEGACY, OSPF_MODE_ROUTER_TRANSIT_ONLY} class Ospf(Layer): """! @@ -142,43 +129,74 @@ def isMasked(self, asn: int, netname: str) -> bool: """ return (asn, netname) in self.__masked - def render(self, emulator: Emulator): + def __is_router_transit_network(self, node: Node, netname: str) -> bool: + for iface in node.getInterfaces(): + net = iface.getNet() + if str(net.getName()) != str(netname): + continue + if net.getType() != NetworkType.Local: + return False + router_count = 0 + for candidate in net.getAssociations(): + role = candidate.getRole() + if role in {NodeRole.Router, NodeRole.BorderRouter, NodeRole.OpenVpnRouter}: + router_count += 1 + return router_count >= 2 + return False + + def __classify_router_transit_interfaces( + self, + router: Node, + stubs: List[str], + masked: List[str] + ) -> Tuple[List[str], List[str]]: + stub_names = {str(name) for name in stubs} + masked_names = {str(name) for name in masked} + active: List[str] = [] + passive: List[str] = ["dummy0"] + for iface in router.getInterfaces(): + net = iface.getNet() + name = str(net.getName()) + if name in masked_names: + continue + if name in stub_names: + passive.append(name) + continue + if self.__is_router_transit_network(router, name): + active.append(name) + else: + passive.append(name) + return active, passive + + def configure(self, emulator: Emulator): reg = emulator.getRegistry() + base: Base = reg.get('seedemu', 'layer', 'Base') for ((scope, type, name), obj) in reg.getAll().items(): if type != 'rnode': continue router: Node = obj if router.getAsn() in self.__masked_asn: continue - stubs: List[str] = ['dummy0'] - active: List[str] = [] - self._log('setting up OSPF for router as{}/{}...'.format(scope, name)) - for iface in router.getInterfaces(): - net = iface.getNet() - - if (int(scope), net.getName()) in self.__masked: continue - - if (int(scope), net.getName()) in self.__stubs or net.getType() != NetworkType.Local: - stubs.append(net.getName()) - continue - - active.append(net.getName()) - - ospf_interfaces = '' - for name in stubs: ospf_interfaces += OspfFileTemplates['ospf_stub_interface'].format( - interfaceName = name - ) - for name in active: ospf_interfaces += OspfFileTemplates['ospf_interface'].format( - interfaceName = name - ) - - if ospf_interfaces != '': - router.addTable('t_ospf') - router.addProtocol('ospf', 'ospf1', OspfFileTemplates['ospf_body'].format( - interfaces = ospf_interfaces - )) - router.addTablePipe('t_ospf') + asobj = base.getAutonomousSystem(int(scope)) + stub_networks = [net for (asn, net) in self.__stubs if asn == int(scope)] + masked_networks = [net for (asn, net) in self.__masked if asn == int(scope)] + if asobj.getOspfMode() == OSPF_MODE_ROUTER_TRANSIT_ONLY: + active, stubs = self.__classify_router_transit_interfaces( + router, + stubs=stub_networks, + masked=masked_networks, + ) + else: + active, stubs = classify_ospf_interfaces( + router, + stubs=stub_networks, + masked=masked_networks, + ) + set_ospf_interface_intents(router, active, stubs) + + def render(self, emulator: Emulator): + pass def print(self, indent: int) -> str: out = ' ' * indent @@ -210,4 +228,3 @@ def print(self, indent: int) -> str: out += 'as{}\n'.format(asn) return out - diff --git a/seedemu/layers/Routing.py b/seedemu/layers/Routing.py index 8cdbda534..f46a7f065 100644 --- a/seedemu/layers/Routing.py +++ b/seedemu/layers/Routing.py @@ -2,8 +2,20 @@ Layer, Router, BaseSystem, promote_to_real_world_router) from seedemu.core.enums import NetworkType -from typing import List, Dict +from typing import Dict, List, Set, Tuple from ipaddress import IPv4Network +from ._bgp_metadata import ( + BGP_EXPORT_LOCAL_AND_CUSTOMER, + BGP_BACKEND_BIRD, + BGP_BACKEND_FRR, + get_bgp_backend, + get_bgp_sessions, + get_ospf_interface_intents, + has_bgp_connected_export, + ensure_bird_bgp_base, + render_frr_community, + render_bird_protocol_body, +) RoutingFileTemplates: Dict[str, str] = {} @@ -40,6 +52,113 @@ {interfaces} """ +RoutingFileTemplates['bird_ospf_body'] = """ + ipv4 {{ + table t_ospf; + import all; + export all; + }}; + area 0 {{ +{interfaces} + }}; +""" + +RoutingFileTemplates['bird_ospf_interface'] = """\ + interface "{interfaceName}" {{ hello 1; dead count 2; }}; +""" + +RoutingFileTemplates['bird_ospf_stub_interface'] = """\ + interface "{interfaceName}" {{ stub; }}; +""" + +FrrFileTemplates: Dict[str, str] = {} + +FrrFileTemplates["managed_block"] = """\ +! ===== seedemu-routing-frr begin ===== +frr defaults traditional +service integrated-vtysh-config +hostname {hostname} +! +{body} +! ===== seedemu-routing-frr end ===== +""" + +FrrFileTemplates["start_script"] = """\ +#!/bin/bash +set -e +sed -i 's/bgpd=no/bgpd=yes/' /etc/frr/daemons +sed -i 's/zebra=no/zebra=yes/' /etc/frr/daemons +sed -i 's/staticd=no/staticd=yes/' /etc/frr/daemons +sed -i 's/ospfd=no/ospfd=yes/' /etc/frr/daemons +service frr start +""" + +FrrFileTemplates["connected_prefix_list"] = """\ +ip prefix-list PL_CONNECTED4_TO_BGP seq {seq} permit {prefix} +""" + +FrrFileTemplates["route_map_connected"] = """\ +route-map RM_CONNECTED_TO_BGP permit 10 + match ip address prefix-list PL_CONNECTED4_TO_BGP + set large-community {local_comm} additive + set local-preference 40 +! +""" + +FrrFileTemplates["community_lists"] = """\ +bgp large-community-list standard LC_LOCAL permit {local_comm} +bgp large-community-list standard LC_CUSTOMER permit {customer_comm} +bgp large-community-list standard LC_LOCAL_OR_CUSTOMER permit {local_comm} +bgp large-community-list standard LC_LOCAL_OR_CUSTOMER permit {customer_comm} +! +""" + +FrrFileTemplates["import_route_map"] = """\ +route-map {name} permit 10 + set large-community {community} additive + set local-preference {local_pref} +! +""" + +FrrFileTemplates["export_route_map_local_customer"] = """\ +route-map {name} permit 10 + match large-community LC_LOCAL_OR_CUSTOMER +! +route-map {name} deny 100 +! +""" + +FrrFileTemplates["export_route_map_all"] = """\ +route-map {name} permit 10 +! +""" + +FrrFileTemplates["ospf_interface_active"] = """\ +interface {interface} + ip ospf area 0 + ip ospf hello-interval 1 + ip ospf dead-interval 2 +! +""" + +FrrFileTemplates["ospf_interface_passive"] = """\ +interface {interface} + ip ospf area 0 + ip ospf passive +! +""" + +FrrFileTemplates["ospf_router"] = """\ +router ospf + ospf router-id {router_id} +! +""" + + +def _frr_map_name(prefix: str, session_id: str) -> str: + safe = "".join(ch if ch.isalnum() else "_" for ch in str(session_id or "session")) + return "{}_{}".format(prefix, safe)[:64] + class Routing(Layer): """! @@ -84,25 +203,41 @@ def _installBird(self, node: Node): node.addBuildCommand('touch /usr/share/doc/bird2/examples/bird.conf') node.addSoftware('bird2') + self._ensureRouterBaseSystem(node) + + def _ensureRouterBaseSystem(self, node: Node): + """! + @brief Ensure a routing backend keeps the router base system. + """ base = node.getBaseSystem() - if not BaseSystem.doesAContainB(base,BaseSystem.SEEDEMU_ROUTER) and base !=BaseSystem.SEEDEMU_ROUTER: + if not BaseSystem.doesAContainB(base, BaseSystem.SEEDEMU_ROUTER): node.setBaseSystem(BaseSystem.SEEDEMU_ROUTER) - def _configure_rs(self, rs_node: Node): - rs_node.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird') - rs_node.appendStartCommand('bird -d', True) - self._log("Bootstrapping bird.conf for RS {}...".format(rs_node.getName())) + def _installFrr(self, node: Node): + """! + @brief Install FRRouting on node. + """ + node.addSoftware('frr') + self._ensureRouterBaseSystem(node) + def _configure_rs(self, rs_node: Node): + backend = get_bgp_backend(rs_node) rs_ifaces = rs_node.getInterfaces() assert len(rs_ifaces) == 1, "rs node {} has != 1 interfaces".format(rs_node.getName()) - rs_iface = rs_ifaces[0] assert issubclass(rs_node.__class__, Router) rs_node.setBorderRouter(True) - rs_node.setFile("/etc/bird/bird.conf", RoutingFileTemplates["rs_bird"].format( - routerId = rs_iface.getAddress() - )) + if backend == BGP_BACKEND_BIRD: + rs_node.appendStartCommand('[ ! -d /run/bird ] && mkdir /run/bird') + rs_node.appendStartCommand('bird -d', True) + self._log("Bootstrapping bird.conf for RS {}...".format(rs_node.getName())) + rs_node.setFile("/etc/bird/bird.conf", RoutingFileTemplates["rs_bird"].format( + routerId = rs_iface.getAddress() + )) + else: + self._log("Bootstrapping frr.conf for RS {}...".format(rs_node.getName())) + self._configure_frr_router(rs_node) def _configure_bird_router(self, rnode: Router): ifaces = '' @@ -123,13 +258,195 @@ def _configure_bird_router(self, rnode: Router): rnode.addProtocol('direct', 'local_nets', RoutingFileTemplates['rnode_bird_direct'].format(interfaces = ifaces)) + def _configure_frr_router(self, rnode: Router): + rnode.setFile("/frr_start", FrrFileTemplates["start_script"]) + rnode.appendStartCommand("chmod +x /frr_start") + rnode.appendStartCommand("/frr_start") + + def _render_bird_ospf(self, rnode: Router): + intents = get_ospf_interface_intents(rnode) + if not intents["active"] and not intents["passive"]: + return + + ospf_interfaces = '' + for name in intents["passive"]: + ospf_interfaces += RoutingFileTemplates['bird_ospf_stub_interface'].format(interfaceName=name) + for name in intents["active"]: + ospf_interfaces += RoutingFileTemplates['bird_ospf_interface'].format(interfaceName=name) + + if ospf_interfaces != '': + rnode.addTable('t_ospf') + rnode.addProtocol('ospf', 'ospf1', RoutingFileTemplates['bird_ospf_body'].format( + interfaces=ospf_interfaces + )) + rnode.addTablePipe('t_ospf') + + def _render_bird_control_plane(self, rnode: Router): + sessions = get_bgp_sessions(rnode) + self._render_bird_ospf(rnode) + if sessions or has_bgp_connected_export(rnode): + include_tables = not sessions or any(not session["route_server_client"] for session in sessions) + ensure_bird_bgp_base(rnode, include_tables=include_tables) + for session in sessions: + rnode.addProtocol('bgp', session["render_name"], render_bird_protocol_body(session)) + + def _render_frr_connected_export(self, router: Router) -> Tuple[str, bool]: + prefixes: List[str] = [] + seen: Set[str] = set() + for iface in router.getInterfaces(): + net = iface.getNet() + if net.getType() == NetworkType.Bridge: + continue + if not net.isDirect(): + continue + prefix = str(net.getPrefix()) + if iface.getAddress() is not None and prefix not in seen: + seen.add(prefix) + prefixes.append(prefix) + + body: List[str] = [] + for index, prefix in enumerate(prefixes, start=1): + body.append(FrrFileTemplates["connected_prefix_list"].format(seq=index * 10, prefix=prefix)) + if prefixes: + body.append(FrrFileTemplates["route_map_connected"].format(local_comm="{}:0:0".format(router.getAsn()))) + return "".join(body), bool(prefixes) + + def _render_frr_route_maps(self, local_asn: int, sessions: List[Dict]) -> Tuple[str, Dict[str, Dict[str, str]]]: + body: List[str] = [] + map_names: Dict[str, Dict[str, str]] = {} + for session in sessions: + session_id = str(session.get("render_name") or session.get("name") or "session") + import_name = "" + import_community = str(session.get("import_community") or "").strip() + local_pref = session.get("local_pref") + if import_community and local_pref is not None: + import_name = _frr_map_name("RM_IMPORT", session_id) + body.append( + FrrFileTemplates["import_route_map"].format( + name=import_name, + community=render_frr_community(local_asn, import_community), + local_pref=int(local_pref), + ) + ) + + export_name = _frr_map_name("RM_EXPORT", session_id) + if session["export_policy"] == BGP_EXPORT_LOCAL_AND_CUSTOMER: + body.append(FrrFileTemplates["export_route_map_local_customer"].format(name=export_name)) + else: + body.append(FrrFileTemplates["export_route_map_all"].format(name=export_name)) + map_names[session_id] = {"import": import_name, "export": export_name} + return "".join(body), map_names + + def _render_frr_bgp(self, router: Router) -> str: + sessions = get_bgp_sessions(router) + if not sessions and not has_bgp_connected_export(router): + return "" + + body: List[str] = [] + body.append( + FrrFileTemplates["community_lists"].format( + local_comm="{}:0:0".format(router.getAsn()), + customer_comm="{}:1:0".format(router.getAsn()), + ) + ) + if has_bgp_connected_export(router): + connected_body, has_connected = self._render_frr_connected_export(router) + else: + connected_body, has_connected = "", False + body.append(connected_body) + route_maps, map_names = self._render_frr_route_maps(router.getAsn(), sessions) + body.append(route_maps) + + bgp: List[str] = [ + "router bgp {}".format(router.getAsn()), + " bgp router-id {}".format(self._get_frr_router_id(router)), + " no bgp ebgp-requires-policy", + " no bgp default ipv4-unicast", + ] + rr_cluster_ids = { + session["route_reflector_cluster_id"] + for session in sessions + if session["route_reflector_client"] + } + if len(rr_cluster_ids) > 1: + raise ValueError( + "FRR supports one route-reflector cluster-id per router, got {}".format( + ", ".join(sorted(rr_cluster_ids)) + ) + ) + if rr_cluster_ids: + bgp.append(" bgp cluster-id {}".format(next(iter(rr_cluster_ids)))) + for session in sessions: + bgp.append(" neighbor {} remote-as {}".format(session["peer_address"], session["peer_asn"])) + bgp.append(" neighbor {} update-source {}".format(session["peer_address"], session["local_address"])) + bgp.append(" neighbor {} description {}".format(session["peer_address"], session["name"])) + if session["passive"]: + bgp.append(" neighbor {} passive".format(session["peer_address"])) + bgp.append(" !") + bgp.append(" address-family ipv4 unicast") + if has_connected: + bgp.append(" redistribute connected route-map RM_CONNECTED_TO_BGP") + for session in sessions: + session_id = str(session.get("render_name") or session.get("name") or "session") + maps = map_names.get(session_id, {}) + bgp.append(" neighbor {} activate".format(session["peer_address"])) + if session["next_hop_self"]: + bgp.append(" neighbor {} next-hop-self".format(session["peer_address"])) + if session["route_server_client"]: + bgp.append(" neighbor {} route-server-client".format(session["peer_address"])) + if session["route_reflector_client"]: + bgp.append(" neighbor {} route-reflector-client".format(session["peer_address"])) + if maps.get("import"): + bgp.append(" neighbor {} route-map {} in".format(session["peer_address"], maps["import"])) + if maps.get("export"): + bgp.append(" neighbor {} route-map {} out".format(session["peer_address"], maps["export"])) + bgp.append(" exit-address-family") + bgp.append("!") + body.append("\n".join(bgp) + "\n") + return "".join(body) + + def _get_frr_router_id(self, router: Router) -> str: + loopback = router.getLoopbackAddress() + if loopback is not None: + return str(loopback) + ifaces = router.getInterfaces() + assert len(ifaces) > 0, "router node {}/{} has no interfaces".format(router.getAsn(), router.getName()) + return str(ifaces[0].getAddress()) + + def _render_frr_ospf(self, router: Router) -> str: + intents = get_ospf_interface_intents(router) + if not intents["active"] and not intents["passive"]: + return "" + + body: List[str] = [] + for name in intents["passive"]: + body.append(FrrFileTemplates["ospf_interface_passive"].format(interface=name)) + for name in intents["active"]: + body.append(FrrFileTemplates["ospf_interface_active"].format(interface=name)) + body.append(FrrFileTemplates["ospf_router"].format(router_id=router.getLoopbackAddress())) + return "".join(body) + + def _render_frr_control_plane(self, router: Router): + body = self._render_frr_ospf(router) + self._render_frr_bgp(router) + router.setFile( + "/etc/frr/frr.conf", + FrrFileTemplates["managed_block"].format( + hostname="as{}_{}".format(router.getAsn(), router.getName()), + body=body, + ), + ) + def configure(self, emulator: Emulator): super().configure(emulator) reg = emulator.getRegistry() for ((scope, type, name), obj) in reg.getAll().items(): if type == 'rs': rs_node: Node = obj - self._installBird(rs_node) + backend = get_bgp_backend(rs_node) + if backend == BGP_BACKEND_BIRD: + self._installBird(rs_node) + else: + self._installFrr(rs_node) self._configure_rs(rs_node) if type == 'rnode': rnode: Router = obj @@ -139,6 +456,8 @@ def configure(self, emulator: Emulator): if rnode.getLoopbackAddress() == None: lbaddr = self._loopback_assigner[self._loopback_pos] + else: + lbaddr = rnode.getLoopbackAddress() rnode.appendStartCommand('ip li add dummy0 type dummy') rnode.appendStartCommand('ip li set dummy0 up') @@ -147,14 +466,21 @@ def configure(self, emulator: Emulator): rnode.setLoopbackAddress(lbaddr) self._loopback_pos += 1 - self._log("Bootstrapping bird.conf for AS{} Router {}...".format(scope, name)) + backend = get_bgp_backend(rnode) + self._log("Bootstrapping {} routing config for AS{} Router {}...".format(backend, scope, name)) - self._installBird(rnode) + if backend == BGP_BACKEND_BIRD: + self._installBird(rnode) + else: + self._installFrr(rnode) r_ifaces = rnode.getInterfaces() assert len(r_ifaces) > 0, "router node {}/{} has no interfaces".format(rnode.getAsn(), rnode.getName()) - self._configure_bird_router(rnode) + if backend == BGP_BACKEND_BIRD: + self._configure_bird_router(rnode) + else: + self._configure_frr_router(rnode) def render(self, emulator: Emulator): reg = emulator.getRegistry() @@ -186,8 +512,22 @@ def render(self, emulator: Emulator): if type == 'rs' or type == 'rnode': assert issubclass(obj.__class__, Router), 'routing: render: adding new RS/Router after routing layer configured is not currently supported.' + if type == 'rs': + rs_node: Router = obj + backend = get_bgp_backend(rs_node) + if backend == BGP_BACKEND_BIRD: + self._render_bird_control_plane(rs_node) + else: + self._render_frr_control_plane(rs_node) + if type == 'rnode': rnode: Router = obj + backend = get_bgp_backend(rnode) + if backend == BGP_BACKEND_BIRD: + self._render_bird_control_plane(rnode) + else: + self._render_frr_control_plane(rnode) + if rnode.hasExtension('RealWorldRouter'): # could also be ScionRouter which needs RealWorldAccess # this is an exception - Only for service net (not part of simulation) @@ -221,6 +561,14 @@ def render(self, emulator: Emulator): rif = riface break + if rif == None and hnet.getType() != NetworkType.Local: + services = hnode.getAttribute('services', {}) or {} + if "ExaBgpService" in services: + self._log("Skipping default route for ExaBGP speaker {} in as{} on non-local network {}.".format( + name, scope, hnet.getName() + )) + continue + assert rif != None, 'Host {} in as{} in network {}: no router'.format(name, scope, hnet.getName()) self._log("Setting default route for host {} ({}) to router {}".format(name, hif.getAddress(), rif.getAddress())) hnode.appendStartCommand('ip rou del default 2> /dev/null') @@ -228,6 +576,6 @@ def render(self, emulator: Emulator): def print(self, indent: int) -> str: out = ' ' * indent - out += 'RoutingLayer: BIRD 2.0.x\n' + out += 'RoutingLayer: BIRD 2.0.x / FRRouting\n' - return out \ No newline at end of file + return out diff --git a/seedemu/layers/_bgp_metadata.py b/seedemu/layers/_bgp_metadata.py new file mode 100644 index 000000000..3d5b31aed --- /dev/null +++ b/seedemu/layers/_bgp_metadata.py @@ -0,0 +1,393 @@ +from __future__ import annotations + +from hashlib import sha1 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +from seedemu.core import Router +from seedemu.core.enums import NetworkType + + +BGP_SESSION_INTENTS_ATTR = "__bgp_session_intents" +BGP_CONNECTED_EXPORT_ATTR = "__bgp_connected_export" +BGP_CONNECTED_EXPORT_RENDERED_ATTR = "__bgp_connected_export_rendered" +BGP_BOOTSTRAPPED_ATTR = "__bgp_bootstrapped" +OSPF_INTERFACE_INTENTS_ATTR = "__ospf_interface_intents" + +BGP_BACKEND_BIRD = "bird" +BGP_BACKEND_FRR = "frr" + +BGP_KIND_EBGP = "ebgp" +BGP_KIND_IBGP = "ibgp" +BGP_EXPORT_ALL = "all" +BGP_EXPORT_LOCAL_AND_CUSTOMER = "local_and_customer" + +BGP_COMMUNITY_LOCAL = "local" +BGP_COMMUNITY_CUSTOMER = "customer" +BGP_COMMUNITY_PEER = "peer" +BGP_COMMUNITY_PROVIDER = "provider" + +BGP_COMMUNITY_ALIASES = { + "LOCAL_COMM": BGP_COMMUNITY_LOCAL, + "CUSTOMER_COMM": BGP_COMMUNITY_CUSTOMER, + "PEER_COMM": BGP_COMMUNITY_PEER, + "PROVIDER_COMM": BGP_COMMUNITY_PROVIDER, + "local": BGP_COMMUNITY_LOCAL, + "customer": BGP_COMMUNITY_CUSTOMER, + "peer": BGP_COMMUNITY_PEER, + "provider": BGP_COMMUNITY_PROVIDER, +} + +BIRD_COMMUNITY_NAMES = { + BGP_COMMUNITY_LOCAL: "LOCAL_COMM", + BGP_COMMUNITY_CUSTOMER: "CUSTOMER_COMM", + BGP_COMMUNITY_PEER: "PEER_COMM", + BGP_COMMUNITY_PROVIDER: "PROVIDER_COMM", +} + +FRR_COMMUNITY_VALUES = { + BGP_COMMUNITY_LOCAL: "0:0", + BGP_COMMUNITY_CUSTOMER: "1:0", + BGP_COMMUNITY_PEER: "2:0", + BGP_COMMUNITY_PROVIDER: "3:0", +} + +BIRD_BGP_COMMONS_TEMPLATE = """\ +define LOCAL_COMM = ({localAsn}, 0, 0); +define CUSTOMER_COMM = ({localAsn}, 1, 0); +define PEER_COMM = ({localAsn}, 2, 0); +define PROVIDER_COMM = ({localAsn}, 3, 0); +""" + +BIRD_RS_PEER_TEMPLATE = """\ + ipv4 {{ + import all; + export all; + }}; + rs client; + local {localAddress} as {localAsn}; + neighbor {peerAddress} as {peerAsn}; +""" + +BIRD_ROUTER_PEER_TEMPLATE = """\ + ipv4 {{ + table t_bgp; + import {importClause}; + export {exportClause}; +{nextHopSelf} }}; + local {localAddress} as {localAsn}; + neighbor {peerAddress} as {peerAsn}; +""" + +BIRD_IBGP_PEER_TEMPLATE = """\ +{passive}\ + ipv4 {{ + table t_bgp; + import all; + export all; + igp table {igpTable}; +{nextHopSelf}\ + }}; + local {localAddress} as {localAsn}; + neighbor {peerAddress} as {peerAsn}; +{routeReflector}\ +""" + +CONNECTED_EXPORT_FILTER = "filter { bgp_large_community.add(LOCAL_COMM); bgp_local_pref = 40; accept; }" + + +def get_bgp_backend(node: Router) -> str: + """Return the router daemon backend used for BGP rendering.""" + backend = node.getRoutingBackend() + value = str(backend or BGP_BACKEND_BIRD).strip().lower() + if value not in {BGP_BACKEND_BIRD, BGP_BACKEND_FRR}: + raise ValueError("unsupported routing backend: {}".format(backend)) + return value + + +def _normalize_export_policy(policy: Any) -> str: + # This v1 intent model intentionally keeps export policy narrow. Richer + # policy controls such as prepend, MED, and AS-path filters should become a + # structured policy object instead of overloading these string tokens. + value = str(policy or BGP_EXPORT_ALL).strip().lower() + if value not in {BGP_EXPORT_ALL, BGP_EXPORT_LOCAL_AND_CUSTOMER}: + raise ValueError("unsupported export policy: {}".format(policy)) + return value + + +def _normalize_community(community: Any) -> Optional[str]: + if community in {None, ""}: + return None + value = str(community).strip() + if value == "": + return None + if value not in BGP_COMMUNITY_ALIASES: + raise ValueError("unsupported BGP community intent: {}".format(community)) + return BGP_COMMUNITY_ALIASES[value] + + +def render_bird_community(community: str) -> str: + normalized = _normalize_community(community) + if normalized is None: + raise ValueError("BIRD community rendering requires a community intent") + return BIRD_COMMUNITY_NAMES[normalized] + + +def render_frr_community(local_asn: int, community: str) -> str: + normalized = _normalize_community(community) + if normalized is None: + raise ValueError("FRR community rendering requires a community intent") + return "{}:{}".format(local_asn, FRR_COMMUNITY_VALUES[normalized]) + + +def _safe_identifier(value: str) -> str: + ident = "".join(ch if ch.isalnum() else "_" for ch in str(value or "session")) + ident = ident.strip("_") or "session" + if ident[0].isdigit(): + ident = "s_{}".format(ident) + return ident + + +def _session_identity(session: Dict[str, Any]) -> Tuple[str, str, str, int, int, bool]: + return ( + str(session["kind"]), + str(session["local_address"]), + str(session["peer_address"]), + int(session["local_asn"]), + int(session["peer_asn"]), + bool(session["route_server_client"]), + ) + + +def _assign_render_names(sessions: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + used: Set[str] = set() + named: List[Dict[str, Any]] = [] + for session in sessions: + normalized = normalize_bgp_session(session) + base = _safe_identifier(normalized["name"]) + render_name = base + if render_name in used: + digest = sha1("|".join(str(part) for part in _session_identity(normalized)).encode()).hexdigest()[:8] + render_name = "{}_{}".format(base[:54], digest) + counter = 2 + while render_name in used: + suffix = "_{}".format(counter) + render_name = "{}{}{}".format(base[:54 - len(suffix)], digest, suffix) + counter += 1 + used.add(render_name) + normalized["render_name"] = render_name + named.append(normalized) + return named + + +def normalize_bgp_session(session: Dict[str, Any]) -> Dict[str, Any]: + name = str(session.get("name") or "session").strip() or "session" + kind = str(session.get("kind") or BGP_KIND_EBGP).strip().lower() or BGP_KIND_EBGP + if kind not in {BGP_KIND_EBGP, BGP_KIND_IBGP}: + raise ValueError("unsupported BGP session kind: {}".format(kind)) + + local_address = str(session.get("local_address") or "").strip() + peer_address = str(session.get("peer_address") or "").strip() + local_asn = int(session.get("local_asn") or 0) + peer_asn = int(session.get("peer_asn") or 0) + if not local_address or not peer_address: + raise ValueError("BGP session must include local_address and peer_address") + if local_asn <= 0 or peer_asn <= 0: + raise ValueError("BGP session must include positive local_asn and peer_asn") + + route_server_client = bool(session.get("route_server_client", False)) + route_reflector_client = bool(session.get("route_reflector_client", False)) + if kind == BGP_KIND_IBGP and local_asn != peer_asn: + raise ValueError("iBGP session must use the same local_asn and peer_asn") + if route_server_client and kind != BGP_KIND_EBGP: + raise ValueError("route_server_client is only valid for eBGP sessions") + if route_reflector_client and kind != BGP_KIND_IBGP: + raise ValueError("route_reflector_client is only valid for iBGP sessions") + + route_reflector_cluster_id = session.get("route_reflector_cluster_id") + route_reflector_cluster_id = ( + str(route_reflector_cluster_id).strip() + if route_reflector_cluster_id not in {None, ""} + else None + ) + if route_reflector_client and route_reflector_cluster_id is None: + raise ValueError("route_reflector_client requires route_reflector_cluster_id") + + import_community = _normalize_community(session.get("import_community")) + local_pref_value = session.get("local_pref") + local_pref = int(local_pref_value) if local_pref_value not in {None, ""} else None + + return { + "name": name, + "kind": kind, + "local_address": local_address, + "local_asn": local_asn, + "peer_address": peer_address, + "peer_asn": peer_asn, + "import_community": import_community, + "local_pref": local_pref, + "export_policy": _normalize_export_policy(session.get("export_policy")), + "next_hop_self": bool(session.get("next_hop_self", False)), + "route_server_client": route_server_client, + "route_reflector_client": route_reflector_client, + "route_reflector_cluster_id": route_reflector_cluster_id, + "passive": bool(session.get("passive", False)), + "igp_table": str(session.get("igp_table") or "t_ospf").strip() or "t_ospf", + } + + +def record_bgp_session(node: Router, session: Dict[str, Any]) -> Dict[str, Any]: + normalized = normalize_bgp_session(session) + sessions = [ + normalize_bgp_session(item) + for item in list(node.getAttribute(BGP_SESSION_INTENTS_ATTR, []) or []) + if isinstance(item, dict) + ] + replaced = False + updated: List[Dict[str, Any]] = [] + for item in sessions: + if _session_identity(item) == _session_identity(normalized): + updated.append(normalized) + replaced = True + else: + updated.append(item) + if not replaced: + updated.append(normalized) + updated = _assign_render_names(updated) + node.setAttribute(BGP_SESSION_INTENTS_ATTR, updated) + for item in updated: + if _session_identity(item) == _session_identity(normalized): + return dict(item) + return dict(updated[-1]) + + +def get_bgp_sessions(node: Router) -> List[Dict[str, Any]]: + sessions: List[Dict[str, Any]] = [] + for item in list(node.getAttribute(BGP_SESSION_INTENTS_ATTR, []) or []): + if isinstance(item, dict): + sessions.append(normalize_bgp_session(item)) + return _assign_render_names(sessions) + + +def mark_bgp_connected_export(node: Router) -> None: + node.setAttribute(BGP_CONNECTED_EXPORT_ATTR, True) + + +def has_bgp_connected_export(node: Router) -> bool: + return bool(node.getAttribute(BGP_CONNECTED_EXPORT_ATTR, False)) + + +def ensure_bird_bgp_base(node: Router, include_tables: bool = True) -> None: + """Install BIRD's shared BGP scaffolding for routers that still use BIRD.""" + if get_bgp_backend(node) != BGP_BACKEND_BIRD: + return + if not node.getAttribute(BGP_BOOTSTRAPPED_ATTR, False): + node.setAttribute(BGP_BOOTSTRAPPED_ATTR, True) + node.appendFile("/etc/bird/bird.conf", BIRD_BGP_COMMONS_TEMPLATE.format(localAsn=node.getAsn())) + if not include_tables: + return + node.addTable("t_bgp") + node.addTablePipe("t_bgp") + if has_bgp_connected_export(node) and not node.getAttribute(BGP_CONNECTED_EXPORT_RENDERED_ATTR, False): + node.addTablePipe("t_direct", "t_bgp", exportFilter=CONNECTED_EXPORT_FILTER) + node.setAttribute(BGP_CONNECTED_EXPORT_RENDERED_ATTR, True) + + +def _bird_import_clause(session: Dict[str, Any]) -> str: + if session["import_community"] and session["local_pref"] is not None: + return ( + "filter {{\n" + " bgp_large_community.add({});\n" + " bgp_local_pref = {};\n" + " accept;\n" + " }}" + ).format(render_bird_community(session["import_community"]), int(session["local_pref"])) + return "all" + + +def _bird_export_clause(session: Dict[str, Any]) -> str: + if session["export_policy"] == BGP_EXPORT_LOCAL_AND_CUSTOMER: + return "where bgp_large_community ~ [LOCAL_COMM, CUSTOMER_COMM]" + return "all" + + +def render_bird_protocol_body(session: Dict[str, Any]) -> str: + normalized = normalize_bgp_session(session) + if normalized["route_server_client"]: + return BIRD_RS_PEER_TEMPLATE.format( + localAddress=normalized["local_address"], + localAsn=normalized["local_asn"], + peerAddress=normalized["peer_address"], + peerAsn=normalized["peer_asn"], + ) + if normalized["kind"] == BGP_KIND_IBGP: + return BIRD_IBGP_PEER_TEMPLATE.format( + localAddress=normalized["local_address"], + localAsn=normalized["local_asn"], + peerAddress=normalized["peer_address"], + peerAsn=normalized["peer_asn"], + igpTable=normalized["igp_table"], + nextHopSelf=" next hop self;\n" if normalized["next_hop_self"] else "", + passive=" passive yes;\n" if normalized["passive"] else "", + routeReflector=( + " rr client;\n" + " rr cluster id {};\n".format(normalized["route_reflector_cluster_id"]) + if normalized["route_reflector_client"] + else "" + ), + ) + return BIRD_ROUTER_PEER_TEMPLATE.format( + localAddress=normalized["local_address"], + localAsn=normalized["local_asn"], + peerAddress=normalized["peer_address"], + peerAsn=normalized["peer_asn"], + importClause=_bird_import_clause(normalized), + exportClause=_bird_export_clause(normalized), + nextHopSelf=" next hop self;\n" if normalized["next_hop_self"] else "", + ) + + +def install_router_bgp_session(node: Router, session: Dict[str, Any]) -> Dict[str, Any]: + normalized = record_bgp_session(node, session) + if not normalized["route_server_client"]: + mark_bgp_connected_export(node) + return normalized + + +def classify_ospf_interfaces( + node: Router, + *, + stubs: Iterable[str] = (), + masked: Iterable[str] = (), +) -> Tuple[List[str], List[str]]: + stub_names = {str(name) for name in stubs} + masked_names = {str(name) for name in masked} + active: List[str] = [] + passive: List[str] = ["dummy0"] + for iface in node.getInterfaces(): + net = iface.getNet() + name = str(net.getName()) + if name in masked_names: + continue + if name in stub_names or net.getType() != NetworkType.Local: + passive.append(name) + continue + active.append(name) + return active, passive + + +def set_ospf_interface_intents(node: Router, active: Iterable[str], passive: Iterable[str]) -> None: + node.setAttribute( + OSPF_INTERFACE_INTENTS_ATTR, + { + "active": sorted({str(name) for name in active}), + "passive": sorted({str(name) for name in passive}), + }, + ) + + +def get_ospf_interface_intents(node: Router) -> Dict[str, List[str]]: + raw = node.getAttribute(OSPF_INTERFACE_INTENTS_ATTR, {}) or {} + return { + "active": [str(name) for name in list(raw.get("active", []) or [])], + "passive": [str(name) for name in list(raw.get("passive", []) or [])], + } diff --git a/seedemu/services/BotnetService.py b/seedemu/services/BotnetService.py index f32c3ae62..f7a3ee41a 100644 --- a/seedemu/services/BotnetService.py +++ b/seedemu/services/BotnetService.py @@ -2,6 +2,7 @@ # encoding: utf-8 # __author__ = 'Demon' from __future__ import annotations +import shlex from seedemu.core import Node, Service, Server, Emulator from typing import Dict @@ -95,6 +96,14 @@ ''' +def _write_requirements_command(path: str) -> str: + requirements = BotnetServerFileTemplates['requirements_override'].replace("\\", "\\\\").replace("\n", "\\n") + return "printf %b {} > {}".format( + shlex.quote(requirements), + shlex.quote(path), + ) + + class BotnetServer(Server): """! @brief The BotnetServer class. @@ -163,12 +172,8 @@ def install(self, node: Node): # override requirements node.addBuildCommand("mkdir -p /tmp/byob/byob") - node.addBuildCommand( - "cat > /tmp/byob/byob/requirements.txt <<'EOF'\n" - + BotnetServerFileTemplates['requirements_override'] + - "\nEOF" - ) - node.addBuildCommand('pip3 install -r /tmp/byob/byob/requirements.txt') + node.addBuildCommand(_write_requirements_command('/tmp/byob/byob/requirements.txt')) + node.addBuildCommand('pip3 install --break-system-packages -r /tmp/byob/byob/requirements.txt || pip3 install -r /tmp/byob/byob/requirements.txt') node.setFile('/tmp/byob_patch.py', BotnetServerFileTemplates['byob_patch_py']) node.appendStartCommand('chmod +x /tmp/byob_patch.py') @@ -268,12 +273,9 @@ def install(self, node: Node): # get byob dependencies. node.addSoftware('python3 git cmake python3-dev gcc g++ make python3-pip') - node.addBuildCommand( - "cat > /tmp/byob-requirements.txt <<'EOF'\n" - + BotnetServerFileTemplates['requirements_override'] + - "\nEOF" - ) - node.addBuildCommand('pip3 install -r /tmp/byob-requirements.txt') + node.addBuildCommand(_write_requirements_command('/tmp/byob-requirements.txt')) + node.addBuildCommand('pip3 install --break-system-packages -r /tmp/byob-requirements.txt || pip3 install -r /tmp/byob-requirements.txt') + fork = False diff --git a/seedemu/services/CDNService.py b/seedemu/services/CDNService.py new file mode 100644 index 000000000..a68402aa4 --- /dev/null +++ b/seedemu/services/CDNService.py @@ -0,0 +1,629 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from ipaddress import IPv4Network +from typing import Dict, List, Optional, Set, Tuple + +from seedemu.core import Emulator, Node, Server, Service +from seedemu.core.enums import NetworkType + +from .DomainNameService import DomainNameService, DomainNameServer + + +CDNServiceFileTemplates: Dict[str, str] = {} + +CDNServiceFileTemplates['origin_nginx_site'] = '''\ +server {{ + listen {port}; + root {root}; + index index.html; + server_name {server_name}; + add_header X-CDN-Origin {origin_name} always; + add_header X-CDN-Origin-IP {origin_ip} always; + + location / {{ + try_files $uri $uri/ =404; + }} +}} +''' + +CDNServiceFileTemplates['edge_nginx_site'] = '''\ +upstream cdn_origin_backend {{ +{upstreams} +}} + +server {{ + listen {port}; + server_name {server_names}; + add_header X-CDN-Edge {site_name} always; + add_header X-CDN-Region {region} always; + add_header X-CDN-Edge-IP {edge_ip} always; + + location / {{ + proxy_set_header Host $host; + proxy_set_header X-CDN-Site {site_name}; + proxy_set_header X-CDN-Region {region}; + proxy_set_header X-CDN-Edge-IP {edge_ip}; + proxy_pass http://cdn_origin_backend; + }} +}} +''' + + +def _choose_service_address(node: Node) -> str: + for iface in node.getInterfaces(): + net = iface.getNet() + if net.getType() == NetworkType.Local: + return str(iface.getAddress()) + + ifaces = node.getInterfaces() + assert len(ifaces) > 0, 'node has no interfaces' + return str(ifaces[0].getAddress()) + + +def _normalize_zone_name(zone: str) -> str: + return zone if zone.endswith('.') else '{}.'.format(zone) + + +def _infer_zone_name(domain: str) -> str: + domain = domain[:-1] if domain.endswith('.') else domain + labels = domain.split('.') + assert len(labels) >= 2, 'cannot infer authoritative zone from "{}"'.format(domain) + return _normalize_zone_name('.'.join(labels[1:])) + + +def _relative_record_name(domain: str, zone: str) -> str: + domain = domain[:-1] if domain.endswith('.') else domain + zone = zone[:-1] if zone.endswith('.') else zone + if domain == zone: + return '@' + suffix = '.{}'.format(zone) + assert domain.endswith(suffix), 'domain "{}" is not inside zone "{}"'.format(domain, zone) + return domain[: -len(suffix)] + + +def _safe_name(value: str) -> str: + return value.replace('.', '_').replace('-', '_').replace('/', '_') + + +@dataclass +class _DomainConfig: + domain: str + dns_vnode: str + edges: List[str] + mode: str = 'static' + zone: Optional[str] = None + include_path: Optional[str] = None + region_map: Dict[str, List[str]] = field(default_factory=dict) + asn_map: Dict[int, List[str]] = field(default_factory=dict) + + +class CDNOriginServer(Server): + __node: Optional[Node] + __service_ip: Optional[str] + __port: int + __server_name: str + __index_content: str + __root: str + + def __init__(self): + super().__init__() + self.__node = None + self.__service_ip = None + self.__port = 8080 + self.__server_name = '_' + self.__index_content = '' + self.__root = '/var/www/html' + + def setPort(self, port: int) -> CDNOriginServer: + self.__port = port + return self + + def getPort(self) -> int: + return self.__port + + def setServerName(self, server_name: str) -> CDNOriginServer: + self.__server_name = server_name + return self + + def setIndexContent(self, content: str) -> CDNOriginServer: + self.__index_content = content + return self + + def setRoot(self, path: str) -> CDNOriginServer: + self.__root = path + return self + + def configure(self, node: Node): + self.__node = node + self.__service_ip = _choose_service_address(node) + + def getNode(self) -> Node: + assert self.__node is not None, 'origin server not configured yet' + return self.__node + + def getServiceAddress(self) -> str: + assert self.__service_ip is not None, 'origin server not configured yet' + return self.__service_ip + + def install(self, node: Node): + origin_name = node.getName() + origin_ip = self.getServiceAddress() + node.addSoftware('nginx-light') + index_content = self.__index_content or '

{}

'.format(node.getName()) + node.setFile('{}/index.html'.format(self.__root), index_content) + node.setFile( + '/etc/nginx/sites-available/default', + CDNServiceFileTemplates['origin_nginx_site'].format( + port=self.__port, + root=self.__root, + server_name=self.__server_name, + origin_name=origin_name, + origin_ip=origin_ip, + ) + ) + node.appendStartCommand('service nginx start') + node.appendClassName('CDNOrigin') + + +class CDNEdgeServer(Server): + __node: Optional[Node] + __service_ip: Optional[str] + __port: int + __region: str + __server_name: str + __origin_vnodes: List[str] + __resolved_origins: List[Tuple[str, int]] + __enable_proxy_headers: bool + __served_domains: Set[str] + + def __init__(self): + super().__init__() + self.__node = None + self.__service_ip = None + self.__port = 8080 + self.__region = 'default' + self.__server_name = '_' + self.__origin_vnodes = [] + self.__resolved_origins = [] + self.__enable_proxy_headers = True + self.__served_domains = set() + + def setPort(self, port: int) -> CDNEdgeServer: + self.__port = port + return self + + def getPort(self) -> int: + return self.__port + + def setRegion(self, region: str) -> CDNEdgeServer: + self.__region = region + return self + + def getRegion(self) -> str: + return self.__region + + def addOrigin(self, vnode: str) -> CDNEdgeServer: + if vnode not in self.__origin_vnodes: + self.__origin_vnodes.append(vnode) + return self + + def getOriginVnodes(self) -> List[str]: + return self.__origin_vnodes + + def setServerName(self, server_name: str) -> CDNEdgeServer: + self.__server_name = server_name + return self + + def enableProxyHeaders(self, enabled: bool = True) -> CDNEdgeServer: + self.__enable_proxy_headers = enabled + return self + + def addServedDomain(self, domain: str): + self.__served_domains.add(domain) + + def getServedDomains(self) -> List[str]: + return sorted(self.__served_domains) + + def configure(self, node: Node): + self.__node = node + self.__service_ip = _choose_service_address(node) + + def setResolvedOrigins(self, origins: List[Tuple[str, int]]): + self.__resolved_origins = origins + + def getNode(self) -> Node: + assert self.__node is not None, 'edge server not configured yet' + return self.__node + + def getServiceAddress(self) -> str: + assert self.__service_ip is not None, 'edge server not configured yet' + return self.__service_ip + + def install(self, node: Node): + assert len(self.__resolved_origins) > 0, 'edge {} has no configured origin'.format(node.getName()) + upstreams = [] + for (addr, port) in self.__resolved_origins: + upstreams.append(' server {}:{};'.format(addr, port)) + + if self.__server_name != '_': + server_names = self.__server_name + elif len(self.__served_domains) > 0: + server_names = ' '.join(sorted(self.__served_domains)) + else: + server_names = '_' + + if self.__enable_proxy_headers: + site_name = node.getName() + region = self.__region + edge_ip = self.getServiceAddress() + else: + site_name = '' + region = '' + edge_ip = '' + + node.addSoftware('nginx-light') + node.setFile( + '/etc/nginx/sites-available/default', + CDNServiceFileTemplates['edge_nginx_site'].format( + port=self.__port, + server_names=server_names, + upstreams='\n'.join(upstreams), + site_name=site_name, + region=region, + edge_ip=edge_ip, + ) + ) + node.appendStartCommand('service nginx start') + node.appendClassName('CDNEdge') + + +class CDNService(Service): + __dns_service_name: str + __origins: Dict[str, CDNOriginServer] + __edges: Dict[str, CDNEdgeServer] + __domains: Dict[str, _DomainConfig] + __region_members: Dict[str, Set[int]] + __metrics_enabled: bool + __dns_artifacts: Dict[str, Dict[str, str]] + + def __init__(self, dnsServiceName: str = 'DomainNameService'): + super().__init__() + self.__dns_service_name = dnsServiceName + self.__origins = {} + self.__edges = {} + self.__domains = {} + self.__region_members = {} + self.__metrics_enabled = False + self.__dns_artifacts = {} + self.addDependency('Base', False, False) + self.addDependency('Routing', False, False) + self.addDependency(dnsServiceName, True, False) + + def _createServer(self) -> Server: + raise AssertionError('Use createOrigin() or createEdge() instead of install().') + + def createOrigin(self, vnode: str) -> CDNOriginServer: + if vnode in self.__edges: + raise AssertionError('vnode "{}" already used as edge'.format(vnode)) + if vnode not in self.__origins: + server = CDNOriginServer() + self.__origins[vnode] = server + self._pending_targets[vnode] = server + return self.__origins[vnode] + + def createEdge(self, vnode: str) -> CDNEdgeServer: + if vnode in self.__origins: + raise AssertionError('vnode "{}" already used as origin'.format(vnode)) + if vnode not in self.__edges: + server = CDNEdgeServer() + self.__edges[vnode] = server + self._pending_targets[vnode] = server + return self.__edges[vnode] + + def setDomain( + self, + domain: str, + *, + dnsVnode: str, + edges: List[str], + mode: str = 'static', + zone: Optional[str] = None, + ) -> CDNService: + mode = mode.lower() + assert mode in ['static', 'round_robin', 'region', 'map'], 'unsupported CDN mode "{}"'.format(mode) + self.__domains[domain] = _DomainConfig( + domain=domain[:-1] if domain.endswith('.') else domain, + dns_vnode=dnsVnode, + edges=list(edges), + mode=mode, + zone=_normalize_zone_name(zone) if zone is not None else None, + ) + return self + + def mapRegion(self, domain: str, region: str, edges: List[str]) -> CDNService: + assert domain in self.__domains, 'unknown domain "{}"'.format(domain) + current_edges = self.__domains[domain].region_map.setdefault(region, []) + for edge in edges: + if edge not in current_edges: + current_edges.append(edge) + return self + + def setRegionMembers(self, region: str, asns: List[int]) -> CDNService: + if region not in self.__region_members: + self.__region_members[region] = set() + self.__region_members[region].update(asns) + return self + + def mapAsn(self, domain: str, asns: List[int], edges: List[str]) -> CDNService: + assert domain in self.__domains, 'unknown domain "{}"'.format(domain) + for asn in asns: + current_edges = self.__domains[domain].asn_map.setdefault(int(asn), []) + for edge in edges: + if edge not in current_edges: + current_edges.append(edge) + return self + + def setIncludeContent( + self, + domain: str, + file_path: str = '/etc/bind/include/custom.local', + ) -> CDNService: + assert domain in self.__domains, 'unknown domain "{}"'.format(domain) + self.__domains[domain].include_path = file_path + return self + + def enableMetrics(self, enabled: bool = True) -> CDNService: + self.__metrics_enabled = enabled + return self + + def getName(self) -> str: + return 'CDNService' + + def _doConfigure(self, node: Node, server: Server): + if isinstance(server, CDNOriginServer) or isinstance(server, CDNEdgeServer): + server.configure(node) + + def configure(self, emulator: Emulator): + super().configure(emulator) + self.__dns_artifacts = {} + + dns_layer: DomainNameService = emulator.getRegistry().get('seedemu', 'layer', self.__dns_service_name) + assert dns_layer is not None, 'CDNService requires DomainNameService' + + self.__prepareDnsZones(emulator, dns_layer) + self.__resolveEdgeOrigins() + self.__configureDns(emulator, dns_layer) + + def __prepareDnsZones(self, emulator: Emulator, dns_layer: DomainNameService): + zones_per_dns: Dict[str, Set[str]] = {} + + for domain_cfg in self.__domains.values(): + zone_name = domain_cfg.zone if domain_cfg.zone is not None else _infer_zone_name(domain_cfg.domain) + domain_cfg.zone = zone_name + zones_per_dns.setdefault(domain_cfg.dns_vnode, set()).add(zone_name) + + for edge_vnode in domain_cfg.edges: + assert edge_vnode in self.__edges, 'unknown edge vnode "{}" for domain "{}"'.format(edge_vnode, domain_cfg.domain) + self.__edges[edge_vnode].addServedDomain(domain_cfg.domain) + + dns_targets = dns_layer.getPendingTargets() + for (dns_vnode, zone_names) in zones_per_dns.items(): + assert dns_vnode in dns_targets, 'DNS vnode "{}" is not installed on DomainNameService'.format(dns_vnode) + dns_server: DomainNameServer = dns_targets[dns_vnode] + dns_node = emulator.getBindingFor(dns_vnode) + existing = set(dns_server.getZones()) + changed = False + for zone_name in zone_names: + if zone_name not in existing: + dns_server.addZone(zone_name) + changed = True + if changed: + dns_server.configure(dns_node, dns_layer) + + def __resolveEdgeOrigins(self): + for edge in self.__edges.values(): + resolved = [] + for origin_vnode in edge.getOriginVnodes(): + assert origin_vnode in self.__origins, 'unknown origin vnode "{}"'.format(origin_vnode) + origin = self.__origins[origin_vnode] + resolved.append((origin.getServiceAddress(), origin.getPort())) + edge.setResolvedOrigins(resolved) + + def __configureDns(self, emulator: Emulator, dns_layer: DomainNameService): + policy_domains: Dict[str, List[_DomainConfig]] = {} + + for domain_cfg in self.__domains.values(): + if domain_cfg.mode in ['static', 'round_robin']: + self.__configureSimpleDomain(dns_layer, domain_cfg) + else: + assert domain_cfg.include_path is not None, ( + 'domain "{}" uses mode "{}" and requires setIncludeContent()'.format( + domain_cfg.domain, domain_cfg.mode + ) + ) + policy_domains.setdefault(domain_cfg.dns_vnode, []).append(domain_cfg) + + for (dns_vnode, domain_cfgs) in policy_domains.items(): + self.__configureIncludeBackedDomains(emulator, dns_layer, dns_vnode, domain_cfgs) + + def __configureSimpleDomain(self, dns_layer: DomainNameService, domain_cfg: _DomainConfig): + zone = dns_layer.getZone(domain_cfg.zone) + record_name = _relative_record_name(domain_cfg.domain, domain_cfg.zone) + selected_edges = domain_cfg.edges[:1] if domain_cfg.mode == 'static' else domain_cfg.edges + for edge_vnode in selected_edges: + edge = self.__edges[edge_vnode] + zone.addRecord('{} A {}'.format(record_name, edge.getServiceAddress())) + + def __configureIncludeBackedDomains( + self, + emulator: Emulator, + dns_layer: DomainNameService, + dns_vnode: str, + domains: List[_DomainConfig], + ): + dns_targets = dns_layer.getPendingTargets() + dns_server: DomainNameServer = dns_targets[dns_vnode] + dns_node = emulator.getBindingFor(dns_vnode) + dns_addr = _choose_service_address(dns_node) + include_groups: Dict[str, List[_DomainConfig]] = {} + for domain_cfg in domains: + include_path = domain_cfg.include_path + assert include_path is not None, 'domain "{}" is missing include path'.format(domain_cfg.domain) + assert dns_server.getIncludePath(domain_cfg.zone) == include_path, ( + 'DNS include path mismatch for zone "{}": expected "{}", got "{}"'.format( + domain_cfg.zone, + include_path, + dns_server.getIncludePath(domain_cfg.zone), + ) + ) + include_groups.setdefault(include_path, []).append(domain_cfg) + + for (include_path, include_domains) in include_groups.items(): + zone_names = sorted(set(domain_cfg.zone for domain_cfg in include_domains)) + view_rules: List[Tuple[str, List[str], Dict[str, List[str]]]] = [] + acl_definitions: List[str] = [] + view_index = 0 + + for domain_cfg in include_domains: + edge_map = self.__buildDomainEdgeMap(emulator, domain_cfg) + for (_rule_name, (cidrs, edges)) in edge_map.items(): + acl_name = 'cdn_acl_{}_{}'.format(_safe_name(domain_cfg.domain), view_index) + view_name = 'cdn_view_{}_{}'.format(_safe_name(domain_cfg.domain), view_index) + acl_definitions.append( + 'acl "{}" {{ {} }};\n'.format(acl_name, ' '.join(['{};'.format(c) for c in cidrs])) + ) + view_rules.append((view_name, [acl_name], {domain_cfg.domain: edges})) + view_index += 1 + + default_domain_edges = { + domain_cfg.domain: domain_cfg.edges[:1] + for domain_cfg in include_domains + } + view_rules.append(('cdn_view_default', [], default_domain_edges)) + + include_content = ''.join(acl_definitions) + for (view_name, acl_names, domain_edges) in view_rules: + match_clients = 'any;' if len(acl_names) == 0 else ' '.join(['{};'.format(name) for name in acl_names]) + view_lines = [ + 'view "{}" {{'.format(view_name), + ' match-clients {{ {} }};'.format(match_clients), + ' recursion no;', + ] + + for zone_name in zone_names: + zone_path = '/etc/bind/zones/{}_{}.zone'.format(_safe_name(zone_name), _safe_name(view_name)) + zone_content = self.__buildZoneFileForView(dns_layer, zone_name, domain_edges, dns_addr) + self.__registerDnsArtifact(dns_vnode, zone_path, zone_content) + view_lines.append( + ' zone "{}" {{ type master; file "{}"; allow-update {{ any; }}; }};'.format( + zone_name if zone_name != '' else '.', zone_path + ) + ) + + view_lines.append('};\n') + include_content += '\n'.join(view_lines) + + self.__registerDnsArtifact(dns_vnode, include_path, include_content) + + def __registerDnsArtifact(self, dns_vnode: str, path: str, content: str): + self.__dns_artifacts.setdefault(dns_vnode, {})[path] = content + + def __buildDomainEdgeMap(self, emulator: Emulator, domain_cfg: _DomainConfig) -> Dict[str, Tuple[List[str], List[str]]]: + mapping: Dict[str, Tuple[List[str], List[str]]] = {} + + if domain_cfg.mode == 'map': + grouped: Dict[Tuple[str, ...], List[int]] = {} + for (asn, edges) in domain_cfg.asn_map.items(): + grouped.setdefault(tuple(edges), []).append(asn) + + for (idx, (edges, asns)) in enumerate(grouped.items()): + cidrs = self.__collectCidrsForAsns(emulator, asns) + if len(cidrs) > 0: + mapping['map_{}'.format(idx)] = (cidrs, list(edges)) + return mapping + + if domain_cfg.mode == 'region': + for (idx, (region, edges)) in enumerate(domain_cfg.region_map.items()): + asns = sorted(self.__region_members.get(region, set())) + cidrs = self.__collectCidrsForAsns(emulator, asns) + if len(cidrs) > 0: + mapping['region_{}'.format(idx)] = (cidrs, list(edges)) + return mapping + + return mapping + + def __collectCidrsForAsns(self, emulator: Emulator, asns: List[int]) -> List[str]: + cidrs: Set[str] = set() + reg = emulator.getRegistry() + target_asns = set(asns) + for ((scope, obj_type, _name), obj) in reg.getAll().items(): + if obj_type not in ['hnode', 'rnode']: + continue + try: + asn = int(scope) + except ValueError: + continue + if asn not in target_asns: + continue + node: Node = obj + for iface in node.getInterfaces(): + net = iface.getNet() + if net.getType() == NetworkType.Local: + cidrs.add(str(net.getPrefix())) + + return sorted(cidrs) + + def __buildZoneFileForView( + self, + dns_layer: DomainNameService, + zone_name: str, + domain_edges: Dict[str, List[str]], + dns_addr: str, + ) -> str: + zone = dns_layer.getZone(zone_name) + records = list(zone.getRecords()) + filtered_records = records[:2] + + if not any(' SOA ' in record for record in records): + zonename = zone_name if zone_name.endswith('.') else '{}.'.format(zone_name) + filtered_records.append('@ SOA ns1.{} admin.{} 1 900 900 1800 60'.format(zonename, zonename)) + + if not any(record.startswith('@ NS ') for record in records): + zonename = zone_name if zone_name.endswith('.') else '{}.'.format(zone_name) + filtered_records.append('@ NS ns1.{}'.format(zonename)) + + if not any(record.endswith(' A {}'.format(dns_addr)) and record.startswith('ns1') for record in records): + filtered_records.append('ns1 A {}'.format(dns_addr)) + + managed_domains = set() + for domain in domain_edges.keys(): + if self.__domains[domain].zone == zone_name: + managed_domains.add(_relative_record_name(domain, zone_name)) + + for record in records[2:]: + skip = False + for record_name in managed_domains: + if record.startswith('{} A '.format(record_name)): + skip = True + break + if not skip: + filtered_records.append(record) + + for (domain, edges) in domain_edges.items(): + if self.__domains[domain].zone != zone_name: + continue + record_name = _relative_record_name(domain, zone_name) + for edge_vnode in edges: + edge = self.__edges[edge_vnode] + filtered_records.append('{} A {}'.format(record_name, edge.getServiceAddress())) + + return '\n'.join(filtered_records) + + def render(self, emulator: Emulator): + super().render(emulator) + + for (dns_vnode, artifacts) in self.__dns_artifacts.items(): + dns_node = emulator.getBindingFor(dns_vnode) + for (path, content) in artifacts.items(): + dns_node.setFile(path, content) diff --git a/seedemu/services/DomainNameService.py b/seedemu/services/DomainNameService.py index 184351b76..2f16b6790 100644 --- a/seedemu/services/DomainNameService.py +++ b/seedemu/services/DomainNameService.py @@ -1,7 +1,7 @@ from __future__ import annotations from seedemu.core import Node, Printable, Emulator, Service, Server from seedemu.core.enums import NetworkType -from typing import List, Dict, Tuple, Set +from typing import List, Dict, Tuple, Set, Optional from re import sub from random import randint import requests @@ -136,7 +136,7 @@ def resolveTo(self, name: str, node: Node) -> Zone: assert len(ifaces) > 0, 'Node has no interfaces.' for iface in ifaces: net = iface.getNet() - if net.getType() == NetworkType.Host or net.getType() == NetworkType.Local: + if net.getType() == NetworkType.Local: address = iface.getAddress() break @@ -240,6 +240,7 @@ class DomainNameServer(Server): __node: Node __is_master: bool __is_real_root: bool + __include_paths: Dict[str, str] def __init__(self): """! @@ -250,6 +251,7 @@ def __init__(self): self.__zones = set() self.__is_master = False self.__is_real_root = False + self.__include_paths = {} def addZone(self, zonename: str, createNsAndSoa: bool = True) -> DomainNameServer: """! @@ -287,6 +289,42 @@ def setRealRootNS(self) -> DomainNameServer: return self + def setInclude(self, zonename: str, file_path: str = '/etc/bind/include/custom.local') -> DomainNameServer: + """! + @brief Register an include file for a hosted zone. + + When any include is registered, DomainNameService treats this DNS node + as include-backed and skips the default global zone stanzas. This is + required for custom BIND view-based configurations. + + @param zonename zone name. + @param file_path include file path inside container. + + @returns self, for chaining API calls. + """ + if zonename != '.' and not zonename.endswith('.'): + zonename += '.' + + self.__include_paths[zonename] = file_path + + return self + + def __usesIncludeConfig(self) -> bool: + return len(self.__include_paths) > 0 + + def getIncludePath(self, zonename: str) -> Optional[str]: + """! + @brief Get include file path for a hosted zone. + + @param zonename zone name. + + @returns include file path or None if not configured. + """ + if zonename != '.' and not zonename.endswith('.'): + zonename += '.' + + return self.__include_paths.get(zonename) + def getNode(self) -> Node: """! @brief get node associated with the server. Note that this only works @@ -363,6 +401,15 @@ def configure(self, node: Node, dns: DomainNameService): if len(zone.findRecords('SOA')) == 0: zone.addRecord('@ SOA {} {} {} 900 900 1800 60'.format('ns1.{}'.format(zonename), 'admin.{}'.format(zonename), randint(1, 0xffffffff))) + existing_ns_addr = False + for record in zone.getRecords(): + if record.endswith(' A {}'.format(addr)) and record.startswith('ns'): + existing_ns_addr = True + break + + if existing_ns_addr: + continue + #If there are multiple zone servers, increase the NS number for ns name. ns_number = 1 while (True): @@ -386,33 +433,60 @@ def install(self, node: Node, dns: DomainNameService): assert node == self.__node, 'configured node differs from install node. Please check if there are conflict bindings' node.addSoftware('bind9') - node.appendStartCommand('echo "include \\"/etc/bind/named.conf.zones\\";" >> /etc/bind/named.conf.local') - node.setFile('/etc/bind/named.conf.options', DomainNameServiceFileTemplates['named_options']) + if not self.__usesIncludeConfig(): + node.setFile( + '/etc/bind/named.conf', + '''// Generated by seedemu DomainNameService +include "/etc/bind/named.conf.options"; +include "/etc/bind/named.conf.local"; +include "/etc/bind/named.conf.default-zones"; +''' + ) + else: + node.setFile( + '/etc/bind/named.conf', + '''// Generated by seedemu DomainNameService +include "/etc/bind/named.conf.options"; +include "/etc/bind/named.conf.local"; +''' + ) + node.setFile( + '/etc/bind/named.conf.options', + DomainNameServiceFileTemplates['named_options'] + ) + + named_conf_local_parts = [] + if not self.__usesIncludeConfig(): + named_conf_local_parts.append('include "/etc/bind/named.conf.zones";\n') + for include_path in dict.fromkeys(self.__include_paths.values()).keys(): + named_conf_local_parts.append('include "{}";\n'.format(include_path)) + node.setFile('/etc/bind/named.conf.local', ''.join(named_conf_local_parts)) node.setFile('/etc/bind/named.conf.zones', '') - for (_zonename, auto_ns_soa) in self.__zones: - zone = dns.getZone(_zonename) - zonename = filename = zone.getName() + if not self.__usesIncludeConfig(): + for (_zonename, auto_ns_soa) in self.__zones: + zone = dns.getZone(_zonename) + zonename = filename = zone.getName() - if zonename == '' or zonename == '.': - filename = 'root' - zonename = '.' - zonepath = '/etc/bind/zones/{}'.format(filename) - node.setFile(zonepath, '\n'.join(zone.getRecords())) + if zonename == '' or zonename == '.': + filename = 'root' + zonename = '.' + zonepath = '/etc/bind/zones/{}'.format(filename) + node.setFile(zonepath, '\n'.join(zone.getRecords())) - if self.__is_master: - node.appendFile('/etc/bind/named.conf.zones', - 'zone "{}" {{ type master; notify yes; allow-transfer {{ any; }}; file "{}"; allow-update {{ any; }}; }};\n'.format(zonename, zonepath) + if self.__is_master: + node.appendFile('/etc/bind/named.conf.zones', + 'zone "{}" {{ type master; notify yes; allow-transfer {{ any; }}; file "{}"; allow-update {{ any; }}; }};\n'.format(zonename, zonepath) + ) + elif zone.getName() in dns.getMasterIp().keys(): # Check if there are some master servers + master_ips = ';'.join(dns.getMasterIp()[zone.getName()]) + node.appendFile('/etc/bind/named.conf.zones', + 'zone "{}" {{ type slave; masters {{ {}; }}; file "{}"; }};\n'.format(zonename, master_ips, zonepath) + ) + else: + node.appendFile('/etc/bind/named.conf.zones', + 'zone "{}" {{ type master; file "{}"; allow-update {{ any; }}; }};\n'.format(zonename, zonepath) ) - elif zone.getName() in dns.getMasterIp().keys(): # Check if there are some master servers - master_ips = ';'.join(dns.getMasterIp()[zone.getName()]) - node.appendFile('/etc/bind/named.conf.zones', - 'zone "{}" {{ type slave; masters {{ {}; }}; file "{}"; }};\n'.format(zonename, master_ips, zonepath) - ) - else: - node.appendFile('/etc/bind/named.conf.zones', - 'zone "{}" {{ type master; file "{}"; allow-update {{ any; }}; }};\n'.format(zonename, zonepath) - ) node.appendStartCommand('chown -R bind:bind /etc/bind/zones') node.appendStartCommand('service named start') diff --git a/seedemu/services/EthereumService/EthTemplates/EthServerFileTemplates.py b/seedemu/services/EthereumService/EthTemplates/EthServerFileTemplates.py index bf4f88dab..4fe24512a 100644 --- a/seedemu/services/EthereumService/EthTemplates/EthServerFileTemplates.py +++ b/seedemu/services/EthereumService/EthTemplates/EthServerFileTemplates.py @@ -16,13 +16,14 @@ def get_file_content(filename): 'bootstrapper': get_file_content("files_ethereum/bootstrapper.sh"), 'beacon_bootstrapper': get_file_content("files_ethereum/beacon_bootstrapper.sh"), 'fetch_bn_enr': get_file_content("files_ethereum/fetch_bn_enr.sh"), + 'vc_bootstrapper': get_file_content("files_ethereum/vc_bootstrapper.sh"), } UtilityServerFileTemplates: Dict[str, str] = { 'fund_account': get_file_content("files_utility/fund_account.py"), 'deploy_contract': get_file_content("files_utility/deploy_contract.py"), 'utility_server': get_file_content("files_utility/utility_server.py"), - 'server_setup': get_file_content("files_utility/utility_server_setup.py") + 'server_setup': get_file_content("files_utility/utility_server_setup.sh") } FaucetServerFileTemplates: Dict[str, str] = { @@ -33,4 +34,3 @@ def get_file_content(filename): 'faucet_fund_url': "http://{address}:{port}/fundme", 'fund_curl': "curl -X POST -d 'address={recipient}&amount={amount}' http://{address}:{port}/fundme" } - diff --git a/seedemu/services/EthereumService/EthTemplates/LighthouseCommandTemplates.py b/seedemu/services/EthereumService/EthTemplates/LighthouseCommandTemplates.py index 50c617e44..8e108f7fe 100644 --- a/seedemu/services/EthereumService/EthTemplates/LighthouseCommandTemplates.py +++ b/seedemu/services/EthereumService/EthTemplates/LighthouseCommandTemplates.py @@ -1,13 +1,14 @@ -LIGHTHOUSE_BN_CMD = """lighthouse --debug-level info bn --datadir /tmp/bn/local-testnet/testnet --testnet-dir /tmp/bn/local-testnet/testnet --enable-private-discovery --staking --enr-address {ip_address} --enr-udp-port 9000 --enr-tcp-port 9000 --port 9000 --http-address {ip_address} --http-port 8000 --http-allow-origin "*" --disable-packet-filter --target-peers {target_peers} --execution-endpoint http://localhost:8551 --execution-jwt /tmp/jwt.hex --subscribe-all-subnets {bootnodes_flag} &""" +LIGHTHOUSE_BN_CMD = """lighthouse --debug-level info bn --datadir /tmp/bn/local-testnet/testnet --testnet-dir /tmp/bn/local-testnet/testnet --enable-private-discovery --staking --enr-address {ip_address} --reconstruct-historic-states --enr-udp-port 9000 --enr-tcp-port 9000 --port 9000 --http-address {ip_address} --http-port 8000 --http-allow-origin "*" --disable-packet-filter --target-peers {target_peers} --execution-endpoint http://{remote_geth}:8551 --execution-jwt /tmp/jwt.hex {bootnodes_flag} &""" -LIGHTHOUSE_VC_CMD = """lighthouse --debug-level info vc --datadir /tmp/vc/local-testnet/testnet --testnet-dir /tmp/vc/local-testnet/testnet --init-slashing-protection --beacon-nodes http://{ip_address}:8000 --suggested-fee-recipient {acct_address} --http --http-address 0.0.0.0 --http-allow-origin "*" --unencrypted-http-transport &""" +LIGHTHOUSE_VC_CMD = """lighthouse --debug-level info vc --datadir /tmp/vc/local-testnet/testnet --testnet-dir /tmp/vc/local-testnet/testnet --init-slashing-protection --beacon-nodes http://{beacon_node}:8000 --suggested-fee-recipient {acct_address} --http --http-address 0.0.0.0 --http-allow-origin "*" --unencrypted-http-transport &""" LIGHTHOUSE_WALLET_CREATE_CMD = """lighthouse account_manager wallet create --testnet-dir /tmp/vc/local-testnet/testnet --datadir /tmp/vc/local-testnet/testnet --name "seed" --password-file /tmp/seed.pass""" LIGHTHOUSE_VALIDATOR_CREATE_CMD = """lighthouse --testnet-dir /tmp/vc/local-testnet/testnet --datadir /tmp/vc/local-testnet/testnet account validator create --wallet-name seed --wallet-password /tmp/seed.pass --count 1""" + VALIDATOR_DEPOSIT_PY = """\ from web3 import Web3 from eth_account import Account @@ -76,4 +77,3 @@ print(f"[deposit.py] Deposit tx sent! Tx hash: {{w3.to_hex(tx_hash)}}") """ - diff --git a/seedemu/services/EthereumService/EthTemplates/files_ethereum/beacon_bootstrapper.sh b/seedemu/services/EthereumService/EthTemplates/files_ethereum/beacon_bootstrapper.sh index 179b3faf2..74e3b05e0 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_ethereum/beacon_bootstrapper.sh +++ b/seedemu/services/EthereumService/EthTemplates/files_ethereum/beacon_bootstrapper.sh @@ -13,52 +13,15 @@ while read -r node; do {{ break }} }}; done - ($ok) && {{ - validatorAtGenesis={is_validator_at_genesis} - validatorAtRunning={is_validator_at_running} - + curl --http0.9 -s http://$node/testnet > /tmp/testnet.tar.gz mkdir /tmp/bn tar -xzvf /tmp/testnet.tar.gz -C /tmp/bn - ($validatorAtGenesis || $validatorAtRunning) && {{ - mkdir /tmp/vc - tar -xzvf /tmp/testnet.tar.gz -C /tmp/vc - }} - - ($validatorAtGenesis) && {{ - eth2-val-tools keystores --source-mnemonic "{validator_mnemonic}" --source-max {validator_key_end} --source-min {validator_key_start} --out-loc "/tmp/vc/assigned_data" - mkdir /tmp/vc/local-testnet/testnet/validators - cp -r /tmp/vc/assigned_data/keys/* /tmp/vc/local-testnet/testnet/validators/ - cp -r /tmp/vc/assigned_data/secrets /tmp/vc/local-testnet/testnet/ - }} - - - echo "[beacon_client] starting lighthouse beacon client" {bc_start_command} - ($validatorAtRunning) && {{ - echo "creating validator wallet and deposit" - {wallet_create_command} - {validator_create_command} - {validator_deposit_sh} - - }} - - # while loop till bc client is ready - while ! curl --http0.9 -sHf http://{ip_address}:8000 > /dev/null; do - echo "eth: local beacon node not ready, waiting..." - sleep 3 - done - - - ($validatorAtGenesis || $validatorAtRunning) && {{ - echo "[validator client] starting lighthouse validator client" - {vc_start_command} - }} - - }} + }}; done < /tmp/beacon-setup-node diff --git a/seedemu/services/EthereumService/EthTemplates/files_ethereum/bootstrapper.sh b/seedemu/services/EthereumService/EthTemplates/files_ethereum/bootstrapper.sh index bc2445e0b..dc4878242 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_ethereum/bootstrapper.sh +++ b/seedemu/services/EthereumService/EthTemplates/files_ethereum/bootstrapper.sh @@ -19,4 +19,4 @@ while read -r node; do { break fi }; done -}; done < /tmp/eth-nodes +}; done < /tmp/geth-eth-nodes diff --git a/seedemu/services/EthereumService/EthTemplates/files_ethereum/fetch_bn_enr.sh b/seedemu/services/EthereumService/EthTemplates/files_ethereum/fetch_bn_enr.sh index a2536d7b7..1b337e7af 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_ethereum/fetch_bn_enr.sh +++ b/seedemu/services/EthereumService/EthTemplates/files_ethereum/fetch_bn_enr.sh @@ -1,7 +1,7 @@ #!/bin/bash -BOOTNODES_FILE="/tmp/eth-nodes" -OUTPUT_ENR_FILE="/tmp/bc_enrs.txt" +BOOTNODES_FILE="/tmp/beacon-eth-nodes" ## geth and beacon +OUTPUT_ENR_FILE="/tmp/bc_enrs.txt" MAX_RETRIES=60 SLEEP_SECONDS=3 diff --git a/seedemu/services/EthereumService/EthTemplates/files_ethereum/vc_bootstrapper.sh b/seedemu/services/EthereumService/EthTemplates/files_ethereum/vc_bootstrapper.sh new file mode 100644 index 000000000..3ece12af1 --- /dev/null +++ b/seedemu/services/EthereumService/EthTemplates/files_ethereum/vc_bootstrapper.sh @@ -0,0 +1,63 @@ +#!/bin/bash +while read -r node; do {{ + let count=0 + ok=true + validator=false + until curl --http0.9 -sHf http://$node/testnet > /dev/null; do {{ + echo "eth: beacon-setup node $node not ready, waiting..." + sleep 10 + let count++ + [ $count -gt 60 ] && {{ + echo "eth: connection to beacon-setup node $node failed too many times, skipping." + ok=false + break + }} + }}; done + ($ok) && {{ + validatorAtGenesis={is_validator_at_genesis} + validatorAtRunning={is_validator_at_running} + + curl --http0.9 -s http://$node/testnet > /tmp/testnet.tar.gz + mkdir /tmp/bn + tar -xzvf /tmp/testnet.tar.gz -C /tmp/bn + + ($validatorAtGenesis || $validatorAtRunning) && {{ + mkdir /tmp/vc + tar -xzvf /tmp/testnet.tar.gz -C /tmp/vc + }} + + + ($validatorAtGenesis) && {{ + eth2-val-tools keystores --source-mnemonic "{validator_mnemonic}" --source-max {validator_key_end} --source-min {validator_key_start} --out-loc "/tmp/vc/assigned_data" + mkdir /tmp/vc/local-testnet/testnet/validators + cp -r /tmp/vc/assigned_data/keys/* /tmp/vc/local-testnet/testnet/validators/ + cp -r /tmp/vc/assigned_data/secrets /tmp/vc/local-testnet/testnet/ + }} + + + + + + ($validatorAtRunning) && {{ + echo "creating validator wallet and deposit" + {wallet_create_command} + {validator_create_command} + {validator_deposit_sh} + }} + + # while loop till bc client is ready + while ! curl --http0.9 -sHf http://{ip_address}:8000 > /dev/null; do + echo "eth: local beacon node not ready, waiting..." + sleep 3 + done + + + ($validatorAtGenesis || $validatorAtRunning) && {{ + echo "[validator client] starting lighthouse validator client" + {vc_start_command} + }} + + }} +}}; done < /tmp/beacon-setup-node + + diff --git a/seedemu/services/EthereumService/EthTemplates/files_faucet/faucet_server.py b/seedemu/services/EthereumService/EthTemplates/files_faucet/faucet_server.py index c808033c6..1c3ec2300 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_faucet/faucet_server.py +++ b/seedemu/services/EthereumService/EthTemplates/files_faucet/faucet_server.py @@ -1,4 +1,7 @@ from flask import Flask, request, jsonify +import inspect +if not hasattr(inspect, "getargspec"): + inspect.getargspec = inspect.getfullargspec from web3 import Web3 from web3.middleware import geth_poa_middleware import sys, time diff --git a/seedemu/services/EthereumService/EthTemplates/files_faucet/fundme.py b/seedemu/services/EthereumService/EthTemplates/files_faucet/fundme.py index ddcbb2f67..d63a0eb23 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_faucet/fundme.py +++ b/seedemu/services/EthereumService/EthTemplates/files_faucet/fundme.py @@ -2,6 +2,9 @@ import time, sys, json import requests +import inspect +if not hasattr(inspect, "getargspec"): + inspect.getargspec = inspect.getfullargspec from eth_account import Account import logging diff --git a/seedemu/services/EthereumService/EthTemplates/files_utility/deploy_contract.py b/seedemu/services/EthereumService/EthTemplates/files_utility/deploy_contract.py index f4d664473..25ecdf0f7 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_utility/deploy_contract.py +++ b/seedemu/services/EthereumService/EthTemplates/files_utility/deploy_contract.py @@ -1,6 +1,9 @@ #!/bin/env python3 import time +import inspect +if not hasattr(inspect, "getargspec"): + inspect.getargspec = inspect.getfullargspec from web3 import Web3, HTTPProvider import os import logging diff --git a/seedemu/services/EthereumService/EthTemplates/files_utility/fund_account.py b/seedemu/services/EthereumService/EthTemplates/files_utility/fund_account.py index 73fc975b7..8d034b9b2 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_utility/fund_account.py +++ b/seedemu/services/EthereumService/EthTemplates/files_utility/fund_account.py @@ -1,6 +1,9 @@ #!/bin/env python3 import time +import inspect +if not hasattr(inspect, "getargspec"): + inspect.getargspec = inspect.getfullargspec from web3 import Web3, HTTPProvider import requests import logging diff --git a/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server.py b/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server.py index d35485b99..c01f94356 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server.py +++ b/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server.py @@ -2,6 +2,9 @@ from flask import Flask, request, jsonify import os +import inspect +if not hasattr(inspect, "getargspec"): + inspect.getargspec = inspect.getfullargspec from web3 import Web3, HTTPProvider import json diff --git a/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.py b/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.sh similarity index 99% rename from seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.py rename to seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.sh index d0666f97f..b0c38ab6f 100644 --- a/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.py +++ b/seedemu/services/EthereumService/EthTemplates/files_utility/utility_server_setup.sh @@ -5,4 +5,3 @@ python3 ./fund_account.py python3 ./deploy_contract.py - diff --git a/seedemu/services/EthereumService/EthUtilityServer.py b/seedemu/services/EthereumService/EthUtilityServer.py index d8e1eb95a..c0b25e8ee 100644 --- a/seedemu/services/EthereumService/EthUtilityServer.py +++ b/seedemu/services/EthereumService/EthUtilityServer.py @@ -132,7 +132,7 @@ def install(self, node: Node): node.appendClassName("EthUtilityServer") node.addSoftware('python3 python3-pip') - node.addBuildCommand('pip3 install flask web3==5.31.1') + node.addBuildCommand('pip3 install --break-system-packages flask web3==5.31.1 || pip3 install flask web3==5.31.1') self._installScriptFile(node) diff --git a/seedemu/services/EthereumService/EthereumServer.py b/seedemu/services/EthereumService/EthereumServer.py index bebb33b69..fb1406764 100644 --- a/seedemu/services/EthereumService/EthereumServer.py +++ b/seedemu/services/EthereumService/EthereumServer.py @@ -17,6 +17,7 @@ class EthereumServer(Server): _id: int _blockchain: Blockchain _is_bootnode: bool + _is_beaconviewer: bool _bootnode_http_port: int _smart_contract: SmartContract _accounts: List[AccountStructure] @@ -55,6 +56,7 @@ def __init__(self, id: int, blockchain:Blockchain): self._id = id self._blockchain = blockchain self._is_bootnode = False + self._is_beaconviewer = False self._bootnode_http_port = 8088 self._smart_contract = None self._accounts = [] @@ -161,30 +163,51 @@ def install(self, node: Node, eth: EthereumService): # copy keystore to the proper folder for account in self._accounts: + # if isinstance(self, PoSGethServer) or isinstance(self, PoAServer): node.appendStartCommand("cp /tmp/keystore/{} /root/.ethereum/keystore/".format(account.keystore_filename)) if self._is_bootnode: - # generate enode url. other nodes will access this to bootstrap the network. - node.appendStartCommand('[ ! -e "/root/.ethereum/geth/bootkey" ] && bootnode -genkey /root/.ethereum/geth/bootkey') - node.appendStartCommand('echo "enode://$(bootnode -nodekey /root/.ethereum/geth/bootkey -writeaddress)@{}:30301" > /tmp/eth-enode-url'.format(addr)) - - # Default port is 30301, use -addr : to specify a custom port - node.appendStartCommand('bootnode -nodekey /root/.ethereum/geth/bootkey -verbosity 9 -addr {}:30301 2> /tmp/bootnode-logs &'.format(addr)) - node.appendStartCommand('python3 -m http.server {} -d /tmp'.format(self._bootnode_http_port), True) - - # get other nodes IP for the bootstrapper. + if isinstance(self, PoSGethServer) or isinstance(self, PoAServer): + # generate enode url. other nodes will access this to bootstrap the network. + node.appendStartCommand('[ ! -e "/root/.ethereum/geth/bootkey" ] && bootnode -genkey /root/.ethereum/geth/bootkey') + node.appendStartCommand('echo "enode://$(bootnode -nodekey /root/.ethereum/geth/bootkey -writeaddress)@{}:30301" > /tmp/eth-enode-url'.format(addr)) + + # Default port is 30301, use -addr : to specify a custom port + node.appendStartCommand('bootnode -nodekey /root/.ethereum/geth/bootkey -verbosity 9 -addr {}:30301 2> /tmp/bootnode-logs &'.format(addr)) + node.appendStartCommand('python3 -m http.server {} -d /tmp'.format(self._bootnode_http_port), True) + + # get other nodes IP for the bootstrapper. bootnodes = self._blockchain.getBootNodes()[:] if len(bootnodes) > 0: - node.setFile('/tmp/eth-nodes', '\n'.join(bootnodes)) + node.setFile('/tmp/geth-eth-nodes', '\n'.join(bootnodes)) node.setFile('/tmp/eth-bootstrapper', EthServerFileTemplates['bootstrapper']) # load enode urls from other nodes node.appendStartCommand('chmod +x /tmp/eth-bootstrapper') node.appendStartCommand('/tmp/eth-bootstrapper') + try: + beacon_bootnodes = list(self._blockchain.getBeaconBootNodes()[:]) + except Exception: + pass + try: + geth_bootnodes = list(self._blockchain.getGethBootNodes()[:]) + except Exception: + pass + + if len(beacon_bootnodes) > 0 and isinstance(self, PoSBeaconServer): + node.setFile('/tmp/beacon-eth-nodes', '\n'.join(beacon_bootnodes)) + if len(geth_bootnodes) > 0 and isinstance(self, PoSGethServer): + node.setFile('/tmp/geth-eth-nodes', '\n'.join(geth_bootnodes)) + + node.setFile('/tmp/eth-bootstrapper', EthServerFileTemplates['bootstrapper']) + node.appendStartCommand('chmod +x /tmp/eth-bootstrapper') + node.appendStartCommand('/tmp/eth-bootstrapper') + # node.appendStartCommand(self._geth_start_command, True) # launch Ethereum process. - node.appendStartCommand(self._geth_start_command, True) + if isinstance(self, PoSGethServer) or isinstance(self, PoAServer): + node.appendStartCommand(self._geth_start_command, True) # Rarely used and tentatively not supported. @@ -279,6 +302,26 @@ def isBootNode(self) -> bool: """ return self._is_bootnode + + def addViewercmd(self, isBeaconViewer: bool) -> EthereumServer: + """! + @brief set bootnode status of this node. + Note: if no nodes are configured as boot nodes, all nodes will be each + other's boot nodes. + @param isBootNode True to set this node as a bootnode, False otherwise. + + @returns self, for chaining API calls. + """ + self._is_beaconviewer = isBeaconViewer + + return self + def isBeaconViewer(self) -> bool: + """! + @brief get bootnode status of this node. + @returns True if this node is a boot node. False otherwise. + """ + return self._is_beaconviewer + def setBootNodeHttpPort(self, port: int) -> EthereumServer: """! @brief set the http server port number hosting the enode url file. @@ -728,30 +771,175 @@ def setBeaconSetupHttpPort(self, port:int): -class BeaconSetupServer(): +class PoSGethServer(EthereumServer): + def __init__(self, id: int, blockchain:Blockchain): + """! + @brief Create new geth server. + + @param id The serial number of this server. + """ + super().__init__(id, blockchain) + def _generateGethStartCommand(self, addr:str): + self._geth_options['pos'] = GethCommandTemplates['pos'] + super()._generateGethStartCommand(addr) + def install(self, node: Node, eth: EthereumService): + # if self.__is_beacon_setup_node: + # beacon_setup_node = PoSBeaconSetupServer() + # beacon_setup_node.install(self, node, self._blockchain) + # return + super().install(node,eth) + node.setFile('/tmp/jwt.hex', '0xae7177335e3d4222160e08cecac0ace2cecce3dc3910baada14e26b11d2009fc') +class PoSBeaconServer(EthereumServer): + __beacon_peer_counts:int + def __init__(self, id: int, blockchain:Blockchain): + + """! + @brief Create new beacon server. + + @param id The serial number of this server. + """ + + super().__init__(id, blockchain) + + self.__beacon_peer_counts = 30 + self.__connect_geth_vnode = "" + self.__connected_geth_ip = "" + # self.__validator_mnemonic = "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" + + def install(self, node: Node, eth: EthereumService): + """! + @brief Install the beacon server on the node. + + @param node The node to install the beacon server on. + @param eth The Ethereum service to use. + """ + # if self.__is_beacon_setup_node: + # beacon_setup_node = PoSBeaconSetupServer() + # beacon_setup_node.install(self, node, self._blockchain) + # return + + # if self.__is_beacon_validator_at_genesis: + # self._role.append("validator_at_genesis") + # if self.__is_beacon_validator_at_running: + # self._role.append("validator_at_running") + + super().install(node,eth) + self.__install_beacon(node, eth) + def __install_beacon(self, node:Node, eth:EthereumService): + ifaces = node.getInterfaces() + assert len(ifaces) > 0, 'EthereumServer::install: node as{}/{} has no interfaces'.format(node.getAsn(), node.getName()) + addr = str(ifaces[0].getAddress()) + # Get the IP address of beacon_setup_node. + beacon_setup_node = self._blockchain.getBeaconSetupNodeIp() + + assert beacon_setup_node != "", 'EthereumServer::install: Ethereum Service has no beacon_setup_node.' + + geth_node_ip = self.getConnectedGethIp() + + # connect_geth_node = self.__blockchain.getIpByVnodeName(self.__connect_geth_vnode) + bootnode_start_command = "" + bc_start_command = LIGHTHOUSE_BN_CMD.format(ip_address=addr, target_peers=self.__beacon_peer_counts, bootnodes_flag="" if self._is_bootnode else f'--boot-nodes "$(cat /tmp/bc_enrs.txt)"',remote_geth=geth_node_ip) + # bc_start_command = LIGHTHOUSE_BN_CMD.format(ip_address=addr, target_peers=self.__beacon_peer_counts, bootnodes_flag="" if self._is_bootnode else f'--boot-nodes "$(cat /tmp/bc_enrs.txt)"', viewer_cmd = f'--reconstruct-historic-states' if self.isBeaconViewer() else "",remote_geth=geth_node_ip) + + ## div beacon node and geth node + if not self._is_bootnode: + node.setFile('/tmp/fetch_bn_enr', EthServerFileTemplates['fetch_bn_enr']) + node.appendStartCommand('chmod +x /tmp/fetch_bn_enr') + node.appendStartCommand('/tmp/fetch_bn_enr') + + # if self._is_bootnode: + # bootnode_start_command = LIGHTHOUSE_BOOTNODE_CMD.format(ip_address=addr) + # if self.__is_beacon_validator_at_running: + # node.setFile('/tmp/seed.pass', 'seedseedseed') + # wallet_create_command = LIGHTHOUSE_WALLET_CREATE_CMD.format(eth_id=self.getId()) + # validator_create_command = LIGHTHOUSE_VALIDATOR_CREATE_CMD.format(eth_id=self.getId()) + + # if len(self._accounts) > 0: + # print(self._accounts[0].keystore_filename) + # node.setFile('/tmp/deposit.py', VALIDATOR_DEPOSIT_PY.format(keystore_filename=self._accounts[0].keystore_filename)) + + # if not self.__is_manual_deposit_for_validator: + # validator_deposit_sh = "sleep 2 && python3 /tmp/deposit.py" + + # node.appendStartCommand('chmod +x /tmp/deposit.sh') + # if not self.__is_manual_deposit_for_validator: + # validator_deposit_sh = "/tmp/deposit.sh" + # if self.__is_beacon_validator_at_genesis or self.__is_beacon_validator_at_running: + # vc_start_command = LIGHTHOUSE_VC_CMD.format(ip_address=addr, acct_address=self._accounts[0].address) + # Write the IP address of beacon_setup_node to a file. + node.setFile('/tmp/beacon-setup-node', beacon_setup_node) + + # get current node validator index if it's validator at genesis. + # validatorIds = self._blockchain.getValidatorIds() + # validator_idx = -1 + # if self.isValidatorAtGenesis(): + # # only get validator id if current node is validator at genesis. + # for i, v in enumerate(validatorIds): + # if int(v) == self._id: + # validator_idx = i + # break + # assert validator_idx >= 0, "EthereumServer::__install_beacon: validator id should be set at genesis." + # validator_idx = int(validator_idx) + # # this validator idx will be used to create validator keys properly. + # # Each validator at genesis node has 4 keys which were set at Genesis + + node.setFile('/tmp/beacon-bootstrapper', EthServerFileTemplates['beacon_bootstrapper'].format( + bc_start_command=bc_start_command, + )) + node.setFile('/tmp/jwt.hex', '0xae7177335e3d4222160e08cecac0ace2cecce3dc3910baada14e26b11d2009fc') + + node.appendStartCommand('chmod +x /tmp/beacon-bootstrapper') + node.appendStartCommand('/tmp/beacon-bootstrapper') + + def setBeaconPeerCounts(self, peer_counts:int): + self.__beacon_peer_counts = peer_counts + return self + def connectToGethNode(self, vnode:str): + self.__connect_geth_vnode = vnode + return self + + def setConnectedGethIp(self, ip:str): + self.__connected_geth_ip = ip + return self + def getConnectedGethIp(self) -> str: + return self.__connected_geth_ip + def getConnectGethVNode(self) -> str: + return self.__connect_geth_vnode + + def getValidatorMnemonic(self): + return self.__validator_mnemonic + + def setValidatorMnemonic(self, mnemonic:str): + """! + @brief Set the validator mnemonic for the beacon setup node. + This mnemonic will be used to generate validator keys at genesis and running. + + @param mnemonic The mnemonic to set. + """ + self.__validator_mnemonic = mnemonic + return self +class PoSBeaconSetupServer(EthereumServer): """! @brief The WebServer class. """ - - __beacon_setup_http_port: int - def __init__(self, consensus:ConsensusMechanism = ConsensusMechanism.POA): + def __init__(self, id: int, blockchain:Blockchain): """! @brief BeaconSetupServer constructor. """ - + super().__init__(id, blockchain) self.__beacon_setup_http_port = 8090 - self.__consensus_mechanism = consensus + self.__validator_mnemonic = "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" - def install(self, server:PoSServer, node: Node, blockchain: Blockchain): + def install(self, node: Node,eth: EthereumService ): """! @brief Install the service. """ - validator_ids = blockchain.getValidatorIds() + validator_ids = self._blockchain.getValidatorIds() validator_counts = len(validator_ids) assert validator_counts > 0, "BeaconSetupServer::install: At least one node should be set as validator at genesis." @@ -761,14 +949,27 @@ def install(self, server:PoSServer, node: Node, blockchain: Blockchain): #node.addBuildCommand('apt-get update && apt-get install -y --no-install-recommends software-properties-common python3 python3-pip') #node.addBuildCommand('pip install web3') # [remove] node.appendStartCommand('lcli generate-bootnode-enr --ip {} --udp-port 30305 --tcp-port 30305 --genesis-fork-version 0x42424242 --output-dir /local-testnet/bootnode'.format(bootnode_ip)) - node.setFile("/tmp/config.yaml", BEACON_GENESIS.format(chain_id=blockchain.getChainId(), - target_committee_size=blockchain.getTargetCommitteeSize(), - target_aggregator_per_committee=blockchain.getTargetAggregatorPerCommittee())) + node.setFile("/tmp/config.yaml", BEACON_GENESIS.format(chain_id=self._blockchain.getChainId(), + target_committee_size=self._blockchain.getTargetCommitteeSize(), + target_aggregator_per_committee=self._blockchain.getTargetAggregatorPerCommittee())) # [remove] node.setFile("/tmp/validator-ids", "\n".join(validator_ids)) - self.__genesis = blockchain.getGenesis() + self.__genesis = self._blockchain.getGenesis() node.setFile('/tmp/eth1-genesis.json', self.__genesis.getGenesis()) - node.setFile("/tmp/mnemonic.yaml", BEACON_MNEMONIC_YAML.format(validator_count = validator_counts, validator_mnemonic=server.getValidatorMnemonic())) + mnemonic_lines = [] + for idx, vid in enumerate(validator_ids): + wd_address = self._blockchain.getValidatorWithdrawAddress(vid) + assert wd_address, f"PoSBeaconSetupServer::install: missing withdraw address for validator id {vid}." + mnemonic_lines.append( + f'- mnemonic: "{self.getValidatorMnemonic()}"\n' + f" start: {idx}\n" + " count: 1\n" + " balance: 32000000000\n" + f' wd_address: "{wd_address}"\n' + ' wd_prefix: "0x01"\n' + " status: 0\n" + ) + node.setFile("/tmp/mnemonic.yaml", "\n".join(mnemonic_lines)) node.appendStartCommand('mkdir /local-testnet/testnet') # [remove] node.appendStartCommand('bootnode_enr=`cat /local-testnet/bootnode/enr.dat`') # [remove] node.appendStartCommand('echo "- $bootnode_enr" > /local-testnet/testnet/boot_enr.yaml') @@ -810,3 +1011,215 @@ def print(self, indent: int) -> str: out += 'Beacon Setup server object.\n' return out + def getValidatorMnemonic(self): + return self.__validator_mnemonic + + +class PoSVcServer(EthereumServer): + + def __init__(self, id: int, blockchain:Blockchain): + """! + @brief Create new vc server. + + @param id The serial number of this server. + """ + super().__init__(id, blockchain) + self.__is_beacon_validator_at_genesis = False + self.__is_beacon_validator_at_running = False + self.__is_manual_deposit_for_validator = False + self.__validator_mnemonic = "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" + self.__withdraw_mnemonic = "legal winner thank year wave sausage worth useful legal winner thank yellow" + self.__withdraw_account = EthAccount.createEmulatorAccountFromMnemonic( + self._id, + mnemonic=self.__withdraw_mnemonic, + balance=0, + index=0, + password="admin", + ) + self.__withdraw_address_override = "" + self.__is_connect_beacon_node = True + self.__conect_beacon_vnode :str ="" + self.__connected_beacon_ip :str = "" + + + def _createAccounts(self, eth:EthereumService) -> EthereumServer: + """! + @brief Create withdraw account. + + """ + super()._createAccounts(eth) + return self + + def getWithdrawMnemonic(self) -> str: + """! + @brief Get the withdraw account mnemonic. + + @return The withdraw account mnemonic. + """ + return self.__withdraw_mnemonic + + def getWithdrawAddress(self) -> str: + """! + @brief Get the withdraw account address. + + @return The withdraw account address. + """ + if self.__withdraw_address_override: + return self.__withdraw_address_override + return self.__withdraw_account.address + + def getWithdrawAccount(self) -> AccountStructure: + return self.__withdraw_account + + def setWithdrawAddress(self, address: str) -> PoSVcServer: + """! + @brief Set the withdraw account address. + + @param address The withdraw account address. + """ + if address == "": + return self + assert address, "PoSVcServer::setWithdrawAddress: address cannot be empty." + from web3 import Web3 + self.__withdraw_address_override = Web3.toChecksumAddress(address) + return self + + def setWithdrawMnemonic(self, mnemonic: str, index: int = 0, password: str = "admin") -> PoSVcServer: + """! + @brief Set the withdraw account mnemonic. + + @param mnemonic The withdraw account mnemonic. + @param index The index of the withdraw account. + @param password The password to use. + """ + if mnemonic == "": + return self + assert mnemonic, "PoSVcServer::setWithdrawMnemonic: mnemonic cannot be empty." + self.__withdraw_mnemonic = mnemonic + self.__withdraw_account = EthAccount.createEmulatorAccountFromMnemonic( + self._id, + mnemonic=self.__withdraw_mnemonic, + balance=0, + index=index, + password=password, + ) + self.__withdraw_address_override = "" + return self + + def install(self, node: Node, eth: EthereumService): + """! + @brief Install the vc server on the node. + + """ + # if self.__is_beacon_setup_node: + # beacon_setup_node = PoSBeaconSetupServer() + # beacon_setup_node.install(self, node, self._blockchain) + # return + + if self.__is_beacon_validator_at_genesis: + self._role.append("validator_at_genesis") + if self.__is_beacon_validator_at_running: + self._role.append("validator_at_running") + + super().install(node,eth) + self.__install_vc(node, eth) + def __install_vc(self, node:Node, eth:EthereumService): + ifaces = node.getInterfaces() + assert len(ifaces) > 0, 'EthereumServer::install: node as{}/{} has no interfaces'.format(node.getAsn(), node.getName()) + addr = str(ifaces[0].getAddress()) + # Get the IP address of beacon_setup_node. + beacon_setup_node = self._blockchain.getBeaconSetupNodeIp() + + assert beacon_setup_node != "", 'EthereumServer::install: Ethereum Service has no beacon_setup_node.' + + beacon_node_ip = self.getConnectedBeaconIp() + assert beacon_node_ip != "", 'EthereumServer::install: Ethereum Service has no beacon_node.' + # bootnode_start_command = "" + # bc_start_command = "" + # bc_start_command = LIGHTHOUSE_BN_CMD.format(ip_address=addr, target_peers=self.__beacon_peer_counts, bootnodes_flag="" if self._is_bootnode else f'--boot-nodes "$(cat /tmp/bc_enrs.txt)"') + vc_start_command = "" + wallet_create_command = "" + validator_create_command = "" + validator_deposit_sh = "" + # ## div beacon node and geth node + # if not self._is_bootnode: + # node.setFile('/tmp/fetch_bn_enr', EthServerFileTemplates['fetch_bn_enr']) + # node.appendStartCommand('chmod +x /tmp/fetch_bn_enr') + # node.appendStartCommand('/tmp/fetch_bn_enr') + + # if self._is_bootnode: + # bootnode_start_command = LIGHTHOUSE_BOOTNODE_CMD.format(ip_address=addr) + if self.__is_beacon_validator_at_running: + node.setFile('/tmp/seed.pass', 'seedseedseed') + # wallet_create_command = LIGHTHOUSE_WALLET_CREATE_CMD.format(eth_id=self.getId()) + # validator_create_command = LIGHTHOUSE_VC_VALIDATOR_CREATE_CMD.format(withdraw_address=self.getWithdrawAddress()) + + # if len(self._accounts) > 0: + # print(self._accounts[0].keystore_filename) + # node.setFile('/tmp/deposit.py', VALIDATOR_DEPOSIT_PY.format(keystore_filename=self._accounts[0].keystore_filename)) + + # if not self.__is_manual_deposit_for_validator: + # validator_deposit_sh = "sleep 2 && python3 /tmp/deposit.py" + + # node.appendStartCommand('chmod +x /tmp/deposit.sh') + # if not self.__is_manual_deposit_for_validator: + # validator_deposit_sh = "/tmp/deposit.sh" + if self.__is_beacon_validator_at_genesis: + vc_start_command = LIGHTHOUSE_VC_CMD.format(ip_address=addr, acct_address=self._accounts[0].address, beacon_node=beacon_node_ip) + # Write the IP and port of beacon_setup_node to the file. + node.setFile('/tmp/beacon-setup-node', beacon_setup_node) + node.setFile('/tmp/withdraw-address', self.getWithdrawAddress()) + + # get current node validator index if it's validator at genesis. + validatorIds = self._blockchain.getValidatorIds() + validator_idx = -1 + if self.isValidatorAtGenesis(): + # only get validator id if current node is validator at genesis. + for i, v in enumerate(validatorIds): + if int(v) == self._id: + validator_idx = i + break + assert validator_idx >= 0, "EthereumServer::__install_beacon: validator id should be set at genesis." + validator_idx = int(validator_idx) + # this validator idx will be used to create validator keys properly. + # Each validator at genesis node has 4 keys which were set at Genesis + + node.setFile('/tmp/vc_bootstrapper', EthServerFileTemplates['vc_bootstrapper'].format( + is_validator_at_genesis="true" if self.__is_beacon_validator_at_genesis else "false", + is_validator_at_running="true" if self.__is_beacon_validator_at_running else "false", + validator_mnemonic=self.__validator_mnemonic, + validator_key_start= validator_idx , + validator_key_end= (validator_idx + 1), + vc_start_command=vc_start_command, + wallet_create_command=wallet_create_command, + validator_create_command=validator_create_command, + validator_deposit_sh=validator_deposit_sh, + ip_address=beacon_node_ip + )) + node.setFile('/tmp/jwt.hex', '0xae7177335e3d4222160e08cecac0ace2cecce3dc3910baada14e26b11d2009fc') + + node.appendStartCommand('chmod +x /tmp/vc_bootstrapper') + node.appendStartCommand('/tmp/vc_bootstrapper') + + + def connectToBeaconNode(self, vnode:str): + self.__conect_beacon_vnode = vnode + return self + def setConnectedBeaconIp(self, ip:str): + self.__connected_beacon_ip = ip + return self + def getConnectedBeaconIp(self) -> str: + return self.__connected_beacon_ip + def getConnectBeaconVNode(self) -> str: + return self.__conect_beacon_vnode + def enablePOSValidatorAtGenesis(self): + self.__is_beacon_validator_at_genesis = True + return self + def enablePOSValidatorAtRunning(self, is_manual:bool=False): + self.__is_beacon_validator_at_running = True + self.__is_manual_deposit_for_validator = is_manual + return self + def isValidatorAtRunning(self): + return self.__is_beacon_validator_at_running + def isValidatorAtGenesis(self): + return self.__is_beacon_validator_at_genesis diff --git a/seedemu/services/EthereumService/EthereumService.py b/seedemu/services/EthereumService/EthereumService.py index 63cbc4421..880fe3247 100644 --- a/seedemu/services/EthereumService/EthereumService.py +++ b/seedemu/services/EthereumService/EthereumService.py @@ -2,7 +2,7 @@ from .EthEnum import ConsensusMechanism, EthUnit from .EthUtil import Genesis, EthAccount, AccountStructure -from .EthereumServer import EthereumServer, PoAServer, PoWServer, PoSServer +from .EthereumServer import EthereumServer, PoAServer, PoSBeaconServer, PoSBeaconSetupServer, PoSGethServer, PoWServer, PoSVcServer,PoSServer from os import mkdir, path, makedirs, rename from seedemu.core import Node, Service, Server, Emulator from seedemu.core.enums import NetworkType @@ -57,17 +57,26 @@ def __init__(self, service:EthereumService, chainName: str, chainId: int, consen self._joined_accounts = [] self._joined_signer_accounts = [] self._validator_ids = [] + self._validator_withdraw_addresses = {} self._beacon_setup_node_address = '' + + self._beacon_boot_node_address = [] + self._geth_boot_node_address = [] + self._pending_targets = [] # self._emu_mnemonic = "great awesome fun seed security lab protect system network prevent attack future" self._total_accounts_per_node = 1 - self._emu_account_balance = 32 * EthUnit.ETHER.value + self._emu_account_balance = 100 * EthUnit.ETHER.value self._local_mnemonic = "great amazing fun seed lab protect network system security prevent attack future" self._local_accounts_total = 5 self._local_account_balance = 10 * EthUnit.ETHER.value self._chain_id = chainId self._target_aggregater_per_committee = 2 self._target_committee_size = 3 + self._beacon_to_geth_vnode = {} + self._vc_to_beacon_vnode = {} + self._beacon_to_geth_ip = {} + self._vc_to_beacon_ip = {} def _doConfigure(self, node:Node, server:Server): @@ -86,31 +95,46 @@ def _doConfigure(self, node:Node, server:Server): assert len(ifaces) > 0, 'EthereumService::_doConfigure(): node as{}/{} has not interfaces'.format() addr = '{}:{}'.format(str(ifaces[0].getAddress()), server.getBootNodeHttpPort()) + + if server.isBootNode(): - self._log('adding as{}/{} as consensus-{} bootnode...'.format(node.getAsn(), node.getName(), self._consensus.value)) - self._boot_node_addresses.append(str(ifaces[0].getAddress())) + if isinstance(server, PoSBeaconServer): + self._beacon_boot_node_address.append(str(ifaces[0].getAddress())) + elif isinstance(server, PoSGethServer): + self._geth_boot_node_address.append(str(ifaces[0].getAddress())) + elif isinstance(server,PoAServer): + self._log('adding as{}/{} as consensus-{} bootnode...'.format(node.getAsn(), node.getName(), self._consensus.value)) + self._boot_node_addresses.append(str(ifaces[0].getAddress())) + if self._consensus == ConsensusMechanism.POS: - if server.isStartMiner(): - self._log('adding as{}/{} as consensus-{} miner...'.format(node.getAsn(), node.getName(), self._consensus.value)) - self._miner_node_address.append(str(ifaces[0].getAddress())) - if server.isBeaconSetupNode(): + if isinstance(server, PoSBeaconSetupServer): self._beacon_setup_node_address = '{}:{}'.format(ifaces[0].getAddress(), server.getBeaconSetupHttpPort()) + # if server.isStartMiner(): + # self._log('adding as{}/{} as consensus-{} miner...'.format(node.getAsn(), node.getName(), self._consensus.value)) + # self._miner_node_address.append(str(ifaces[0].getAddress())) + # if server.isBeaconSetupNode(): + # self._beacon_setup_node_address = '{}:{}'.format(ifaces[0].getAddress(), server.getBeaconSetupHttpPort()) server._createAccounts(self) accounts = server._getAccounts() if len(accounts) > 0: - if self._consensus == ConsensusMechanism.POS and server.isValidatorAtRunning(): - accounts[0].balance = 33 * EthUnit.ETHER.value + if isinstance(server, PoSVcServer): + if self._consensus == ConsensusMechanism.POS and server.isValidatorAtRunning(): + accounts[0].balance = 100 * EthUnit.ETHER.value self._joined_accounts.extend(accounts) if self._consensus in [ConsensusMechanism.POA] and server.isStartMiner(): self._joined_signer_accounts.append(accounts[0]) + if isinstance(server, PoSVcServer): + if self._consensus == ConsensusMechanism.POS and server.isValidatorAtGenesis(): + vid = str(server.getId()) + self._validator_ids.append(vid) + self._validator_withdraw_addresses[vid] = server.getWithdrawAddress() - if self._consensus == ConsensusMechanism.POS and server.isValidatorAtGenesis(): - self._validator_ids.append(str(server.getId())) - + server._generateGethStartCommand(str(ifaces[0].getAddress())) + if self._eth_service.isSave(): save_path = self._eth_service.getSavePath() @@ -148,7 +172,20 @@ def configure(self, emulator:Emulator): server.setFaucetUrl(self.__getIpByVnodeName(emulator, linked_faucet_node_name)) faucet_server:FaucetServer = emulator.getServerByVirtualNodeName(linked_faucet_node_name) server.setFaucetPort(faucet_server.getPort()) - + + elif isinstance(server, PoSBeaconServer): + geth_vnode = server.getConnectGethVNode() + if geth_vnode != '': + ip = self.__getIpByVnodeName(emulator, geth_vnode) + if ip: + server.setConnectedGethIp(ip) + elif isinstance(server, PoSVcServer): + beacon_vnode = server.getConnectBeaconVNode() + if beacon_vnode != '': + ip = self.__getIpByVnodeName(emulator, beacon_vnode) + if ip: + server.setConnectedBeaconIp(ip) + self._genesis.addAccounts(self.getAllAccounts()) if self._consensus in [ConsensusMechanism.POA] : @@ -164,7 +201,8 @@ def __getIpByVnodeName(self, emulator, nodename:str) -> str: if net.getType() == NetworkType.Local: address = iface.getAddress() return address - + def getIpByVnodeName(self, emulator, nodename:str) -> str: + return self.__getIpByVnodeName(emulator, nodename) def getAllServerNames(self): server_names = {} pending_targets = self._eth_service.getPendingTargets() @@ -216,6 +254,9 @@ def getValidatorIds(self) -> List[str]: """ return self._validator_ids + def getValidatorWithdrawAddress(self, validator_id: str) -> str: + return self._validator_withdraw_addresses.get(str(validator_id), "") + def getBeaconSetupNodeIp(self) -> str: """! @brief Get the IP of a beacon setup node. @@ -223,6 +264,24 @@ def getBeaconSetupNodeIp(self) -> str: @returns The IP address. """ return self._beacon_setup_node_address + def getBeaconBootNodeIP(self) -> str: + """! + @brief Get the IP of a beacon boot node. + + @returns The IP address. + """ + return self._beacon_boot_node_address + def getGethBootNodeIP(self) -> str: + """! + @brief Get the IP of a geth boot node. + + @returns The IP address. + """ + return self._geth_boot_node_address + def getBeaconBootNodes(self) -> List[str]: + return list(self._beacon_boot_node_address) + def getGethBootNodes(self) -> List[str]: + return list(self._geth_boot_node_address) def setGenesis(self, genesis:str) -> EthereumServer: """! @@ -300,6 +359,56 @@ def createNode(self, vnode: str) -> EthereumServer: eth = self._eth_service self._pending_targets.append(vnode) return eth.installByBlockchain(vnode, self) + def createGethNode(self, vnode: str) -> EthereumServer: + """! + @brief Create a node belongs to this blockchain. + + @param vnode The name of vnode. + + @returns EthereumServer + """ + nodetype = 'geth' + eth = self._eth_service + self._pending_targets.append(vnode) + return eth.installByBlockchain(vnode, self,nodetype="geth") + def createBeaconNode(self, vnode: str) -> EthereumServer: + """! + @brief Create a node belongs to this blockchain. + + @param vnode The name of vnode. + + @returns EthereumServer + """ + nodetype = 'beacon' + eth = self._eth_service + self._pending_targets.append(vnode) + return eth.installByBlockchain(vnode, self,nodetype="beacon") + + def createVcNode(self, vnode: str) -> EthereumServer: + """! + @brief Create a node belongs to this blockchain. + + @param vnode The name of vnode. + + @returns EthereumServer + """ + nodetype = 'beacon' + eth = self._eth_service + self._pending_targets.append(vnode) + return eth.installByBlockchain(vnode, self,nodetype="vc") + + def createBeaconSetupNode(self, vnode: str) -> EthereumServer: + """! + @brief Create a node belongs to this blockchain. + + @param vnode The name of vnode. + + @returns EthereumServer + """ + nodetype = 'beaconsetup' + eth = self._eth_service + self._pending_targets.append(vnode) + return eth.installByBlockchain(vnode, self, nodetype="beaconsetup") def addCode(self, address: str, code: str) -> Blockchain: """! @@ -636,7 +745,7 @@ def _createSharedFolder(self): exit(1) mkdir(self.__save_path) - def _doInstall(self, node: Node, server: Server): + def _doInstall(self, node: Node, server: Server): ##这里的server是基类 service的render 传进来的 self._log('installing eth on as{}/{}...'.format(node.getAsn(), node.getName())) if isinstance(server, EthereumServer): server.install(node, self) @@ -645,7 +754,7 @@ def _doInstall(self, node: Node, server: Server): elif isinstance(server, EthUtilityServer): server.install(node) - def _createServer(self, blockchain: Blockchain = None) -> Server: + def _createServer(self, blockchain: Blockchain = None,nodetype:str =None) -> Server: self.__serial += 1 assert blockchain != None, 'EthereumService::_createServer(): create server using Blockchain::createNode() not EthereumService::install()'.format() consensus = blockchain.getConsensusMechanism() @@ -654,7 +763,16 @@ def _createServer(self, blockchain: Blockchain = None) -> Server: if consensus == ConsensusMechanism.POW: return PoWServer(self.__serial, blockchain) if consensus == ConsensusMechanism.POS: - return PoSServer(self.__serial, blockchain) + if nodetype == 'geth': + return PoSGethServer(self.__serial, blockchain) + if nodetype == "beacon": + return PoSBeaconServer(self.__serial, blockchain) + if nodetype == "beaconsetup": + return PoSBeaconSetupServer(self.__serial, blockchain) + if nodetype == "vc": + return PoSVcServer(self.__serial, blockchain) + if nodetype == None: + return PoSServer(self.__serial, blockchain) def _createFaucetServer(self, blockchain:Blockchain, linked_eth_node:str, port:int, balance:int, max_fund_amount:int) -> FaucetServer: return FaucetServer(blockchain, linked_eth_node, port, balance, max_fund_amount) @@ -662,7 +780,7 @@ def _createFaucetServer(self, blockchain:Blockchain, linked_eth_node:str, port:i def _createEthUtilityServer(self, blockchain:Blockchain, port:int, linked_eth_node:str, linked_faucet_node:str) -> EthUtilityServer: return EthUtilityServer(blockchain, port, linked_eth_node, linked_faucet_node) - def installByBlockchain(self, vnode: str, blockchain: Blockchain) -> EthereumServer: + def installByBlockchain(self, vnode: str, blockchain: Blockchain,nodetype:str = None) -> EthereumServer: """! @brief Install the service on a node identified by given name. This API is called by Blockchain Class. @@ -674,7 +792,8 @@ def installByBlockchain(self, vnode: str, blockchain: Blockchain) -> EthereumSer """ if vnode in self._pending_targets.keys(): return self._pending_targets[vnode] - s = self._createServer(blockchain) + + s = self._createServer(blockchain,nodetype) self._pending_targets[vnode] = s return self._pending_targets[vnode] diff --git a/seedemu/services/EthereumService/FaucetServer.py b/seedemu/services/EthereumService/FaucetServer.py index 7531468a8..9c4c5efb5 100644 --- a/seedemu/services/EthereumService/FaucetServer.py +++ b/seedemu/services/EthereumService/FaucetServer.py @@ -148,7 +148,7 @@ def install(self, node: Node): node.addSoftware('python3 python3-pip') self._installScriptFiles(node) - node.addBuildCommand('pip3 install flask web3==5.31.1') + node.addBuildCommand('pip3 install --break-system-packages flask web3==5.31.1 || pip3 install flask web3==5.31.1') # Start the faucet server node.appendStartCommand('python3 {}/app.py &'.format(self.DIR_PREFIX)) diff --git a/seedemu/services/EthereumService/FaucetUserService.py b/seedemu/services/EthereumService/FaucetUserService.py index 4d161342d..358ee918c 100644 --- a/seedemu/services/EthereumService/FaucetUserService.py +++ b/seedemu/services/EthereumService/FaucetUserService.py @@ -52,13 +52,13 @@ def install(self, node: Node): """ node.appendClassName("FaucetUserService") node.addSoftware('python3 python3-pip') - node.addBuildCommand('pip3 install eth_account==0.5.9 requests') + node.addBuildCommand('pip3 install --break-system-packages eth_account==0.5.9 requests || pip3 install eth_account==0.5.9 requests') node.setFile(self.DIR_PREFIX + '/fundme.py', FaucetServerFileTemplates['fundme'].format( faucet_url=self.__faucet_util.getFacuetUrl(), faucet_fund_url=self.__faucet_util.getFaucetFundUrl())) - node.appendStartCommand('chmod +x {}/fund.py'.format(self.DIR_PREFIX)) - node.appendStartCommand(self.DIR_PREFIX + '/fund.py') + node.appendStartCommand('chmod +x {}/fundme.py'.format(self.DIR_PREFIX)) + node.appendStartCommand(self.DIR_PREFIX + '/fundme.py') def print(self, indent: int) -> str: out = ' ' * indent diff --git a/seedemu/services/ExaBgpService.py b/seedemu/services/ExaBgpService.py new file mode 100644 index 000000000..d8ab829dc --- /dev/null +++ b/seedemu/services/ExaBgpService.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +from ipaddress import ip_network +from typing import Dict, List, Optional, Tuple + +from seedemu.core import Emulator, Network, Node, Router, ScopedRegistry, Server, Service +from seedemu.layers._bgp_metadata import ( + BGP_COMMUNITY_CUSTOMER, + BGP_COMMUNITY_PEER, + BGP_COMMUNITY_PROVIDER, + install_router_bgp_session, +) + + +ExaBgpFileTemplates: Dict[str, str] = {} + +ExaBgpFileTemplates["config"] = """\ +process manual-control {{ + run /bin/sh -c "mkdir -p /run/exabgp; mkfifo /run/exabgp/manual.in 2>/dev/null || true; chmod 666 /run/exabgp/manual.in; tail -f /run/exabgp/manual.in"; + encoder text; +}} + +{neighbor_blocks} +""" + +ExaBgpFileTemplates["static_block"] = """\ + static {{ +{routes} + }} +""" + + +class ExaBgpServer(Server): + """! + @brief ExaBGP external BGP speaker installed through Service + Binding. + """ + + __emulator: Optional[Emulator] + __local_asn: int + __announce_prefixes: List[str] + __peers: List[Dict[str, object]] + __resolved_peers: List[Tuple[Dict[str, object], Router, str, str]] + + def __init__(self): + super().__init__() + self.__emulator = None + self.__local_asn = 65010 + self.__announce_prefixes = [] + self.__peers = [] + self.__resolved_peers = [] + self.setDisplayName("ExaBGP Control Plane Speaker") + + def bind(self, emulator: Emulator): + self.__emulator = emulator + + def setLocalAsn(self, asn: int) -> "ExaBgpServer": + self.__local_asn = int(asn) + return self + + def addAnnouncement(self, prefix: str) -> "ExaBgpServer": + network = ip_network(prefix, strict=False) + assert network.version == 4, "ExaBgpService foundation currently supports IPv4 announcements only" + self.__announce_prefixes.append(str(network)) + return self + + def attachToRouter(self, router_name: str, router_asn: Optional[int] = None) -> "ExaBgpServer": + self.__peers = [] + self.addPeerByRouter(router_name, router_asn=router_asn) + return self + + def addPeer( + self, + ix: int, + peer_asn: Optional[int] = None, + session_name: Optional[str] = None, + router_relationship: str = "customer", + router_asn: Optional[int] = None, + ) -> "ExaBgpServer": + if isinstance(ix, str): + return self.addPeerByRouter( + ix, + router_asn=router_asn if router_asn is not None else peer_asn, + session_name=session_name, + router_relationship=router_relationship, + ) + + assert peer_asn is not None, "peer_asn is required for IX-based ExaBGP peering" + relationship = self._normalize_relationship(router_relationship) + self.__peers.append( + { + "type": "ix", + "ix": int(ix), + "peer_asn": int(peer_asn), + "session_name": str(session_name).strip() if session_name is not None else None, + "router_relationship": relationship, + } + ) + return self + + def addPeerByRouter( + self, + router_name: str, + router_asn: Optional[int] = None, + session_name: Optional[str] = None, + router_relationship: str = "customer", + ) -> "ExaBgpServer": + relationship = self._normalize_relationship(router_relationship) + self.__peers.append( + { + "type": "router", + "router_name": str(router_name), + "router_asn": int(router_asn) if router_asn is not None else None, + "session_name": str(session_name).strip() if session_name is not None else None, + "router_relationship": relationship, + } + ) + return self + + def _normalize_relationship(self, router_relationship: str) -> str: + relationship = str(router_relationship or "customer").strip().lower() or "customer" + if relationship not in {"customer", "peer", "provider", "unfiltered"}: + raise ValueError("unsupported router relationship: {}".format(router_relationship)) + return relationship + + def _relationship_params(self, relationship: str) -> Tuple[Optional[str], Optional[int], str]: + if relationship == "customer": + return BGP_COMMUNITY_CUSTOMER, 30, "all" + if relationship == "peer": + return BGP_COMMUNITY_PEER, 20, "local_and_customer" + if relationship == "provider": + return BGP_COMMUNITY_PROVIDER, 10, "local_and_customer" + return None, None, "all" + + def _resolve_ix_net(self, ix: int) -> Network: + assert self.__emulator is not None, "ExaBgpServer not bound to emulator" + ix_scope = ScopedRegistry("ix", self.__emulator.getRegistry()) + ix_name = "ix{}".format(ix) + assert ix_scope.has("net", ix_name), "IX{} not found for ExaBGP peer".format(ix) + return ix_scope.get("net", ix_name) + + def _resolve_ix_peer(self, node: Node, peer: Dict[str, object]) -> Tuple[Router, str, str]: + assert self.__emulator is not None, "ExaBgpServer not bound to emulator" + ix = int(peer["ix"]) + peer_asn = int(peer["peer_asn"]) + ix_net = self._resolve_ix_net(ix) + + local_address: Optional[str] = None + for iface in node.getInterfaces(): + if iface.getNet() == ix_net: + local_address = str(iface.getAddress()) + break + + assert local_address is not None, ( + "ExaBGP node as{}/{} is not connected to IX{}".format(node.getAsn(), node.getName(), ix) + ) + + scope = ScopedRegistry(str(peer_asn), self.__emulator.getRegistry()) + candidates: List[Tuple[Router, str]] = [] + for router in scope.getByType("rnode"): + assert isinstance(router, Router) + for iface in router.getInterfaces(): + if iface.getNet() == ix_net: + candidates.append((router, str(iface.getAddress()))) + break + + assert candidates, "cannot resolve ExaBGP peer: AS{} is not connected to IX{}".format(peer_asn, ix) + assert len(candidates) == 1, ( + "cannot resolve ExaBGP peer: AS{} has multiple routers on IX{}: {}. " + "Use addPeerByRouter(...) to choose one explicitly." + ).format(peer_asn, ix, ", ".join(router.getName() for router, _addr in candidates)) + + router, peer_address = candidates[0] + return router, local_address, peer_address + + def _resolve_router_peer(self, node: Node, peer: Dict[str, object]) -> Tuple[Router, str, str]: + assert self.__emulator is not None, "ExaBgpServer not bound to emulator" + router_asn = int(peer["router_asn"]) if peer.get("router_asn") is not None else node.getAsn() + scope = ScopedRegistry(str(router_asn), self.__emulator.getRegistry()) + router_name = str(peer["router_name"]) + assert scope.has("rnode", router_name), "router as{}/{} not found for ExaBGP peer".format(router_asn, router_name) + router = scope.get("rnode", router_name) + assert isinstance(router, Router) + + for node_iface in node.getInterfaces(): + for router_iface in router.getInterfaces(): + if node_iface.getNet() != router_iface.getNet(): + continue + return router, str(node_iface.getAddress()), str(router_iface.getAddress()) + + raise AssertionError( + "ExaBGP node as{}/{} does not share a network with as{}/{}".format( + node.getAsn(), node.getName(), router.getAsn(), router.getName() + ) + ) + + def _resolve_peer(self, node: Node, peer: Dict[str, object]) -> Tuple[Router, str, str]: + if peer.get("type") == "ix": + return self._resolve_ix_peer(node, peer) + return self._resolve_router_peer(node, peer) + + def configureOnNode(self, node: Node): + assert self.__peers, "ExaBgpServer requires at least one peer" + if self.__resolved_peers: + return + + for index, peer in enumerate(self.__peers): + router, local_address, peer_address = self._resolve_peer(node, peer) + session_name = str(peer.get("session_name") or "") + if not session_name: + session_name = "exabgp_{}".format(self.__local_asn) + if len(self.__peers) > 1: + session_name = "{}_{}_{}".format(session_name, router.getName(), index) + + import_community, local_pref, export_policy = self._relationship_params( + str(peer.get("router_relationship") or "customer") + ) + install_router_bgp_session( + router, + { + "name": session_name, + "kind": "ebgp", + "local_address": peer_address, + "local_asn": router.getAsn(), + "peer_address": local_address, + "peer_asn": self.__local_asn, + "import_community": import_community, + "local_pref": local_pref, + "export_policy": export_policy, + "next_hop_self": True, + "route_server_client": False, + }, + ) + self.__resolved_peers.append((peer, router, local_address, peer_address)) + + def install(self, node: Node): + self.configureOnNode(node) + node.addSoftware("exabgp") + + routes = "\n".join(" route {} next-hop self;".format(prefix) for prefix in self.__announce_prefixes) + static_block = ExaBgpFileTemplates["static_block"].format(routes=routes) if routes else "" + neighbor_blocks: List[str] = [] + for _peer, router, local_address, peer_address in self.__resolved_peers: + neighbor_blocks.append( + ( + "neighbor {peer_address} {{\n" + " router-id {router_id};\n" + " local-address {local_address};\n" + " local-as {local_asn};\n" + " peer-as {peer_asn};\n" + " family {{\n" + " ipv4 unicast;\n" + " }}\n" + " api {{\n" + " processes [ manual-control ];\n" + " }}\n" + "{static_block}" + "}}\n" + ).format( + peer_address=peer_address, + router_id=local_address, + local_address=local_address, + local_asn=self.__local_asn, + peer_asn=router.getAsn(), + static_block=static_block, + ) + ) + + node.setFile( + "/etc/exabgp/exabgp.conf", + ExaBgpFileTemplates["config"].format(neighbor_blocks="\n".join(neighbor_blocks)), + ) + node.appendStartCommand("mkdir -p /var/log/exabgp") + node.appendStartCommand( + "env exabgp.daemon.drop=false exabgp.daemon.user=root " + "exabgp /etc/exabgp/exabgp.conf >/var/log/exabgp/exabgp.log 2>&1", + True, + ) + + +class ExaBgpService(Service): + """! + @brief ExaBGP service layer. + """ + + __emulator: Optional[Emulator] + + def __init__(self): + super().__init__() + self.__emulator = None + self.addDependency("Routing", False, False) + self.addDependency("Ebgp", False, True) + + def _createServer(self) -> Server: + return ExaBgpServer() + + def _doConfigure(self, node: Node, server: ExaBgpServer): + assert self.__emulator is not None + server.bind(self.__emulator) + server.configureOnNode(node) + super()._doConfigure(node, server) + + def configure(self, emulator: Emulator): + self.__emulator = emulator + return super().configure(emulator) + + def getName(self) -> str: + return "ExaBgpService" + + def print(self, indent: int) -> str: + out = ' ' * indent + out += 'ExaBgpServiceLayer\n' + return out diff --git a/seedemu/services/SolanaService/SolanaEnum.py b/seedemu/services/SolanaService/SolanaEnum.py new file mode 100644 index 000000000..909d90572 --- /dev/null +++ b/seedemu/services/SolanaService/SolanaEnum.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from enum import Enum + + +class SolanaNodeRole(Enum): + """! + @brief Role of a Solana node within a private cluster. + + A private Agave/Solana cluster is bootstrapped by exactly one *bootstrap* + (genesis) validator. That node creates the genesis ledger and runs the + faucet. Every other validator joins the cluster over gossip using the + bootstrap node as its entrypoint. + """ + + ## The genesis / bootstrap validator. Creates the genesis ledger, stakes + # itself, runs the faucet, and is the gossip entrypoint for the cluster. + BOOTSTRAP = "bootstrap" + + ## A regular validator that joins an existing cluster via the bootstrap's + # gossip entrypoint, funds itself from the faucet, and votes. + VALIDATOR = "validator" + + +class SolanaNetworkType(Enum): + """! + @brief Cluster type passed to ``solana-genesis --cluster-type``. + + For emulation we use ``development``, which is the self-contained local + cluster type (no connection to Solana's public devnet/testnet/mainnet). + """ + + DEVELOPMENT = "development" diff --git a/seedemu/services/SolanaService/SolanaServer.py b/seedemu/services/SolanaService/SolanaServer.py new file mode 100644 index 000000000..9528fd64c --- /dev/null +++ b/seedemu/services/SolanaService/SolanaServer.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +from typing import Optional + +from seedemu.core.Node import Node +from seedemu.core.Service import Server + +from .SolanaEnum import SolanaNodeRole + + +# Default ports for a Solana node. RPC/pubsub/gossip/faucet match the upstream +# defaults; the dynamic range carries TPU / turbine / repair traffic. +DEFAULT_RPC_PORT = 8899 +DEFAULT_GOSSIP_PORT = 8001 +DEFAULT_FAUCET_PORT = 9900 +DEFAULT_DYNAMIC_PORT_LOW = 8002 +DEFAULT_DYNAMIC_PORT_HIGH = 8030 + + +class SolanaServer(Server): + """! + @brief Base class shared by all Solana node servers. + + A ``SolanaServer`` renders, at install time, a self-contained start script + that drives the real Agave (``agave-validator``) toolchain. Connectivity + information (this node's own IP, and the bootstrap node's gossip/RPC + endpoint) is injected by :class:`SolanaNetwork` during the *configure* + phase, where it is resolved from the virtual-node bindings. Nothing about + addresses is hard-coded here (PRINCIPLES.md P4). + """ + + def __init__(self, network: "SolanaNetwork", role: SolanaNodeRole): + super().__init__() + self._network = network + self._role = role + + # Ports (sensible defaults; overridable via the fluent API). + self._rpc_port = DEFAULT_RPC_PORT + self._gossip_port = DEFAULT_GOSSIP_PORT + self._faucet_port = DEFAULT_FAUCET_PORT + self._dyn_low = DEFAULT_DYNAMIC_PORT_LOW + self._dyn_high = DEFAULT_DYNAMIC_PORT_HIGH + + # Runtime binding information (populated during configure()). + self._self_ip: Optional[str] = None + self._bootstrap_ip: Optional[str] = None + self._bootstrap_gossip_port: Optional[int] = None + self._bootstrap_rpc_port: Optional[int] = None + + # ------------------------------------------------------------------ # + # Introspection + # ------------------------------------------------------------------ # + @property + def role(self) -> SolanaNodeRole: + return self._role + + def is_bootstrap(self) -> bool: + return self._role == SolanaNodeRole.BOOTSTRAP + + def get_gossip_port(self) -> int: + return self._gossip_port + + def get_rpc_port(self) -> int: + return self._rpc_port + + # ------------------------------------------------------------------ # + # Fluent configuration API (PRINCIPLES.md P3) + # ------------------------------------------------------------------ # + def setRpcPort(self, port: int) -> "SolanaServer": + """!@brief Set the JSON-RPC port (default 8899).""" + self._rpc_port = port + return self + + def setGossipPort(self, port: int) -> "SolanaServer": + """!@brief Set the gossip port (default 8001).""" + self._gossip_port = port + return self + + def setDynamicPortRange(self, low: int, high: int) -> "SolanaServer": + """!@brief Set the dynamic (TPU/turbine/repair) port range.""" + self._dyn_low = low + self._dyn_high = high + return self + + # ------------------------------------------------------------------ # + # Runtime-information setters (called by SolanaNetwork.configure) + # ------------------------------------------------------------------ # + def set_self_ip(self, ip: str): + self._self_ip = ip + + def set_bootstrap_endpoint(self, ip: str, gossip_port: int, rpc_port: int): + self._bootstrap_ip = ip + self._bootstrap_gossip_port = gossip_port + self._bootstrap_rpc_port = rpc_port + + # ------------------------------------------------------------------ # + # Installation + # ------------------------------------------------------------------ # + def install(self, node: Node): # pragma: no cover - executed by runtime + raise NotImplementedError("install must be implemented by a subclass") + + +class SolanaBootstrapServer(SolanaServer): + """! + @brief The genesis / bootstrap validator of a private Solana cluster. + + Creates all genesis keypairs, builds the genesis ledger with + ``solana-genesis``, runs ``solana-faucet`` so other validators can fund + themselves, and starts ``agave-validator`` as the cluster's first staked + validator and gossip entrypoint. + """ + + def __init__(self, network: "SolanaNetwork"): + super().__init__(network, SolanaNodeRole.BOOTSTRAP) + self._faucet_lamports = 500000000000000000 + self._hashes_per_tick = "sleep" # low-CPU PoH, suited to emulation (P9) + + def setFaucetLamports(self, lamports: int) -> "SolanaBootstrapServer": + """!@brief Set the amount of lamports held by the genesis faucet.""" + self._faucet_lamports = lamports + return self + + def setHashesPerTick(self, value: str) -> "SolanaBootstrapServer": + """!@brief Set solana-genesis --hashes-per-tick ("sleep", "auto", or a number).""" + self._hashes_per_tick = value + return self + + def install(self, node: Node): # pragma: no cover - executed by runtime + assert self._self_ip is not None, \ + "SolanaBootstrapServer not configured; did SolanaNetwork.configure() run?" + + script = _BOOTSTRAP_SCRIPT.format( + self_ip=self._self_ip, + rpc_port=self._rpc_port, + gossip_port=self._gossip_port, + faucet_port=self._faucet_port, + dyn_low=self._dyn_low, + dyn_high=self._dyn_high, + faucet_lamports=self._faucet_lamports, + hashes_per_tick=self._hashes_per_tick, + ) + node.setFile("/opt/solana/run-bootstrap.sh", script) + node.addBuildCommandAtEnd("chmod +x /opt/solana/run-bootstrap.sh") + node.appendStartCommand("/opt/solana/run-bootstrap.sh", fork=True) + + +class SolanaValidatorServer(SolanaServer): + """! + @brief A validator that joins an existing private cluster. + + Waits for the bootstrap RPC, generates its own identity / vote keypairs, + funds its identity from the faucet (an airdrop relayed through the + bootstrap RPC), creates its vote account, and then starts + ``agave-validator`` with the bootstrap node as its gossip entrypoint. + """ + + def __init__(self, network: "SolanaNetwork"): + super().__init__(network, SolanaNodeRole.VALIDATOR) + self._node_sol = 500 # SOL airdropped for fees + vote-account rent + + def setNodeSol(self, sol: int) -> "SolanaValidatorServer": + """!@brief Set the amount of SOL this validator airdrops to itself.""" + self._node_sol = sol + return self + + def install(self, node: Node): # pragma: no cover - executed by runtime + assert self._self_ip is not None, \ + "SolanaValidatorServer not configured; did SolanaNetwork.configure() run?" + assert self._bootstrap_ip is not None, \ + "SolanaValidatorServer has no bootstrap endpoint; the network needs a bootstrap validator." + + script = _VALIDATOR_SCRIPT.format( + self_ip=self._self_ip, + rpc_port=self._rpc_port, + gossip_port=self._gossip_port, + dyn_low=self._dyn_low, + dyn_high=self._dyn_high, + entrypoint_ip=self._bootstrap_ip, + boot_gossip_port=self._bootstrap_gossip_port, + boot_rpc_port=self._bootstrap_rpc_port, + node_sol=self._node_sol, + ) + node.setFile("/opt/solana/run-validator.sh", script) + node.addBuildCommandAtEnd("chmod +x /opt/solana/run-validator.sh") + node.appendStartCommand("/opt/solana/run-validator.sh", fork=True) + + +# --------------------------------------------------------------------------- # +# Start-script templates. +# +# These reproduce the exact command sequence used by Agave's own +# multinode-demo scripts (setup.sh / bootstrap-validator.sh / validator.sh), +# adapted for a containerised, fully-private cluster: +# * --allow-private-addr : permit RFC1918 (10.x) gossip in the emulator +# * --no-poh-speed-test / --no-os-network-limits-test : skip host benchmarks +# * --hashes-per-tick sleep : low-CPU PoH for emulation +# --------------------------------------------------------------------------- # + +_BOOTSTRAP_SCRIPT = r"""#!/bin/bash +# seedemu-solana: genesis / bootstrap validator +set -euo pipefail + +CONFIG_DIR=/opt/solana/config +LEDGER_DIR="$CONFIG_DIR/bootstrap-validator" +SELF_IP={self_ip} +RPC_PORT={rpc_port} +GOSSIP_PORT={gossip_port} +FAUCET_PORT={faucet_port} +DYNAMIC_PORT_RANGE={dyn_low}-{dyn_high} + +mkdir -p "$LEDGER_DIR" + +# 1) Generate genesis keypairs (idempotent across container restarts). +[ -f "$CONFIG_DIR/faucet.json" ] || solana-keygen new --no-passphrase -fso "$CONFIG_DIR/faucet.json" +[ -f "$LEDGER_DIR/identity.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/identity.json" +[ -f "$LEDGER_DIR/vote-account.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/vote-account.json" +[ -f "$LEDGER_DIR/stake-account.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/stake-account.json" + +# 2) Build the genesis ledger (only once). +if [ ! -f "$LEDGER_DIR/genesis.bin" ]; then + solana-genesis \ + --max-genesis-archive-unpacked-size 1073741824 \ + --enable-warmup-epochs \ + --bootstrap-validator "$LEDGER_DIR/identity.json" "$LEDGER_DIR/vote-account.json" "$LEDGER_DIR/stake-account.json" \ + --ledger "$LEDGER_DIR" \ + --faucet-pubkey "$CONFIG_DIR/faucet.json" \ + --faucet-lamports {faucet_lamports} \ + --hashes-per-tick {hashes_per_tick} \ + --cluster-type development +fi + +# 3) Point the local CLI at this node's RPC. +solana config set --url "http://127.0.0.1:$RPC_PORT" >/dev/null 2>&1 || true + +# 4) Start the faucet so other validators can fund themselves. +solana-faucet --keypair "$CONFIG_DIR/faucet.json" >/var/log/solana-faucet.log 2>&1 & + +echo "[seedemu-solana] starting bootstrap validator on $SELF_IP (rpc:$RPC_PORT gossip:$GOSSIP_PORT)" + +# 5) Start the bootstrap validator (foreground; keeps the container alive). +exec agave-validator \ + --identity "$LEDGER_DIR/identity.json" \ + --vote-account "$LEDGER_DIR/vote-account.json" \ + --ledger "$LEDGER_DIR" \ + --rpc-port "$RPC_PORT" \ + --rpc-bind-address 0.0.0.0 \ + --rpc-faucet-address "127.0.0.1:$FAUCET_PORT" \ + --gossip-host "$SELF_IP" \ + --gossip-port "$GOSSIP_PORT" \ + --dynamic-port-range "$DYNAMIC_PORT_RANGE" \ + --snapshot-interval-slots 200 \ + --no-incremental-snapshots \ + --full-rpc-api \ + --enable-rpc-transaction-history \ + --allow-private-addr \ + --no-poh-speed-test \ + --no-os-network-limits-test \ + --no-wait-for-vote-to-start-leader \ + --log - +""" + +_VALIDATOR_SCRIPT = r"""#!/bin/bash +# seedemu-solana: validator joining the private cluster +set -uo pipefail + +CONFIG_DIR=/opt/solana/config +LEDGER_DIR="$CONFIG_DIR/validator" +SELF_IP={self_ip} +RPC_PORT={rpc_port} +GOSSIP_PORT={gossip_port} +DYNAMIC_PORT_RANGE={dyn_low}-{dyn_high} +ENTRYPOINT={entrypoint_ip}:{boot_gossip_port} +RPC_URL=http://{entrypoint_ip}:{boot_rpc_port} +NODE_SOL={node_sol} + +mkdir -p "$LEDGER_DIR" + +# Keypairs for this validator (idempotent). +[ -f "$LEDGER_DIR/identity.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/identity.json" +[ -f "$LEDGER_DIR/vote-account.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/vote-account.json" +[ -f "$LEDGER_DIR/withdrawer.json" ] || solana-keygen new --no-passphrase -so "$LEDGER_DIR/withdrawer.json" + +# Wait for the bootstrap RPC to come up. +echo "[seedemu-solana] waiting for bootstrap RPC at $RPC_URL ..." +until solana --url "$RPC_URL" cluster-version >/dev/null 2>&1; do sleep 2; done +echo "[seedemu-solana] bootstrap reachable; setting up validator accounts." + +# Fund identity from the faucet (relayed via bootstrap RPC) and create the +# vote account, but only the first time. +IDENTITY_PUBKEY=$(solana-keygen pubkey "$LEDGER_DIR/identity.json") +if ! solana --url "$RPC_URL" vote-account "$LEDGER_DIR/vote-account.json" >/dev/null 2>&1; then + until solana --url "$RPC_URL" airdrop "$NODE_SOL" "$IDENTITY_PUBKEY" >/dev/null 2>&1; do + echo "[seedemu-solana] airdrop not ready, retrying ..."; sleep 2 + done + solana --url "$RPC_URL" create-vote-account \ + --fee-payer "$LEDGER_DIR/identity.json" \ + "$LEDGER_DIR/vote-account.json" "$LEDGER_DIR/identity.json" "$LEDGER_DIR/withdrawer.json" \ + || true +fi + +echo "[seedemu-solana] starting validator on $SELF_IP (entrypoint $ENTRYPOINT)" + +exec agave-validator \ + --identity "$LEDGER_DIR/identity.json" \ + --vote-account "$LEDGER_DIR/vote-account.json" \ + --ledger "$LEDGER_DIR" \ + --entrypoint "$ENTRYPOINT" \ + --rpc-port "$RPC_PORT" \ + --rpc-bind-address 0.0.0.0 \ + --gossip-host "$SELF_IP" \ + --gossip-port "$GOSSIP_PORT" \ + --dynamic-port-range "$DYNAMIC_PORT_RANGE" \ + --no-incremental-snapshots \ + --full-rpc-api \ + --enable-rpc-transaction-history \ + --allow-private-addr \ + --no-poh-speed-test \ + --no-os-network-limits-test \ + --log - +""" diff --git a/seedemu/services/SolanaService/SolanaService.py b/seedemu/services/SolanaService/SolanaService.py new file mode 100644 index 000000000..514c672e1 --- /dev/null +++ b/seedemu/services/SolanaService/SolanaService.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +from typing import Dict, List, Optional + +from seedemu.core.Emulator import Emulator +from seedemu.core.Node import Node +from seedemu.core.Service import Server, Service +from seedemu.core.enums import NetworkType +from seedemu.core.BaseSystem import BaseSystem + +from .SolanaEnum import SolanaNodeRole, SolanaNetworkType +from .SolanaServer import SolanaServer, SolanaBootstrapServer, SolanaValidatorServer + + +class SolanaService(Service): + """! + @brief Entry point for the Solana blockchain service. + + Mirrors the structure of ``EthereumService`` / ``MoneroService``: the + service tracks the mapping between virtual nodes and one or more private + Solana clusters (:class:`SolanaNetwork`). During the *configure* phase it + resolves the virtual-node bindings and wires every validator to the + cluster's bootstrap node; during *render* each server installs the real + Agave toolchain onto its physical node. + + Nodes run on the ``seedemu-solana`` base image (BaseSystem.SEEDEMU_SOLANA), + which ships the pinned Agave binaries (see docker_images/seedemu-solana). + """ + + def __init__(self): + super().__init__() + self.__networks: Dict[str, "SolanaNetwork"] = {} + # The cluster needs the base topology and routing to exist first + # (PRINCIPLES.md P2: declare render-order dependencies explicitly). + self.addDependency('Base', False, False) + self.addDependency('Routing', False, False) + + # ------------------------------------------------------------------ # + # Service interface + # ------------------------------------------------------------------ # + def getName(self) -> str: + return "SolanaService" + + def _createServer(self) -> Server: + raise AssertionError( + "Create Solana nodes via SolanaNetwork.createBootstrapValidator()/createValidator()" + ) + + def configure(self, emulator: Emulator): + # Resolve cluster connectivity first (needs the bindings), then let the + # base Service bind/configure each pending target. + for network in self.__networks.values(): + network.configure(emulator) + super().configure(emulator) + + def _doInstall(self, node: Node, server: Server): + node.setBaseSystem(BaseSystem.SEEDEMU_SOLANA) + self._log("install {} on as{}/{}".format( + server.__class__.__name__, node.getAsn(), node.getName())) + super()._doInstall(node, server) + + # ------------------------------------------------------------------ # + # Blockchain management + # ------------------------------------------------------------------ # + def createBlockchain( + self, + name: str, + net_type: SolanaNetworkType = SolanaNetworkType.DEVELOPMENT, + ) -> "SolanaNetwork": + """! + @brief Create and register a logical private Solana cluster. + + @param name unique cluster identifier. + @param net_type cluster type (only DEVELOPMENT is supported for now). + + @returns the new :class:`SolanaNetwork`. + """ + assert name not in self.__networks, f"Duplicated Solana blockchain name: {name}" + network = SolanaNetwork(self, name, net_type) + self.__networks[name] = network + return network + + def getBlockchainNames(self) -> List[str]: + """!@brief Return the names of all registered clusters.""" + return list(self.__networks.keys()) + + def getBlockchain(self, name: str) -> "SolanaNetwork": + """!@brief Return the cluster with the given name.""" + return self.__networks[name] + + # ------------------------------------------------------------------ # + # Internal helpers + # ------------------------------------------------------------------ # + def _register_node(self, vnode: str, server: SolanaServer) -> SolanaServer: + assert vnode not in self._pending_targets, \ + f"Duplicated Solana virtual node: {vnode}" + self._pending_targets[vnode] = server + return server + + +class SolanaNetwork: + """! + @brief Representation of a single private Solana cluster. + + Holds the cluster's nodes and, during ``configure``, resolves each virtual + node's bound IP and points every validator at the bootstrap node's gossip + entrypoint and RPC. Exactly one bootstrap validator is required per cluster. + """ + + def __init__(self, service: SolanaService, name: str, net_type: SolanaNetworkType): + self._service = service + self._name = name + self._net_type = net_type + self._nodes: Dict[str, SolanaServer] = {} + self._bootstrap_vnode: Optional[str] = None + + # ------------------------------------------------------------------ # + # Public API (PRINCIPLES.md P3 / P4) + # ------------------------------------------------------------------ # + def getName(self) -> str: + return self._name + + def createBootstrapValidator(self, vnode: str) -> SolanaBootstrapServer: + """! + @brief Create the cluster's genesis / bootstrap validator. + + Only one bootstrap validator may exist per cluster. + + @param vnode virtual node name (resolved to a physical node by a Binding). + @returns the :class:`SolanaBootstrapServer`. + """ + assert self._bootstrap_vnode is None, \ + f"Solana cluster '{self._name}' already has a bootstrap validator ({self._bootstrap_vnode})" + server = SolanaBootstrapServer(self) + self._bootstrap_vnode = vnode + self._nodes[vnode] = server + self._service._register_node(vnode, server) + return server + + def createValidator(self, vnode: str) -> SolanaValidatorServer: + """! + @brief Create a validator that joins this cluster via the bootstrap node. + + @param vnode virtual node name (resolved to a physical node by a Binding). + @returns the :class:`SolanaValidatorServer`. + """ + server = SolanaValidatorServer(self) + self._nodes[vnode] = server + self._service._register_node(vnode, server) + return server + + # ------------------------------------------------------------------ # + # Configure (resolve bindings -> connectivity) + # ------------------------------------------------------------------ # + def configure(self, emulator: Emulator): + if not self._nodes: + return + assert self._bootstrap_vnode is not None, \ + f"Solana cluster '{self._name}' needs a bootstrap validator (createBootstrapValidator)." + + # Resolve every node's primary IP from its binding. + for vnode, server in self._nodes.items(): + node = emulator.getBindingFor(vnode) + server.set_self_ip(self._get_primary_ip(node)) + + # Hand every validator the bootstrap node's gossip + RPC endpoint. + boot = self._nodes[self._bootstrap_vnode] + for vnode, server in self._nodes.items(): + if server is boot: + continue + server.set_bootstrap_endpoint( + ip=boot._self_ip, + gossip_port=boot.get_gossip_port(), + rpc_port=boot.get_rpc_port(), + ) + + # ------------------------------------------------------------------ # + # Internal utilities + # ------------------------------------------------------------------ # + def _get_primary_ip(self, node: Node) -> str: + for iface in node.getInterfaces(): + if iface.getNet().getType() == NetworkType.Local: + return str(iface.getAddress()) + raise AssertionError(f"Node {node.getName()} has no local network interface") + + def getBootstrapNode(self) -> Optional[str]: + """!@brief Return the virtual-node name of the bootstrap validator.""" + return self._bootstrap_vnode + + def getValidatorNodes(self) -> List[str]: + """!@brief Return the virtual-node names of the joining validators.""" + return [v for v in self._nodes if v != self._bootstrap_vnode] diff --git a/seedemu/services/SolanaService/__init__.py b/seedemu/services/SolanaService/__init__.py new file mode 100644 index 000000000..2748504ee --- /dev/null +++ b/seedemu/services/SolanaService/__init__.py @@ -0,0 +1,20 @@ +from .SolanaEnum import ( + SolanaNodeRole, + SolanaNetworkType, +) +from .SolanaServer import ( + SolanaServer, + SolanaBootstrapServer, + SolanaValidatorServer, +) +from .SolanaService import SolanaService, SolanaNetwork + +__all__ = [ + "SolanaService", + "SolanaNetwork", + "SolanaServer", + "SolanaBootstrapServer", + "SolanaValidatorServer", + "SolanaNodeRole", + "SolanaNetworkType", +] diff --git a/seedemu/services/TrafficService/scapy_traffic_service.py b/seedemu/services/TrafficService/scapy_traffic_service.py index 08868c857..40a0129de 100644 --- a/seedemu/services/TrafficService/scapy_traffic_service.py +++ b/seedemu/services/TrafficService/scapy_traffic_service.py @@ -20,7 +20,7 @@ def install_softwares(self, node: Node): """ node.addSoftware("python3") node.addSoftware("python3-pip") - node.addBuildCommand("pip3 install scapy==2.5.0") + node.addBuildCommand("pip3 install --break-system-packages scapy==2.5.0") scapy_generator_file = ( os.path.dirname(os.path.realpath(__file__)) + "/scapy_script.py" ) diff --git a/seedemu/services/__init__.py b/seedemu/services/__init__.py index e7458c13f..9c9194e5d 100644 --- a/seedemu/services/__init__.py +++ b/seedemu/services/__init__.py @@ -10,10 +10,13 @@ from .DHCPService import DHCPServer, DHCPService from .EthereumService import * from .MoneroService import * +from .SolanaService import * from .ScionBwtestService import ScionBwtestService from .ScionBwtestClientService import ScionBwtestClientService from .KuboService import * from .CAService import CAService, CAServer, RootCAStore from .ChainlinkService import * from .TrafficService import * -from .DevService import * \ No newline at end of file +from .DevService import * +from .CDNService import CDNService, CDNOriginServer, CDNEdgeServer +from .ExaBgpService import ExaBgpService, ExaBgpServer diff --git a/seedemu/testing/__init__.py b/seedemu/testing/__init__.py new file mode 100644 index 000000000..639c834a1 --- /dev/null +++ b/seedemu/testing/__init__.py @@ -0,0 +1,20 @@ +"""Testing runners for standardized SEED Emulator examples and emulations.""" + +from .base import TestRunner, TestRunnerError +from .blockchain import BlockchainTestRunner, EthereumRuntimeTest +from .internet import InternetTestRunner +from .registry import create_runner +from .runtime import ComposeRuntimeTest, ComposeService +from .satellite import SatelliteTestRunner + +__all__ = [ + "BlockchainTestRunner", + "ComposeRuntimeTest", + "ComposeService", + "EthereumRuntimeTest", + "InternetTestRunner", + "SatelliteTestRunner", + "TestRunner", + "TestRunnerError", + "create_runner", +] diff --git a/seedemu/testing/base.py b/seedemu/testing/base.py new file mode 100644 index 000000000..c814e7420 --- /dev/null +++ b/seedemu/testing/base.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +try: + from .compose import docker_compose_command +except ImportError: + from compose import docker_compose_command + + +class TestRunnerError(Exception): + """Raised when a test runner manifest or lifecycle operation is invalid.""" + + +class TestRunner: + """Base runner for standardized emulation testing. + + This class owns generic lifecycle behavior: loading manifests, running + commands, writing artifacts, starting/stopping Docker Compose environments, + retrying probes, and executing custom test programs. Domain-specific + runners extend probe handlers and manifest validation. + """ + + runner_name = "generic" + probe_handlers: Dict[str, str] = { + "exec": "probe_exec", + "http": "probe_http", + "tcp": "probe_tcp", + "compose-ps": "probe_compose_ps", + "log-contains": "probe_log_contains", + } + + manifest_path: Path + manifest: Dict[str, Any] + emulation_dir: Path + artifact_dir: Optional[Path] + + def __init__(self, manifest_path: Path, artifact_dir: Optional[Path] = None): + self.manifest_path = manifest_path.resolve() + self.emulation_dir = self.manifest_path.parent + self.artifact_dir = artifact_dir.resolve() if artifact_dir is not None else None + if self.artifact_dir is not None: + self.artifact_dir.mkdir(parents=True, exist_ok=True) + self.manifest = self.load_manifest() + self.validate_manifest() + + def clean(self) -> int: + """Remove generated files declared in compile.clean.""" + + clean_paths = self.manifest.get("compile", {}).get("clean", []) + for item in clean_paths: + target = self.resolve_emulation_path(item) + self.assert_inside_emulation_dir(target) + if target.is_dir(): + shutil.rmtree(target) + self.log("removed directory {}".format(target)) + elif target.exists(): + target.unlink() + self.log("removed file {}".format(target)) + return 0 + + def compile(self) -> int: + """Compile the emulation and check declared outputs.""" + + compile_cfg = self.manifest.get("compile", {}) + if not compile_cfg.get("enabled", True): + self.log("compile is disabled for {}".format(self.emulation_id())) + return 0 + + script = self.resolve_emulation_path(self.manifest["script"]) + output = compile_cfg.get("output", "output") + timeout = int(compile_cfg.get("timeout", 900)) + env = self.merged_env(compile_cfg.get("env", {})) + + command = compile_cfg.get("command") + if command is not None: + cmd = self.coerce_command(command) + else: + cmd = [sys.executable, script.name] + if compile_cfg.get("standard_args", True): + platform = self.manifest.get("platform", "amd") + cmd.extend(["--platform", str(platform), "--output", str(output)]) + cmd.extend(str(arg) for arg in compile_cfg.get("args", [])) + + result = self.run_command("compile", cmd, cwd=script.parent, env=env, timeout=timeout) + if result.returncode != 0: + return result.returncode + + missing = [ + item + for item in compile_cfg.get("expected", []) + if not self.resolve_emulation_path(item).exists() + ] + if missing: + self.log("missing expected outputs: {}".format(", ".join(missing))) + self.write_json("compile-missing.json", {"missing": missing}) + return 1 + return 0 + + def build(self) -> int: + """Run Docker Compose build for the emulation.""" + + build_cfg = self.manifest.get("build", {}) + if not build_cfg.get("enabled", True): + self.log("build is disabled for {}".format(self.emulation_id())) + return 0 + + compose = self.compose_file() + if not compose.is_file(): + self.log("compose file not found: {}".format(compose)) + return 1 + + cmd = docker_compose_command() + ["-f", str(compose), "build"] + return self.run_command( + "build", + cmd, + cwd=compose.parent, + env=self.docker_env(), + timeout=int(build_cfg.get("timeout", 1800)), + ).returncode + + def up(self) -> int: + """Start the emulation with Docker Compose and run readiness probes.""" + + compose = self.compose_file() + if not compose.is_file(): + self.log("compose file not found: {}".format(compose)) + return 1 + + cmd = docker_compose_command() + ["-f", str(compose), "up", "-d"] + result = self.run_command("up", cmd, cwd=compose.parent, env=self.docker_env()) + if result.returncode != 0: + return result.returncode + return self.readiness() + + def down(self) -> int: + """Stop the emulation and remove Compose-created resources.""" + + compose = self.compose_file() + if not compose.exists(): + self.log("compose file not found, skipping down: {}".format(compose)) + return 0 + + cmd = docker_compose_command() + ["-f", str(compose), "down", "--remove-orphans"] + return self.run_command("down", cmd, cwd=compose.parent, env=self.docker_env()).returncode + + def readiness(self) -> int: + """Run runtime.readiness probes.""" + + probes = self.manifest.get("runtime", {}).get("readiness", []) + if not probes: + return 0 + return self.run_probes("readiness", probes) + + def probe(self) -> int: + """Run probes declared under probes.""" + + return self.run_probes("probe", self.manifest.get("probes", [])) + + def test(self) -> int: + """Run custom test programs declared under test_programs.""" + + return self.run_test_programs(self.manifest.get("test_programs", [])) + + def all(self) -> int: + """Run clean, compile, build, up, probe, test, and down.""" + + status = 0 + try: + for step in (self.clean, self.compile, self.build, self.up, self.probe, self.test): + status = step() + if status != 0: + break + finally: + down_status = self.down() + if status == 0 and down_status != 0: + status = down_status + return status + + def compose_file(self) -> Path: + """Return the Compose file path for build/runtime operations.""" + + compose = self.manifest.get("runtime", {}).get("compose") + if compose is None: + output = self.manifest.get("compile", {}).get("output", "output") + compose = str(Path(output) / "docker-compose.yml") + return self.resolve_emulation_path(compose) + + def emulation_id(self) -> str: + """Return the stable emulation ID from the manifest.""" + + return str(self.manifest["id"]) + + def run_probes(self, stage: str, probes: Sequence[Dict[str, Any]]) -> int: + failures = [] + results = [] + for probe in probes: + result = self.run_probe_with_retries(probe) + results.append(result) + self.log("{} {}: {}".format(result["status"], result["name"], result.get("message", ""))) + if result["status"] == "failed" and probe.get("required", True): + failures.append(probe["name"]) + + self.write_json("{}-summary.json".format(stage), {"results": results, "failures": failures}) + if failures: + self.log("{} failures: {}".format(stage, ", ".join(failures))) + return 1 + return 0 + + def run_test_programs(self, tests: Sequence[Dict[str, Any]]) -> int: + failures = [] + results = [] + for test in tests: + name = str(test["name"]) + result = self.run_command( + "test-{}".format(self.slug(name)), + self.test_command(test), + cwd=self.emulation_dir, + env=self.test_env(test.get("env", {})), + timeout=int(test.get("timeout", 300)), + ) + expected = int(test.get("expect_exit", 0)) + status = "passed" if result.returncode == expected else "failed" + message = "exit {}, expected {}".format(result.returncode, expected) + self.log("{} {}: {}".format(status, name, message)) + results.append( + { + "name": name, + "status": status, + "exit": result.returncode, + "expected_exit": expected, + "message": message, + } + ) + if status == "failed" and test.get("required", True): + failures.append(name) + + self.write_json("test-summary.json", {"results": results, "failures": failures}) + if failures: + self.log("test failures: {}".format(", ".join(failures))) + return 1 + return 0 + + def run_probe_with_retries(self, probe: Dict[str, Any]) -> Dict[str, Any]: + name = str(probe["name"]) + retries = int(probe.get("retries", 1)) + interval = float(probe.get("interval", 0)) + last_message = "" + + for attempt in range(1, retries + 1): + self.log("probe {} attempt {}/{}".format(name, attempt, retries)) + passed, message = self.run_one_probe(probe) + if passed: + return { + "name": name, + "type": probe["type"], + "status": "passed", + "attempts": attempt, + "message": message, + } + last_message = message + if attempt < retries and interval > 0: + time.sleep(interval) + + status = "failed" if probe.get("required", True) else "skipped" + return { + "name": name, + "type": probe["type"], + "status": status, + "attempts": retries, + "message": last_message, + } + + def run_one_probe(self, probe: Dict[str, Any]) -> tuple[bool, str]: + probe_type = str(probe["type"]) + handler_name = self.probe_handlers.get(probe_type) + if handler_name is None: + raise TestRunnerError("unknown probe type for {} runner: {}".format(self.runner_name, probe_type)) + return getattr(self, handler_name)(probe) + + def probe_exec(self, probe: Dict[str, Any]) -> tuple[bool, str]: + compose = self.compose_file() + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + [ + *docker_compose_command(), + "-f", + str(compose), + "exec", + "-T", + str(probe["service"]), + "sh", + "-lc", + str(probe["command"]), + ], + cwd=compose.parent, + env=self.docker_env(), + timeout=int(probe.get("timeout", 30)), + ) + expected = int(probe.get("expect_exit", 0)) + if result.returncode != expected: + return False, "exit {}, expected {}".format(result.returncode, expected) + return self.check_text_expectations(probe, result.stdout or "", result.stderr or "") + + def probe_http(self, probe: Dict[str, Any]) -> tuple[bool, str]: + request = urllib.request.Request( + str(probe["url"]), + method=str(probe.get("method", "GET")), + headers={str(k): str(v) for k, v in probe.get("headers", {}).items()}, + ) + data = None + if "body" in probe: + data = str(probe["body"]).encode("utf-8") + + try: + with urllib.request.urlopen(request, data=data, timeout=int(probe.get("timeout", 10))) as response: + body = response.read().decode("utf-8", errors="replace") + expected_status = int(probe.get("expect_status", 200)) + if response.status != expected_status: + return False, "HTTP {}, expected {}".format(response.status, expected_status) + return self.check_body_expectations(probe, body) + except Exception as exc: + return False, str(exc) + + def probe_tcp(self, probe: Dict[str, Any]) -> tuple[bool, str]: + try: + with socket.create_connection( + (str(probe["host"]), int(probe["port"])), + timeout=int(probe.get("timeout", 10)), + ): + return True, "connected" + except OSError as exc: + return False, str(exc) + + def probe_compose_ps(self, probe: Dict[str, Any]) -> tuple[bool, str]: + compose = self.compose_file() + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + docker_compose_command() + ["-f", str(compose), "ps", "--format", "json"], + cwd=compose.parent, + env=self.docker_env(), + timeout=int(probe.get("timeout", 30)), + ) + if result.returncode != 0: + return False, "docker compose ps exited {}".format(result.returncode) + + expected_services = {str(service) for service in probe.get("services", [])} + if not expected_services: + return True, "compose ps succeeded" + + running = set() + observed = [] + for item in self.parse_compose_ps_output(result.stdout or ""): + name = item.get("Service") or item.get("Name") + state = item.get("State") or item.get("Status") + observed.append("{}={}".format(name, state)) + state_text = str(state).lower() + if name in expected_services and ( + state_text.startswith("running") or state_text.startswith("up") + ): + running.add(name) + + missing = sorted(expected_services - running) + if missing: + return False, "services not running: {}; observed: {}".format( + ", ".join(missing), + ", ".join(observed) or "", + ) + return True, "all services running" + + def probe_log_contains(self, probe: Dict[str, Any]) -> tuple[bool, str]: + compose = self.compose_file() + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + docker_compose_command() + ["-f", str(compose), "logs", str(probe["service"])], + cwd=compose.parent, + env=self.docker_env(), + timeout=int(probe.get("timeout", 30)), + ) + if result.returncode != 0: + return False, "docker compose logs exited {}".format(result.returncode) + + text = result.stdout or "" + pattern = str(probe["pattern"]) + if probe.get("regex", False): + return (True, "regex matched") if re.search(pattern, text) else (False, "regex did not match") + return (True, "pattern found") if pattern in text else (False, "pattern not found") + + def load_manifest(self) -> Dict[str, Any]: + if not self.manifest_path.is_file(): + raise TestRunnerError("manifest does not exist: {}".format(self.manifest_path)) + try: + import yaml + except ImportError as exc: + raise TestRunnerError("PyYAML is required to read test manifests") from exc + + data = yaml.safe_load(self.manifest_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise TestRunnerError("manifest must be a YAML mapping") + return data + + def validate_manifest(self) -> None: + if not self.manifest.get("id"): + raise TestRunnerError("test manifest must define id") + if not self.manifest.get("script"): + raise TestRunnerError("test manifest must define script") + script = self.resolve_emulation_path(self.manifest["script"]) + if not script.is_file(): + raise TestRunnerError("emulation script does not exist: {}".format(script)) + + for probe in self.manifest.get("probes", []): + self.validate_probe(probe) + for probe in self.manifest.get("runtime", {}).get("readiness", []): + self.validate_probe(probe) + for test in self.manifest.get("test_programs", []): + self.validate_test_program(test) + + def validate_probe(self, probe: Dict[str, Any]) -> None: + if "name" not in probe or "type" not in probe: + raise TestRunnerError("each probe must define name and type") + required = self.required_probe_fields(str(probe["type"])) + if required is None: + raise TestRunnerError("unknown probe type for {} runner: {}".format(self.runner_name, probe["type"])) + missing = [field for field in required if field not in probe] + if missing: + raise TestRunnerError( + "probe {} missing fields: {}".format(probe.get("name", ""), ", ".join(missing)) + ) + + def validate_test_program(self, test: Dict[str, Any]) -> None: + if "name" not in test: + raise TestRunnerError("each test program must define name") + if "script" not in test and "command" not in test: + raise TestRunnerError( + "test program {} must define script or command".format(test.get("name", "")) + ) + if "script" in test: + script = self.resolve_emulation_path(test["script"]) + if not script.is_file(): + raise TestRunnerError("test program script does not exist: {}".format(script)) + + def required_probe_fields(self, probe_type: str) -> Optional[List[str]]: + return { + "exec": ["service", "command"], + "http": ["url"], + "tcp": ["host", "port"], + "compose-ps": [], + "log-contains": ["service", "pattern"], + }.get(probe_type) + + def resolve_emulation_path(self, value: Any) -> Path: + path = Path(str(value)) + if path.is_absolute(): + return path + return (self.emulation_dir / path).resolve() + + def test_command(self, test: Dict[str, Any]) -> List[str]: + if "command" in test: + return self.coerce_command(test["command"]) + + script = self.resolve_emulation_path(test["script"]) + cmd = [sys.executable, str(script)] + cmd.extend(str(arg) for arg in test.get("args", [])) + return cmd + + def assert_inside_emulation_dir(self, target: Path) -> None: + base = self.emulation_dir.resolve() + if target != base and base not in target.parents: + raise TestRunnerError("refusing to operate outside emulation directory: {}".format(target)) + + def run_command( + self, + name: str, + cmd: Sequence[str], + *, + cwd: Path, + env: Optional[Dict[str, str]] = None, + timeout: Optional[int] = None, + ) -> subprocess.CompletedProcess[str]: + self.log("cwd={} cmd={}".format(cwd, " ".join(cmd))) + result = subprocess.run( + list(cmd), + cwd=str(cwd), + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + self.write_command_log(name, cmd, cwd, result) + return result + + def write_command_log( + self, + name: str, + cmd: Sequence[str], + cwd: Path, + result: subprocess.CompletedProcess[str], + ) -> None: + if self.artifact_dir is None: + return + log_path = self.artifact_dir / "logs" / "{}.log".format(self.slug(name)) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text( + "COMMAND: {}\nCWD: {}\nEXIT: {}\n\nSTDOUT:\n{}\n\nSTDERR:\n{}\n".format( + " ".join(cmd), + cwd, + result.returncode, + result.stdout, + result.stderr, + ), + encoding="utf-8", + ) + + def write_json(self, name: str, data: Dict[str, Any]) -> None: + if self.artifact_dir is None: + return + path = self.artifact_dir / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + def test_env(self, extra: Dict[str, Any]) -> Dict[str, str]: + env = self.docker_env() + env.update( + { + "TEST_RUNNER_NAME": self.runner_name, + "TEST_RUNNER_EMULATION_ID": self.emulation_id(), + "TEST_RUNNER_EMULATION_DIR": str(self.emulation_dir), + "TEST_RUNNER_MANIFEST": str(self.manifest_path), + "TEST_RUNNER_COMPOSE_FILE": str(self.compose_file()), + "TEST_RUNNER_ARTIFACT_DIR": str(self.artifact_dir or ""), + "EXAMPLE_RUNNER_EXAMPLE_ID": self.emulation_id(), + "EXAMPLE_RUNNER_EXAMPLE_DIR": str(self.emulation_dir), + "EXAMPLE_RUNNER_MANIFEST": str(self.manifest_path), + "EXAMPLE_RUNNER_COMPOSE_FILE": str(self.compose_file()), + "EXAMPLE_RUNNER_ARTIFACT_DIR": str(self.artifact_dir or ""), + } + ) + env.update({str(k): str(v) for k, v in extra.items()}) + return env + + @staticmethod + def coerce_command(command: Any) -> List[str]: + if isinstance(command, list): + return [str(item) for item in command] + raise TestRunnerError("command must be a list of arguments") + + @staticmethod + def merged_env(extra: Dict[str, Any]) -> Dict[str, str]: + env = dict(os.environ) + env.update({str(k): str(v) for k, v in extra.items()}) + return env + + @staticmethod + def docker_env() -> Dict[str, str]: + env = TestRunner.merged_env({}) + env.setdefault("DOCKER_BUILDKIT", "0") + env.setdefault("COMPOSE_BAKE", "false") + env.setdefault("COMPOSE_PARALLEL_LIMIT", "1") + return env + + @staticmethod + def parse_compose_ps_output(output: str) -> List[Dict[str, Any]]: + output = output.strip() + if not output: + return [] + + try: + data = json.loads(output) + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + if isinstance(data, dict): + return [data] + except json.JSONDecodeError: + pass + + items = [] + for line in output.splitlines(): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(item, dict): + items.append(item) + return items + + @staticmethod + def check_body_expectations(probe: Dict[str, Any], body: str) -> tuple[bool, str]: + contains = probe.get("expect_body_contains") + if contains is not None and str(contains) not in body: + return False, "body does not contain {}".format(contains) + regex = probe.get("expect_body_regex") + if regex is not None and re.search(str(regex), body) is None: + return False, "body regex did not match" + return True, "body matched" + + @staticmethod + def check_text_expectations(probe: Dict[str, Any], stdout: str, stderr: str) -> tuple[bool, str]: + stdout_contains = probe.get("expect_stdout_contains") + if stdout_contains is not None and str(stdout_contains) not in stdout: + return False, "stdout does not contain {}".format(stdout_contains) + stdout_regex = probe.get("expect_stdout_regex") or probe.get("expect_answer_regex") + if stdout_regex is not None and re.search(str(stdout_regex), stdout) is None: + return False, "stdout regex did not match" + stderr_contains = probe.get("expect_stderr_contains") + if stderr_contains is not None and str(stderr_contains) not in stderr: + return False, "stderr does not contain {}".format(stderr_contains) + answer = probe.get("expect_answer") + if answer is not None and str(answer) not in stdout: + return False, "answer not found in stdout" + return True, "text matched" + + @staticmethod + def slug(value: Any) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value)).strip("-") or "check" + + @staticmethod + def log(message: str) -> None: + print("[test-runner] {}".format(message)) diff --git a/seedemu/testing/blockchain.py b/seedemu/testing/blockchain.py new file mode 100644 index 000000000..93dfe991d --- /dev/null +++ b/seedemu/testing/blockchain.py @@ -0,0 +1,815 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import re +import shlex +import time +import urllib.request +from pathlib import Path +from typing import Any, Dict, List, Optional + +try: + from .base import TestRunner, TestRunnerError + from .runtime import ADDRESS_LABEL, ComposeRuntimeTest, ComposeService +except ImportError: + from base import TestRunner, TestRunnerError + from runtime import ADDRESS_LABEL, ComposeRuntimeTest, ComposeService + + +META_PREFIX = "org.seedsecuritylabs.seedemu.meta." +CLASS_LABEL = META_PREFIX + "class" +DISPLAY_LABEL = META_PREFIX + "displayname" +ETH_NODE_ID_LABEL = META_PREFIX + "ethereum.node_id" +ETH_ROLE_LABEL = META_PREFIX + "ethereum.role" +ETH_CONSENSUS_LABEL = META_PREFIX + "ethereum.consensus" +ETH_CHAIN_ID_LABEL = META_PREFIX + "ethereum.chain_id" + +DEFAULT_TRANSFER_RECIPIENT = "0x1000000000000000000000000000000000000001" +LOCAL_ACCOUNT_CACHE_VERSION = 1 + + +def _label_list(value: object) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value] + text = str(value) + try: + decoded = json.loads(text) + except json.JSONDecodeError: + return [text] + if isinstance(decoded, list): + return [str(item) for item in decoded] + return [str(decoded)] + + +def _ethereum_service_matches( + labels: Dict[str, object], + *, + class_contains: Optional[str] = None, + display_contains: Optional[str] = None, + role: Optional[str] = None, + consensus: Optional[str] = None, +) -> bool: + if class_contains and class_contains not in str(labels.get(CLASS_LABEL, "")): + return False + if display_contains and display_contains not in str(labels.get(DISPLAY_LABEL, "")): + return False + if role and role not in _label_list(labels.get(ETH_ROLE_LABEL)): + return False + if consensus and str(labels.get(ETH_CONSENSUS_LABEL, "")).upper() != consensus.upper(): + return False + if ETH_NODE_ID_LABEL not in labels and not class_contains and not display_contains: + return False + return True + + +def _matching_ethereum_services( + compose: Dict[str, object], + *, + class_contains: Optional[str] = None, + display_contains: Optional[str] = None, + role: Optional[str] = None, + consensus: Optional[str] = None, +) -> List[ComposeService]: + services: List[ComposeService] = [] + for name, service in compose.get("services", {}).items(): + labels = service.get("labels", {}) + if not isinstance(labels, dict): + continue + if not _ethereum_service_matches( + labels, + class_contains=class_contains, + display_contains=display_contains, + role=role, + consensus=consensus, + ): + continue + address = str(labels.get(ADDRESS_LABEL, "")).split("/", 1)[0] + services.append(ComposeService(name=str(name), address=address, labels=dict(labels))) + return sorted(services, key=lambda item: item.name) + + +class BlockchainTestRunner(TestRunner): + """Runner for blockchain emulation tests.""" + + runner_name = "blockchain" + probe_handlers = { + **TestRunner.probe_handlers, + "json-rpc": "probe_json_rpc", + "ethereum-rpc": "probe_json_rpc", + "blockchain-rpc": "probe_json_rpc", + "ethereum-service-count": "probe_ethereum_service_count", + "ethereum-compose-ps": "probe_ethereum_compose_ps", + "ethereum-exec": "probe_ethereum_exec", + "ethereum-block-progress": "probe_ethereum_block_progress", + } + + def probe_json_rpc(self, probe: Dict[str, Any]) -> tuple[bool, str]: + payload = { + "jsonrpc": "2.0", + "id": probe.get("id", 1), + "method": probe["method"], + "params": probe.get("params", []), + } + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + str(probe["url"]), + data=body, + method="POST", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=int(probe.get("timeout", 10))) as response: + text = response.read().decode("utf-8", errors="replace") + if response.status != int(probe.get("expect_status", 200)): + return False, "HTTP {}, expected {}".format(response.status, probe.get("expect_status", 200)) + data = json.loads(text) + except Exception as exc: + return False, str(exc) + + if "expect_result" in probe and data.get("result") != probe["expect_result"]: + return False, "JSON-RPC result did not match" + if "expect_result_contains" in probe and str(probe["expect_result_contains"]) not in str(data.get("result")): + return False, "JSON-RPC result did not contain expected text" + if "expect_error" in probe and data.get("error") != probe["expect_error"]: + return False, "JSON-RPC error did not match" + if probe.get("expect_no_error", True) and data.get("error") is not None: + return False, "JSON-RPC returned error {}".format(data.get("error")) + return True, "JSON-RPC response matched" + + def required_probe_fields(self, probe_type: str) -> Optional[List[str]]: + fields = { + "json-rpc": ["url", "method"], + "ethereum-rpc": ["url", "method"], + "blockchain-rpc": ["url", "method"], + "ethereum-service-count": [], + "ethereum-compose-ps": [], + "ethereum-exec": ["command"], + "ethereum-block-progress": [], + }.get(probe_type) + return fields if fields is not None else super().required_probe_fields(probe_type) + + def probe_ethereum_service_count(self, probe: Dict[str, Any]) -> tuple[bool, str]: + services = self.ethereum_services_from_probe(probe) + return self.check_service_count(probe, services) + + def probe_ethereum_compose_ps(self, probe: Dict[str, Any]) -> tuple[bool, str]: + services = self.ethereum_services_from_probe(probe) + count_ok, count_message = self.check_service_count(probe, services) + if not count_ok: + return False, count_message + + compose = self.compose_file() + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + ["docker", "compose", "-f", str(compose), "ps", "--format", "json"], + cwd=compose.parent, + env=self.docker_env(), + timeout=int(probe.get("timeout", 30)), + ) + if result.returncode != 0: + return False, "docker compose ps exited {}".format(result.returncode) + + expected = {service.name for service in services} + running = set() + observed = [] + for item in self.parse_compose_ps_output(result.stdout or ""): + name = str(item.get("Service") or item.get("Name")) + state = str(item.get("State") or item.get("Status")) + observed.append("{}={}".format(name, state)) + state_text = state.lower() + if name in expected and (state_text.startswith("running") or state_text.startswith("up")): + running.add(name) + + missing = sorted(expected - running) + if missing: + return False, "services not running: {}; observed: {}".format( + ", ".join(missing), + ", ".join(observed) or "", + ) + return True, "{} matched service(s) running".format(len(services)) + + def probe_ethereum_exec(self, probe: Dict[str, Any]) -> tuple[bool, str]: + service = self.ethereum_service_from_probe(probe) + if service is None: + return False, "no matching Ethereum service" + compose = self.compose_file() + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + [ + "docker", + "compose", + "-f", + str(compose), + "exec", + "-T", + service.name, + "sh", + "-lc", + str(probe["command"]), + ], + cwd=compose.parent, + env=self.docker_env(), + timeout=int(probe.get("timeout", 45)), + ) + expected = int(probe.get("expect_exit", 0)) + if result.returncode != expected: + return False, "{} exited {}, expected {}".format(service.name, result.returncode, expected) + matched, message = self.check_text_expectations(probe, result.stdout or "", result.stderr or "") + if not matched: + return False, "{}: {}".format(service.name, message) + return True, "{}: {}".format(service.name, message) + + def probe_ethereum_block_progress(self, probe: Dict[str, Any]) -> tuple[bool, str]: + service = self.ethereum_service_from_probe(probe) + if service is None: + return False, "no matching Ethereum service" + + start_block: Optional[int] = None + last_block: Optional[int] = None + last_error = "" + min_delta = int(probe.get("min_delta", 1)) + retries = int(probe.get("block_retries", probe.get("retries", 60))) + interval = float(probe.get("block_interval", probe.get("interval", 5))) + for attempt in range(1, retries + 1): + try: + block = self.get_ethereum_block_number(service, int(probe.get("timeout", 60))) + except Exception as exc: + last_error = str(exc) + else: + if start_block is None: + start_block = block + last_block = block + if block >= start_block + min_delta: + return True, "{} advanced from {} to {} in {} attempt(s)".format( + service.name, + start_block, + block, + attempt, + ) + if attempt < retries: + time.sleep(interval) + return False, "{} did not advance by {}; start={}, latest={}, error={}".format( + service.name, + min_delta, + start_block, + last_block, + last_error, + ) + + def ethereum_services_from_probe(self, probe: Dict[str, Any]) -> List[ComposeService]: + return _matching_ethereum_services( + self.load_compose_file(), + class_contains=probe.get("class_contains"), + display_contains=probe.get("display_contains"), + role=probe.get("role"), + consensus=probe.get("consensus"), + ) + + def ethereum_service_from_probe(self, probe: Dict[str, Any]) -> Optional[ComposeService]: + services = self.ethereum_services_from_probe(probe) + if not services: + return None + index = int(probe.get("match_index", 0)) + if index < 0 or index >= len(services): + raise TestRunnerError("probe {} match_index out of range".format(probe["name"])) + return services[index] + + def check_service_count(self, probe: Dict[str, Any], services: List[ComposeService]) -> tuple[bool, str]: + count = len(services) + expected = probe.get("expected_count") + if expected is not None and count != int(expected): + return False, "found {} service(s), expected {}; services={}".format( + count, + expected, + ", ".join(service.name for service in services) or "", + ) + minimum = int(probe.get("minimum_count", 1)) + if expected is None and count < minimum: + return False, "found {} service(s), expected at least {}".format(count, minimum) + return True, "found {} service(s): {}".format( + count, + ", ".join(service.name for service in services) or "", + ) + + def get_ethereum_block_number(self, service: ComposeService, timeout: int) -> int: + compose = self.compose_file() + result = self.run_command( + "probe-{}-block-number".format(service.name), + [ + "docker", + "compose", + "-f", + str(compose), + "exec", + "-T", + service.name, + "sh", + "-lc", + "geth attach --exec 'eth.blockNumber'", + ], + cwd=compose.parent, + env=self.docker_env(), + timeout=timeout, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout or "geth attach failed") + matches = re.findall(r"\b\d+\b", result.stdout or "") + if not matches: + raise RuntimeError("could not parse eth.blockNumber from {}".format(result.stdout)) + return int(matches[-1]) + + def load_compose_file(self) -> Dict[str, object]: + try: + import yaml + except ImportError as exc: + raise TestRunnerError("PyYAML is required to read Docker Compose files") from exc + with self.compose_file().open("r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +class EthereumRuntimeTest(ComposeRuntimeTest): + """Runtime helpers for Ethereum examples. + + These helpers intentionally execute from inside generated containers. That + keeps tests independent from host port publishing and from host-side web3 + packages. + """ + + def ethereum_services( + self, + *, + class_contains: Optional[str] = None, + display_contains: Optional[str] = None, + role: Optional[str] = None, + consensus: Optional[str] = None, + ) -> List[ComposeService]: + return _matching_ethereum_services( + self.compose, + class_contains=class_contains, + display_contains=display_contains, + role=role, + consensus=consensus, + ) + + def require_ethereum_services( + self, + name: str, + *, + expected_count: Optional[int] = None, + minimum_count: int = 1, + class_contains: Optional[str] = None, + display_contains: Optional[str] = None, + role: Optional[str] = None, + consensus: Optional[str] = None, + ) -> List[ComposeService]: + services = self.ethereum_services( + class_contains=class_contains, + display_contains=display_contains, + role=role, + consensus=consensus, + ) + count = len(services) + if expected_count is None: + passed = count >= minimum_count + expectation = "at least {}".format(minimum_count) + else: + passed = count == expected_count + expectation = "exactly {}".format(expected_count) + self.structural_check( + name, + passed, + "found {} service(s), expected {}; services={}".format( + count, + expectation, + ", ".join(service.name for service in services) or "", + ), + ) + return services + + def geth_eval(self, service: ComposeService | str, expression: str, timeout: int = 60) -> Dict[str, object]: + return self.exec(service, "geth attach --exec {}".format(shlex.quote(expression)), timeout=timeout) + + def ethereum_block_number(self, service: ComposeService | str, timeout: int = 60) -> int: + result = self.geth_eval(service, "eth.blockNumber", timeout=timeout) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "geth attach failed") + matches = re.findall(r"\b\d+\b", str(result["stdout"])) + if not matches: + raise RuntimeError("could not parse eth.blockNumber from {}".format(result["stdout"])) + return int(matches[-1]) + + def wait_for_ethereum_block_progress( + self, + name: str, + service: ComposeService | str, + *, + min_delta: int = 1, + retries: int = 60, + interval: int = 5, + timeout: int = 60, + ) -> Dict[str, object]: + start_block: Optional[int] = None + last_block: Optional[int] = None + error = "" + for attempt in range(1, retries + 1): + try: + block = self.ethereum_block_number(service, timeout=timeout) + except Exception as exc: + error = str(exc) + else: + if start_block is None: + start_block = block + last_block = block + if block >= start_block + min_delta: + return self._record_runtime_result( + name, + service, + "wait for eth.blockNumber to advance", + 0, + "start_block={} latest_block={} attempts={}".format(start_block, block, attempt), + "", + ) + if attempt < retries: + time.sleep(interval) + + return self._record_runtime_result( + name, + service, + "wait for eth.blockNumber to advance", + 1, + "start_block={} latest_block={}".format(start_block, last_block), + error, + ) + + def send_ether_and_verify( + self, + name: str, + service: ComposeService | str, + *, + to_address: str = DEFAULT_TRANSFER_RECIPIENT, + value_wei: int = 10**18, + password: str = "admin", + receipt_retries: int = 90, + timeout: int = 180, + ) -> Dict[str, object]: + try: + script = self._signed_raw_transfer_script( + service, + to_address=to_address, + value_wei=value_wei, + password=password, + receipt_retries=receipt_retries, + ) + except Exception as exc: + return self._record_runtime_result(name, service, "prepare signed Ethereum transaction", 1, "", str(exc)) + + result = self.geth_eval(service, script, timeout=timeout) + stdout = str(result["stdout"]) + stderr = str(result["stderr"]) + if result["exit"] != 0: + return self._record_runtime_result(name, service, "send Ethereum transaction", 1, stdout, stderr) + + try: + data = self._extract_json(stdout) + self._assert_transfer(data, value_wei) + except Exception as exc: + return self._record_runtime_result(name, service, "send Ethereum transaction", 1, stdout, str(exc)) + + summary = { + "sender": data["sender"], + "recipient": data["recipient"], + "tx_hash": data["txHash"], + "block_number": data["receiptBlockNumber"], + "sender_delta_wei": data["senderDelta"], + "recipient_delta_wei": data["recipientDelta"], + "value_wei": str(value_wei), + } + return self._record_runtime_result( + name, + service, + "send Ethereum transaction", + 0, + json.dumps(summary, sort_keys=True), + "", + ) + + def _record_runtime_result( + self, + name: str, + service: ComposeService | str, + command: str, + exit_code: int, + stdout: str, + stderr: str, + ) -> Dict[str, object]: + service_name = service.name if isinstance(service, ComposeService) else str(service) + result = { + "name": name, + "service": service_name, + "command": command, + "exit": exit_code, + "stdout": stdout[-1000:], + "stderr": stderr[-1000:], + "status": "passed" if exit_code == 0 else "failed", + } + self.results.append(result) + return result + + @staticmethod + def _extract_json(text: str) -> Dict[str, Any]: + for line in reversed(text.splitlines()): + line = line.strip() + if not line: + continue + candidates = [line] + if line.startswith('"') and line.endswith('"'): + try: + decoded = json.loads(line) + except json.JSONDecodeError: + decoded = None + if isinstance(decoded, str): + candidates.insert(0, decoded) + for candidate in candidates: + candidate = candidate.strip() + if candidate.startswith("{") and candidate.endswith("}"): + data = json.loads(candidate) + if isinstance(data, dict): + return data + raise ValueError("could not find JSON object in geth output") + + @staticmethod + def _assert_transfer(data: Dict[str, Any], value_wei: int) -> None: + if not data.get("txHash"): + raise ValueError("missing transaction hash") + if data.get("receiptBlockNumber") in (None, ""): + raise ValueError("transaction was not included in a block") + status = data.get("receiptStatus") + if status not in ("0x1", "1", 1, True, None): + raise ValueError("transaction receipt status is {}".format(status)) + recipient_delta = int(data["recipientDelta"]) + sender_delta = int(data["senderDelta"]) + if recipient_delta != value_wei: + raise ValueError("recipient delta {} != {}".format(recipient_delta, value_wei)) + if sender_delta < value_wei: + raise ValueError("sender delta {} is smaller than transfer value {}".format(sender_delta, value_wei)) + + def _signed_raw_transfer_script( + self, + service: ComposeService | str, + *, + to_address: str, + value_wei: int, + password: str, + receipt_retries: int, + ) -> str: + try: + from eth_account import Account + except ImportError as exc: + raise RuntimeError("eth_account is required to sign Ethereum runtime test transactions") from exc + + state = self._local_account_transaction_state(service) + sender = str(state["sender"]) + chain_id = self._chain_id_for_service(service) + private_key = self._private_key_for_local_account(service, sender, password, chain_id=chain_id) + account = Account.from_key(private_key) + if account.address.lower() != sender.lower(): + raise RuntimeError("decrypted key {} does not match local geth account {}".format(account.address, sender)) + + transaction = { + "chainId": chain_id, + "nonce": self._rpc_int(state["nonce"]), + "gas": 21000, + "gasPrice": self._rpc_int(state["gasPrice"]), + "to": to_address, + "value": int(value_wei), + "data": b"", + } + signed = Account.sign_transaction(transaction, private_key) + raw_transaction = getattr(signed, "rawTransaction", None) + if raw_transaction is None: + raw_transaction = getattr(signed, "raw_transaction") + raw_transaction_hex = raw_transaction.hex() + if not raw_transaction_hex.startswith("0x"): + raw_transaction_hex = "0x" + raw_transaction_hex + + return self._send_raw_transfer_script(sender, to_address, raw_transaction_hex, receipt_retries) + + def _local_account_transaction_state(self, service: ComposeService | str) -> Dict[str, Any]: + result = self.geth_eval(service, self._transaction_state_script()) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "failed to inspect local geth account") + return self._extract_json(str(result["stdout"])) + + def _private_key_for_local_account( + self, + service: ComposeService | str, + sender: str, + password: str, + *, + chain_id: Optional[int] = None, + ) -> str: + try: + from eth_account import Account + except ImportError as exc: + raise RuntimeError("eth_account is required to decrypt geth keystore files") from exc + + cached_private_key = self._load_cached_local_private_key(sender, chain_id) + if cached_private_key: + try: + cached_account = Account.from_key(cached_private_key) + except Exception: + cached_account = None + if cached_account is not None and cached_account.address.lower() == sender.lower(): + return cached_private_key + + passwords = [password] + password_result = self.exec(service, "cat /tmp/eth-password 2>/dev/null || true", timeout=30) + for item in str(password_result["stdout"]).splitlines(): + candidate = item.strip() + if candidate and candidate not in passwords: + passwords.append(candidate) + + paths = self._local_keystore_paths(service) + errors: List[str] = [] + for path_item in paths: + result = self.exec(service, "cat {}".format(shlex.quote(path_item)), timeout=30) + if result["exit"] != 0: + errors.append("{}: {}".format(path_item, result["stderr"] or result["stdout"])) + continue + try: + keyfile = json.loads(str(result["stdout"])) + except json.JSONDecodeError as exc: + errors.append("{}: invalid keystore JSON: {}".format(path_item, exc)) + continue + for candidate_password in passwords: + try: + key = Account.decrypt(keyfile, candidate_password) + private_key = key.hex() + if not private_key.startswith("0x"): + private_key = "0x" + private_key + account = Account.from_key(private_key) + except Exception as exc: + errors.append("{}: decrypt failed: {}".format(path_item, exc)) + continue + if account.address.lower() == sender.lower(): + self._save_cached_local_private_key(sender, chain_id, private_key) + return private_key + + detail = "; ".join(errors[-3:]) if errors else "no readable keystore files" + raise RuntimeError("could not decrypt local geth account {}: {}".format(sender, detail)) + + def _local_account_cache_path(self) -> Optional[Path]: + if self.artifact_dir is None: + return None + return self.artifact_dir / "ethereum-local-account-cache.json" + + @staticmethod + def _local_account_cache_key(sender: str, chain_id: Optional[int]) -> str: + return "{}:{}".format(chain_id if chain_id is not None else "unknown", sender.lower()) + + def _load_cached_local_private_key(self, sender: str, chain_id: Optional[int]) -> Optional[str]: + cache_path = self._local_account_cache_path() + if cache_path is None or not cache_path.is_file(): + return None + try: + data = json.loads(cache_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(data, dict) or data.get("version") != LOCAL_ACCOUNT_CACHE_VERSION: + return None + accounts = data.get("accounts") + if not isinstance(accounts, dict): + return None + entry = accounts.get(self._local_account_cache_key(sender, chain_id)) + if not isinstance(entry, dict): + return None + private_key = entry.get("private_key") + if not isinstance(private_key, str) or not private_key: + return None + if not private_key.startswith("0x"): + private_key = "0x" + private_key + return private_key + + def _save_cached_local_private_key(self, sender: str, chain_id: Optional[int], private_key: str) -> None: + cache_path = self._local_account_cache_path() + if cache_path is None: + return + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + if cache_path.is_file(): + data = json.loads(cache_path.read_text(encoding="utf-8")) + else: + data = {} + except (OSError, json.JSONDecodeError): + data = {} + if not isinstance(data, dict) or data.get("version") != LOCAL_ACCOUNT_CACHE_VERSION: + data = {"version": LOCAL_ACCOUNT_CACHE_VERSION, "accounts": {}} + accounts = data.setdefault("accounts", {}) + if not isinstance(accounts, dict): + accounts = {} + data["accounts"] = accounts + accounts[self._local_account_cache_key(sender, chain_id)] = { + "sender": sender, + "chain_id": chain_id, + "private_key": private_key, + } + tmp_path = cache_path.with_name(cache_path.name + ".tmp") + try: + tmp_path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + tmp_path.chmod(0o600) + tmp_path.replace(cache_path) + cache_path.chmod(0o600) + except OSError: + return + + def _local_keystore_paths(self, service: ComposeService | str) -> List[str]: + command = ( + 'for dir in /root/.ethereum/keystore /tmp/keystore; do ' + '[ -d "$dir" ] && find "$dir" -maxdepth 1 -type f -print; ' + "done | sort -u" + ) + result = self.exec(service, command, timeout=30) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "failed to list geth keystore files") + paths = [line.strip() for line in str(result["stdout"]).splitlines() if line.strip()] + if not paths: + raise RuntimeError("no geth keystore files found in local account directories") + return paths + + def _chain_id_for_service(self, service: ComposeService | str) -> int: + if isinstance(service, ComposeService): + chain_id = service.labels.get(ETH_CHAIN_ID_LABEL) + if chain_id is not None: + return self._rpc_int(chain_id) + + result = self.geth_eval( + service, + "JSON.stringify({chainId: admin.nodeInfo.protocols.eth.config.chainId})", + timeout=30, + ) + if result["exit"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "failed to inspect Ethereum chain id") + return self._rpc_int(self._extract_json(str(result["stdout"]))["chainId"]) + + @staticmethod + def _rpc_int(value: object) -> int: + if isinstance(value, int): + return value + text = str(value).strip() + if text.startswith(("0x", "0X")): + return int(text, 16) + return int(text) + + @staticmethod + def _transaction_state_script() -> str: + return """ +var sender = eth.accounts[0]; +if (!sender) { throw new Error("no local geth account"); } +var nonce = eth.getTransactionCount(sender); +var gasPrice = eth.gasPrice; +JSON.stringify({ + sender: sender, + nonce: nonce && nonce.toString ? nonce.toString(10) : String(nonce), + gasPrice: gasPrice && gasPrice.toString ? gasPrice.toString(10) : String(gasPrice) +}) +""".strip() + + @staticmethod + def _send_raw_transfer_script(sender: str, to_address: str, raw_transaction: str, receipt_retries: int) -> str: + script = """ +var sender = __SENDER__; +var recipient = __RECIPIENT__; +var senderBefore = eth.getBalance(sender); +var recipientBefore = eth.getBalance(recipient); +var txHash = eth.sendRawTransaction(__RAW_TRANSACTION__); +var receipt = null; +for (var i = 0; i < __RECEIPT_RETRIES__; i++) { + receipt = eth.getTransactionReceipt(txHash); + if (receipt !== null && receipt.blockNumber !== null) { break; } + admin.sleep(1); +} +var senderAfter = eth.getBalance(sender); +var recipientAfter = eth.getBalance(recipient); +JSON.stringify({ + sender: sender, + recipient: recipient, + txHash: txHash, + receiptBlockNumber: receipt === null ? null : receipt.blockNumber.toString(10), + receiptStatus: receipt === null || receipt.status === null ? null : receipt.status.toString(10), + senderBefore: senderBefore.toString(10), + senderAfter: senderAfter.toString(10), + recipientBefore: recipientBefore.toString(10), + recipientAfter: recipientAfter.toString(10), + senderDelta: senderBefore.minus(senderAfter).toString(10), + recipientDelta: recipientAfter.minus(recipientBefore).toString(10) +}) +""".strip() + script = script.replace("__SENDER__", json.dumps(sender)) + script = script.replace("__RECIPIENT__", json.dumps(to_address)) + script = script.replace("__RAW_TRANSACTION__", json.dumps(raw_transaction)) + script = script.replace("__RECEIPT_RETRIES__", str(receipt_retries)) + return script diff --git a/seedemu/testing/cli.py b/seedemu/testing/cli.py new file mode 100644 index 000000000..e2c551a52 --- /dev/null +++ b/seedemu/testing/cli.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import sys +from typing import Optional, Sequence + +if __package__: + from .registry import create_runner +else: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from registry import create_runner + + +COMMANDS = ["clean", "compile", "build", "up", "readiness", "probe", "test", "down", "all"] + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Run a standardized SEED Emulator test manifest.") + parser.add_argument("command", choices=COMMANDS) + parser.add_argument("manifest", type=Path, help="Path to a test manifest YAML file.") + parser.add_argument("--artifact-dir", type=Path, help="Optional directory for command logs and summaries.") + parser.add_argument( + "--runner", + choices=["generic", "internet", "satellite", "blockchain"], + help="Runner type. Defaults to the manifest's runner field, then generic.", + ) + args = parser.parse_args(argv) + + os.environ["TEST_RUNNER_COMMAND"] = args.command + runner = create_runner(args.manifest, artifact_dir=args.artifact_dir, runner=args.runner) + return getattr(runner, args.command)() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/seedemu/testing/compose.py b/seedemu/testing/compose.py new file mode 100644 index 000000000..5e96f2648 --- /dev/null +++ b/seedemu/testing/compose.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import shutil +import subprocess +from typing import List + + +def docker_compose_command() -> List[str]: + docker = shutil.which("docker") + if docker is not None: + result = subprocess.run( + [docker, "compose", "version"], + text=True, + capture_output=True, + check=False, + ) + if result.returncode == 0: + return [docker, "compose"] + + docker_compose = shutil.which("docker-compose") + if docker_compose is not None: + return [docker_compose] + + return ["docker", "compose"] diff --git a/seedemu/testing/internet.py b/seedemu/testing/internet.py new file mode 100644 index 000000000..ddfab5013 --- /dev/null +++ b/seedemu/testing/internet.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +try: + from .base import TestRunner +except ImportError: + from base import TestRunner + + +class InternetTestRunner(TestRunner): + """Runner for Internet-style SEED Emulator tests.""" + + runner_name = "internet" + probe_handlers = { + **TestRunner.probe_handlers, + "ping": "probe_ping", + "dns": "probe_dns", + "bgp-route": "probe_bgp_route", + "mpls-config": "probe_mpls_config", + } + + def probe_ping(self, probe: Dict[str, Any]) -> tuple[bool, str]: + command = "ping -c {} {}".format(int(probe.get("count", 3)), probe["target"]) + exec_probe = dict(probe) + exec_probe["type"] = "exec" + exec_probe["command"] = command + exec_probe.setdefault("expect_exit", 0) + return self.probe_exec(exec_probe) + + def probe_dns(self, probe: Dict[str, Any]) -> tuple[bool, str]: + query = str(probe["query"]) + record_type = str(probe.get("record_type", "A")) + command = ( + "getent hosts {}".format(query) + if record_type in {"A", "AAAA"} + else "nslookup -type={} {}".format(record_type, query) + ) + if probe.get("server"): + command = "nslookup -type={} {} {}".format(record_type, query, probe["server"]) + + if probe.get("service"): + exec_probe = dict(probe) + exec_probe["type"] = "exec" + exec_probe["command"] = command + exec_probe.setdefault("expect_exit", 0) + return self.probe_exec(exec_probe) + + result = self.run_command( + "probe-{}".format(self.slug(probe["name"])), + ["sh", "-lc", command], + cwd=self.emulation_dir, + timeout=int(probe.get("timeout", 30)), + ) + if result.returncode != 0: + return False, "DNS command exited {}".format(result.returncode) + return self.check_text_expectations(probe, result.stdout or "", result.stderr or "") + + def probe_bgp_route(self, probe: Dict[str, Any]) -> tuple[bool, str]: + command = str(probe.get("command", "birdc show route {}".format(probe["prefix"]))) + exec_probe = dict(probe) + exec_probe["type"] = "exec" + exec_probe["command"] = command + exec_probe.setdefault("expect_exit", 0) + return self.probe_exec(exec_probe) + + def probe_mpls_config(self, probe: Dict[str, Any]) -> tuple[bool, str]: + interfaces = [str(item) for item in probe.get("interfaces", [])] + checks = ["test -s /mpls_ifaces.txt", "grep -q 'mpls ldp' /etc/frr/frr.conf"] + checks.extend("grep -q '^{}$' /mpls_ifaces.txt".format(re.escape(interface)) for interface in interfaces) + exec_probe = dict(probe) + exec_probe["type"] = "exec" + exec_probe["command"] = " && ".join(checks) + exec_probe.setdefault("expect_exit", 0) + return self.probe_exec(exec_probe) + + def required_probe_fields(self, probe_type: str) -> Optional[List[str]]: + fields = { + "ping": ["service", "target"], + "dns": ["query"], + "bgp-route": ["service", "prefix"], + "mpls-config": ["service"], + }.get(probe_type) + return fields if fields is not None else super().required_probe_fields(probe_type) diff --git a/seedemu/testing/registry.py b/seedemu/testing/registry.py new file mode 100644 index 000000000..f79e32726 --- /dev/null +++ b/seedemu/testing/registry.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +try: + from .base import TestRunner, TestRunnerError + from .blockchain import BlockchainTestRunner + from .internet import InternetTestRunner + from .satellite import SatelliteTestRunner +except ImportError: + from base import TestRunner, TestRunnerError + from blockchain import BlockchainTestRunner + from internet import InternetTestRunner + from satellite import SatelliteTestRunner + + +RUNNER_REGISTRY = { + "generic": TestRunner, + "test": TestRunner, + "internet": InternetTestRunner, + "satellite": SatelliteTestRunner, + "blockchain": BlockchainTestRunner, +} + + +def create_runner( + manifest_path: Path, + artifact_dir: Optional[Path] = None, + runner: Optional[str] = None, +) -> TestRunner: + """Create a runner from an explicit name or the manifest's runner field.""" + + runner_name = runner + if runner_name is None: + runner_name = read_runner_name(manifest_path) + runner_cls = RUNNER_REGISTRY.get(str(runner_name)) + if runner_cls is None: + raise TestRunnerError("unknown runner: {}".format(runner_name)) + return runner_cls(manifest_path, artifact_dir=artifact_dir) + + +def read_runner_name(manifest_path: Path) -> str: + try: + import yaml + except ImportError: + return "generic" + + path = manifest_path.resolve() + if not path.is_file(): + return "generic" + data = yaml.safe_load(path.read_text(encoding="utf-8")) + if isinstance(data, dict): + return str(data.get("runner", "generic")) + return "generic" diff --git a/seedemu/testing/runtime.py b/seedemu/testing/runtime.py new file mode 100644 index 000000000..cdf68d2c7 --- /dev/null +++ b/seedemu/testing/runtime.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import os +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +import yaml + +try: + from .compose import docker_compose_command +except ImportError: + from compose import docker_compose_command + + +ASN_LABEL = "org.seedsecuritylabs.seedemu.meta.asn" +NODE_LABEL = "org.seedsecuritylabs.seedemu.meta.nodename" +ADDRESS_LABEL = "org.seedsecuritylabs.seedemu.meta.net.0.address" + + +@dataclass(frozen=True) +class ComposeService: + """A generated Docker Compose service discovered from SEED Emulator labels.""" + + name: str + address: str + labels: Dict[str, object] + + +class ComposeRuntimeTest: + """Helper for custom runtime tests launched by TestRunner. + + The lifecycle runner already knows how to compile, build, start, and stop an + emulation. This helper is for example-specific test programs that need to + inspect generated Compose metadata and run commands inside containers. + """ + + def __init__(self, test_file: str | Path): + self.test_file = Path(test_file).resolve() + self.example_dir = Path( + os.environ.get("TEST_RUNNER_EMULATION_DIR") + or os.environ.get("EXAMPLE_RUNNER_EXAMPLE_DIR") + or self.test_file.parent + ).resolve() + self.compose_file = Path( + os.environ.get("TEST_RUNNER_COMPOSE_FILE") + or os.environ.get("EXAMPLE_RUNNER_COMPOSE_FILE") + or self.example_dir / "output" / "docker-compose.yml" + ).resolve() + artifact_dir = os.environ.get("TEST_RUNNER_ARTIFACT_DIR") or os.environ.get("EXAMPLE_RUNNER_ARTIFACT_DIR") + self.artifact_dir = Path(artifact_dir).resolve() if artifact_dir else None + self.compose = self.load_compose() + self.results: List[Dict[str, object]] = [] + + def load_compose(self) -> Dict[str, object]: + with self.compose_file.open("r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + def find_service(self, asn: int, node: str) -> Optional[ComposeService]: + """Find a generated service by stable SEED Emulator metadata labels.""" + + for name, service in self.compose.get("services", {}).items(): + labels = service.get("labels", {}) + if str(labels.get(ASN_LABEL)) == str(asn) and labels.get(NODE_LABEL) == node: + address = str(labels.get(ADDRESS_LABEL, "")).split("/", 1)[0] + return ComposeService(name=str(name), address=address, labels=dict(labels)) + return None + + def require_service(self, asn: int, node: str, description: Optional[str] = None) -> Optional[ComposeService]: + """Record a structural check and return the discovered service, if any.""" + + service = self.find_service(asn, node) + label = description or "AS{} {} is generated".format(asn, node) + if service is None: + self.structural_check(label, False, "service for AS{} node {} not found".format(asn, node)) + else: + self.structural_check(label, True, "found {}".format(service.name)) + return service + + def exec(self, service: ComposeService | str, command: str, timeout: int = 45) -> Dict[str, object]: + service_name = service.name if isinstance(service, ComposeService) else str(service) + result = subprocess.run( + docker_compose_command() + + ["-f", str(self.compose_file), "exec", "-T", service_name, "sh", "-lc", command], + cwd=str(self.compose_file.parent), + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + return { + "service": service_name, + "command": command, + "exit": result.returncode, + "stdout": result.stdout[-1000:], + "stderr": result.stderr[-1000:], + } + + def exec_check( + self, + name: str, + service: ComposeService | str, + command: str, + retries: int = 20, + interval: int = 3, + timeout: int = 45, + ) -> Dict[str, object]: + """Run a command with retries and append the check result.""" + + result: Dict[str, object] = {} + for attempt in range(1, retries + 1): + result = self.exec(service, command, timeout=timeout) + if result["exit"] == 0: + break + if attempt < retries: + time.sleep(interval) + result["name"] = name + result["attempts"] = attempt + result["status"] = "passed" if result["exit"] == 0 else "failed" + self.results.append(result) + return result + + def structural_check(self, name: str, passed: bool, message: str) -> Dict[str, object]: + result = { + "name": name, + "service": "", + "command": "inspect generated docker-compose.yml", + "exit": 0 if passed else 1, + "stdout": message, + "stderr": "", + "status": "passed" if passed else "failed", + } + self.results.append(result) + return result + + def summary(self) -> Dict[str, object]: + return { + "compose_file": str(self.compose_file), + "results": self.results, + "failures": [item["name"] for item in self.results if item["status"] == "failed"], + } + + def write_summary(self, filename: str) -> Dict[str, object]: + summary = self.summary() + print(json.dumps(summary, indent=2, sort_keys=True)) + if self.artifact_dir: + path = self.artifact_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + def exit_code(self) -> int: + return 1 if self.summary()["failures"] else 0 diff --git a/seedemu/testing/satellite.py b/seedemu/testing/satellite.py new file mode 100644 index 000000000..e1c9324bb --- /dev/null +++ b/seedemu/testing/satellite.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +try: + from .base import TestRunner +except ImportError: + from base import TestRunner + + +class SatelliteTestRunner(TestRunner): + """Extension point for future satellite emulation tests.""" + + runner_name = "satellite" diff --git a/seedemu/utilities/AutonomousSystemLocation.py b/seedemu/utilities/AutonomousSystemLocation.py new file mode 100644 index 000000000..be06d3d6d --- /dev/null +++ b/seedemu/utilities/AutonomousSystemLocation.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import random +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +try: + import networkx as nx +except ImportError: # pragma: no cover - exercised only when dependency is absent. + nx = None + + +Location = Dict[str, Any] +FixedLocationInput = Dict[str, Any] + + +class LocationGenerationError(Exception): + """Raised when router locations cannot be generated for an AS topology.""" + + +def _require_networkx(): + if nx is None: + raise LocationGenerationError("Autonomous-system location generation requires networkx") + return nx + + +class AutonomousSystemLocationGenerator: + """Generate router locations for an existing autonomous-system topology. + + This generator is intentionally separate from AutonomousSystemTopologyGenerator. + The topology generator decides which routers and links exist; this class only + assigns geographic coordinates after that topology has already been generated. + + Known router locations are fixed. Missing router locations are initialized in + the bounding area of the fixed routers, then adjusted with NetworkX + spring_layout while keeping the known locations unchanged. + + Fixed locations may use either: + - {"lat": 40.7128, "lon": -74.0060} + - {"latitude": 40.7128, "longitude": -74.0060} + - {"y": 40.7128, "x": -74.0060} + - (lat, lon) + """ + + def __init__( + self, + fixed_locations: Optional[Dict[str, FixedLocationInput]] = None, + seed: Optional[int] = None, + iterations: int = 100, + default_span: float = 1.0, + padding_ratio: float = 0.15, + k: Optional[float] = None, + weight: Optional[str] = "weight", + ): + self.fixed_locations = dict(fixed_locations or {}) + self.seed = seed + self.iterations = int(iterations) + self.default_span = float(default_span) + self.padding_ratio = float(padding_ratio) + self.k = k + self.weight = weight + + def generate( + self, + topology_or_graph: Any, + fixed_locations: Optional[Dict[str, FixedLocationInput]] = None, + ) -> Dict[str, Location]: + graph = self._get_graph(topology_or_graph) + routers = sorted(str(node) for node in graph.nodes) + if not routers: + raise LocationGenerationError("cannot generate locations for an empty topology") + + merged_fixed = dict(self.fixed_locations) + if fixed_locations: + merged_fixed.update(fixed_locations) + + fixed_positions = self._normalize_fixed_locations(merged_fixed, routers) + if fixed_positions: + initial_positions = self._initial_positions(routers, fixed_positions) + layout = _require_networkx().spring_layout( + graph, + pos=initial_positions, + fixed=list(fixed_positions.keys()), + iterations=self.iterations, + seed=self.seed, + k=self.k, + weight=self.weight, + ) + else: + layout = _require_networkx().spring_layout( + graph, + iterations=self.iterations, + seed=self.seed, + k=self.k, + weight=self.weight, + ) + + return self._format_locations(routers, layout, fixed_positions) + + def _get_graph(self, topology_or_graph: Any): + nx_mod = _require_networkx() + if hasattr(topology_or_graph, "graph") and callable(topology_or_graph.graph): + return topology_or_graph.graph() + return nx_mod.Graph(topology_or_graph) + + def _normalize_fixed_locations( + self, + fixed_locations: Dict[str, FixedLocationInput], + routers: Iterable[str], + ) -> Dict[str, Tuple[float, float]]: + router_set = set(routers) + positions: Dict[str, Tuple[float, float]] = {} + + for router, location in fixed_locations.items(): + router_name = str(router) + if router_name not in router_set: + raise LocationGenerationError( + "fixed location references unknown router: {}".format(router_name) + ) + lat, lon = self._parse_location(location) + positions[router_name] = (lon, lat) + + return positions + + def _parse_location(self, location: FixedLocationInput) -> Tuple[float, float]: + if isinstance(location, dict): + if "lat" in location and "lon" in location: + return float(location["lat"]), float(location["lon"]) + if "latitude" in location and "longitude" in location: + return float(location["latitude"]), float(location["longitude"]) + if "y" in location and "x" in location: + return float(location["y"]), float(location["x"]) + raise LocationGenerationError( + "fixed location dictionaries must contain lat/lon, latitude/longitude, or x/y" + ) + + if isinstance(location, (list, tuple)) and len(location) == 2: + return float(location[0]), float(location[1]) + + raise LocationGenerationError( + "fixed locations must be dictionaries or two-item (lat, lon) sequences" + ) + + def _initial_positions( + self, + routers: List[str], + fixed_positions: Dict[str, Tuple[float, float]], + ) -> Dict[str, Tuple[float, float]]: + rng = random.Random(self.seed) + min_lon, max_lon, min_lat, max_lat = self._bounds(fixed_positions.values()) + positions = dict(fixed_positions) + + for router in routers: + if router in positions: + continue + positions[router] = ( + rng.uniform(min_lon, max_lon), + rng.uniform(min_lat, max_lat), + ) + + return positions + + def _bounds(self, positions: Iterable[Tuple[float, float]]) -> Tuple[float, float, float, float]: + lon_values = [position[0] for position in positions] + lat_values = [position[1] for position in positions] + min_lon, max_lon = min(lon_values), max(lon_values) + min_lat, max_lat = min(lat_values), max(lat_values) + + lon_span = max(max_lon - min_lon, self.default_span) + lat_span = max(max_lat - min_lat, self.default_span) + lon_padding = lon_span * self.padding_ratio + lat_padding = lat_span * self.padding_ratio + + return ( + min_lon - lon_padding, + max_lon + lon_padding, + min_lat - lat_padding, + max_lat + lat_padding, + ) + + def _format_locations( + self, + routers: List[str], + layout: Dict[str, Tuple[float, float]], + fixed_positions: Dict[str, Tuple[float, float]], + ) -> Dict[str, Location]: + locations: Dict[str, Location] = {} + fixed_router_names = set(fixed_positions.keys()) + + for router in routers: + lon, lat = layout[router] + locations[router] = { + "lat": float(lat), + "lon": float(lon), + "source": "fixed" if router in fixed_router_names else "generated", + } + + return locations diff --git a/seedemu/utilities/AutonomousSystemTopology.py b/seedemu/utilities/AutonomousSystemTopology.py new file mode 100644 index 000000000..15a76b101 --- /dev/null +++ b/seedemu/utilities/AutonomousSystemTopology.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import random +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + + +try: + import networkx as nx +except ImportError: # pragma: no cover - exercised only when dependency is absent. + nx = None + + +GraphModel = Union[str, Callable[..., Any]] + + +class TopologyGenerationError(Exception): + """Raised when an autonomous-system topology cannot be generated or validated.""" + + +def _require_networkx(): + if nx is None: + raise TopologyGenerationError("Autonomous-system topology generation requires networkx") + return nx + + +def _call_graph_model(model: GraphModel, n: int, params: Dict[str, Any], seed: Optional[int]): + nx_mod = _require_networkx() + params = dict(params or {}) + + if callable(model): + return model(n=n, seed=seed, **params) + + model_name = str(model).strip().lower() + registry: Dict[str, Callable[[], Any]] = { + "cycle": lambda: nx_mod.cycle_graph(n), + "path": lambda: nx_mod.path_graph(n), + "complete": lambda: nx_mod.complete_graph(n), + "random": lambda: nx_mod.erdos_renyi_graph(n=n, seed=seed, **params), + "erdos_renyi": lambda: nx_mod.erdos_renyi_graph(n=n, seed=seed, **params), + "gnp_random": lambda: nx_mod.gnp_random_graph(n=n, seed=seed, **params), + "regular": lambda: nx_mod.random_regular_graph(n=n, seed=seed, **params), + "random_regular": lambda: nx_mod.random_regular_graph(n=n, seed=seed, **params), + "small_world": lambda: nx_mod.connected_watts_strogatz_graph(n=n, seed=seed, **params), + "watts_strogatz": lambda: nx_mod.watts_strogatz_graph(n=n, seed=seed, **params), + "connected_watts_strogatz": lambda: nx_mod.connected_watts_strogatz_graph(n=n, seed=seed, **params), + "scale_free": lambda: nx_mod.barabasi_albert_graph(n=n, seed=seed, **params), + "barabasi_albert": lambda: nx_mod.barabasi_albert_graph(n=n, seed=seed, **params), + "powerlaw_cluster": lambda: nx_mod.powerlaw_cluster_graph(n=n, seed=seed, **params), + } + + if model_name not in registry: + raise TopologyGenerationError("unsupported NetworkX graph model: {}".format(model)) + + try: + return registry[model_name]() + except TypeError as exc: + raise TopologyGenerationError("invalid parameters for graph model {}: {}".format(model, exc)) from exc + + +class AutonomousSystemTopology: + """A generated autonomous-system topology independent from any ASN or IX mapping.""" + + def __init__( + self, + graph: Any, + ebgp_routers: List[str], + ebgp_to_internal: Dict[str, str], + seed: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + ): + _require_networkx() + self._graph = graph.copy() + self._ebgp_routers = [str(router) for router in ebgp_routers] + self._ebgp_to_internal = {str(router): str(internal) for router, internal in ebgp_to_internal.items()} + self.seed = seed + self.metadata = dict(metadata or {}) + + @classmethod + def from_networkx( + cls, + internal_graph: Any, + ebgp_router_count: int, + ebgp_attach_policy: str = "spread", + ebgp_to_internal: Optional[Dict[str, str]] = None, + ebgp_router_prefix: str = "r", + seed: Optional[int] = None, + ) -> "AutonomousSystemTopology": + generator = AutonomousSystemTopologyGenerator( + ebgp_router_count=ebgp_router_count, + internal_graph=internal_graph, + ebgp_attach_policy=ebgp_attach_policy, + ebgp_to_internal=ebgp_to_internal, + ebgp_router_prefix=ebgp_router_prefix, + seed=seed, + ) + return generator.generate() + + def graph(self): + return self._graph.copy() + + def to_networkx(self): + return self.graph() + + def internal_graph(self): + nx_mod = _require_networkx() + return self._graph.subgraph(self.internal_routers()).copy() + + def routers(self) -> List[str]: + return sorted(str(node) for node in self._graph.nodes) + + def ebgp_routers(self) -> List[str]: + return list(self._ebgp_routers) + + def internal_routers(self) -> List[str]: + return sorted(str(node) for node, data in self._graph.nodes(data=True) if data.get("role") == "internal") + + def links(self) -> List[Tuple[str, str]]: + return sorted((str(a), str(b)) for a, b in self._graph.edges) + + def link_networks(self) -> List[Tuple[str, str, str]]: + return [(a, b, self.network_name_for_link(a, b)) for a, b in self.links()] + + def network_name_for_link(self, a: str, b: str) -> str: + return self._network_name(str(a), str(b)) + + def ebgp_to_internal(self) -> Dict[str, str]: + return dict(self._ebgp_to_internal) + + def validate(self) -> None: + nx_mod = _require_networkx() + + if len(self._graph.nodes) == 0: + raise TopologyGenerationError("autonomous-system topology has no routers") + + if not nx_mod.is_connected(self._graph): + raise TopologyGenerationError("autonomous-system topology is not connected") + + if list(nx_mod.selfloop_edges(self._graph)): + raise TopologyGenerationError("autonomous-system topology contains self loops") + + router_names = self.routers() + if len(router_names) != len(set(router_names)): + raise TopologyGenerationError("autonomous-system topology contains duplicate router names") + + internal = set(self.internal_routers()) + ebgp_routers = set(self.ebgp_routers()) + if not internal: + raise TopologyGenerationError("autonomous-system topology has no internal routers") + if not ebgp_routers: + raise TopologyGenerationError("autonomous-system topology has no eBGP routers") + + for ebgp_router in ebgp_routers: + if ebgp_router not in self._ebgp_to_internal: + raise TopologyGenerationError("eBGP router {} is not attached to an internal router".format(ebgp_router)) + internal_router = self._ebgp_to_internal[ebgp_router] + if internal_router not in internal: + raise TopologyGenerationError( + "eBGP router {} attaches to unknown internal router {}".format(ebgp_router, internal_router) + ) + if not self._graph.has_edge(ebgp_router, internal_router): + raise TopologyGenerationError( + "eBGP router {} is missing graph link to {}".format(ebgp_router, internal_router) + ) + + def to_dict(self) -> Dict[str, Any]: + return { + "seed": self.seed, + "metadata": dict(self.metadata), + "ebgp_routers": self.ebgp_routers(), + "internal_routers": self.internal_routers(), + "ebgp_to_internal": self.ebgp_to_internal(), + "links": [list(link) for link in self.links()], + "link_networks": [ + {"endpoints": [a, b], "network": network} + for a, b, network in self.link_networks() + ], + } + + def to_yaml(self, path: Union[str, Path]) -> None: + import yaml + + with open(path, "w", encoding="utf-8") as file: + yaml.safe_dump(self.to_dict(), file, sort_keys=False) + + def to_dot(self, path: Union[str, Path]) -> None: + nx_mod = _require_networkx() + nx_mod.drawing.nx_pydot.write_dot(self._graph, path) + + def summary(self) -> str: + return "{} eBGP routers, {} internal routers, {} links".format( + len(self.ebgp_routers()), + len(self.internal_routers()), + len(self.links()), + ) + + def _network_name(self, a: str, b: str) -> str: + left, right = sorted([a, b]) + name = "n_{}_{}".format(self._short_router_name(left), self._short_router_name(right)) + assert len(name) <= 15, "generated network name is too long for a Linux interface: {}".format(name) + return name + + def _short_router_name(self, name: str) -> str: + if name.startswith("core"): + return "c{}".format(name[4:]) + return name.replace("-", "_") + + +class AutonomousSystemTopologyGenerator: + """Generate autonomous-system topologies using NetworkX internal graph models.""" + + def __init__( + self, + ebgp_router_count: int, + internal_router_count: int = 4, + graph_model: GraphModel = "small_world", + graph_params: Optional[Dict[str, Any]] = None, + internal_graph: Any = None, + seed: Optional[int] = None, + ebgp_attach_policy: str = "spread", + ebgp_to_internal: Optional[Dict[str, str]] = None, + ebgp_router_prefix: str = "r", + internal_router_prefix: str = "core", + require_connected: bool = True, + max_attempts: int = 100, + ): + self.ebgp_router_count = int(ebgp_router_count) + self.internal_router_count = int(internal_router_count) + self.graph_model = graph_model + self.graph_params = dict(graph_params or self._default_graph_params(graph_model)) + self.internal_graph = internal_graph + self.seed = seed + self.ebgp_attach_policy = str(ebgp_attach_policy) + self.ebgp_to_internal = {str(router): str(internal) for router, internal in ebgp_to_internal.items()} if ebgp_to_internal else None + self.ebgp_router_prefix = str(ebgp_router_prefix) + self.internal_router_prefix = str(internal_router_prefix) + self.require_connected = bool(require_connected) + self.max_attempts = int(max_attempts) + + @classmethod + def from_graph( + cls, + graph: Any, + ebgp_router_count: int, + **kwargs, + ) -> "AutonomousSystemTopologyGenerator": + return cls(ebgp_router_count=ebgp_router_count, internal_graph=graph, **kwargs) + + def generate(self) -> AutonomousSystemTopology: + nx_mod = _require_networkx() + self._validate_inputs() + + ebgp_routers = self._build_ebgp_routers() + internal_graph = self._generate_internal_graph() + internal_graph = self._relabel_internal_graph(internal_graph) + internal_routers = sorted(str(node) for node in internal_graph.nodes) + ebgp_to_internal = self.ebgp_to_internal or self._select_ebgp_attachments(ebgp_routers, internal_graph) + + graph = nx_mod.Graph() + for router in internal_routers: + graph.add_node(router, role="internal") + for a, b in internal_graph.edges: + graph.add_edge(str(a), str(b)) + + for ebgp_router in ebgp_routers: + graph.add_node(ebgp_router, role="ebgp") + internal_router = ebgp_to_internal[ebgp_router] + graph.add_edge(ebgp_router, internal_router) + + topology = AutonomousSystemTopology( + graph=graph, + ebgp_routers=ebgp_routers, + ebgp_to_internal=ebgp_to_internal, + seed=self.seed, + metadata={ + "graph_model": self.graph_model if isinstance(self.graph_model, str) else getattr(self.graph_model, "__name__", "callable"), + "graph_params": dict(self.graph_params), + "ebgp_attach_policy": self.ebgp_attach_policy, + }, + ) + topology.validate() + return topology + + def _validate_inputs(self) -> None: + if self.ebgp_router_count <= 0: + raise TopologyGenerationError("ebgp_router_count must be positive") + if self.internal_graph is None and self.internal_router_count <= 0: + raise TopologyGenerationError("internal_router_count must be positive") + if self.max_attempts <= 0: + raise TopologyGenerationError("max_attempts must be positive") + + def _build_ebgp_routers(self) -> List[str]: + return ["{}{}".format(self.ebgp_router_prefix, index) for index in range(self.ebgp_router_count)] + + def _generate_internal_graph(self): + nx_mod = _require_networkx() + if self.internal_graph is not None: + graph = nx_mod.Graph(self.internal_graph) + if self.require_connected and not nx_mod.is_connected(graph): + raise TopologyGenerationError("provided internal graph is not connected") + return graph + + last_error = None + for attempt in range(self.max_attempts): + attempt_seed = None if self.seed is None else self.seed + attempt + try: + graph = nx_mod.Graph( + _call_graph_model(self.graph_model, self.internal_router_count, self.graph_params, attempt_seed) + ) + except TopologyGenerationError as exc: + last_error = exc + break + + if not self.require_connected or nx_mod.is_connected(graph): + return graph + + if last_error is not None: + raise last_error + raise TopologyGenerationError( + "could not generate a connected internal graph after {} attempts".format(self.max_attempts) + ) + + def _relabel_internal_graph(self, graph): + nx_mod = _require_networkx() + mapping = {} + for index, node in enumerate(sorted(graph.nodes, key=str)): + name = str(node) + if not name.startswith(self.internal_router_prefix): + name = "{}{}".format(self.internal_router_prefix, index) + mapping[node] = name + return nx_mod.relabel_nodes(graph, mapping, copy=True) + + def _select_ebgp_attachments(self, ebgp_routers: List[str], internal_graph) -> Dict[str, str]: + internal_routers = sorted(str(node) for node in internal_graph.nodes) + if not internal_routers: + raise TopologyGenerationError("cannot attach eBGP routers without internal routers") + + policy = self.ebgp_attach_policy.lower() + if policy == "manual": + raise TopologyGenerationError("ebgp_attach_policy=manual requires ebgp_to_internal") + + sorted_ebgp_routers = sorted(ebgp_routers) + if policy in {"spread", "round_robin"}: + return {router: internal_routers[index % len(internal_routers)] for index, router in enumerate(sorted_ebgp_routers)} + + if policy == "random": + rng = random.Random(self.seed) + return {router: rng.choice(internal_routers) for router in sorted_ebgp_routers} + + if policy == "degree": + ranked = sorted(internal_graph.degree, key=lambda item: (-item[1], str(item[0]))) + ranked_nodes = [str(node) for node, _degree in ranked] + return {router: ranked_nodes[index % len(ranked_nodes)] for index, router in enumerate(sorted_ebgp_routers)} + + raise TopologyGenerationError("unsupported eBGP attachment policy: {}".format(self.ebgp_attach_policy)) + + def _default_graph_params(self, graph_model: GraphModel) -> Dict[str, Any]: + if not isinstance(graph_model, str): + return {} + + model = graph_model.strip().lower() + if model in {"random", "erdos_renyi", "gnp_random"}: + return {"p": 0.4} + if model in {"regular", "random_regular"}: + return {"d": 2} + if model in {"small_world", "connected_watts_strogatz", "watts_strogatz"}: + return {"k": 2, "p": 0.25} + if model in {"scale_free", "barabasi_albert"}: + return {"m": 1} + if model == "powerlaw_cluster": + return {"m": 1, "p": 0.25} + return {} diff --git a/seedemu/utilities/__init__.py b/seedemu/utilities/__init__.py index 13a67efcc..40933538d 100644 --- a/seedemu/utilities/__init__.py +++ b/seedemu/utilities/__init__.py @@ -1,2 +1,4 @@ from .Makers import * from .BuildtimeDocker import BuildtimeDockerFile, BuildtimeDockerImage, BuildtimeDockerContainer +from .AutonomousSystemTopology import AutonomousSystemTopology, AutonomousSystemTopologyGenerator, TopologyGenerationError +from .AutonomousSystemLocation import AutonomousSystemLocationGenerator, LocationGenerationError diff --git a/setup.py b/setup.py index c02852039..b8d405579 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,9 @@ "Operating System :: OS Independent", ], install_requires = [ - 'requests' + 'geopy', + 'requests', + 'PyYAML', ], - python_requires = '>=3.6' + python_requires = '>=3.10' ) diff --git a/test_frr_exabgp_foundation.py b/test_frr_exabgp_foundation.py new file mode 100644 index 000000000..06e04db18 --- /dev/null +++ b/test_frr_exabgp_foundation.py @@ -0,0 +1,890 @@ +from __future__ import annotations + +import importlib + +import pytest + +from seedemu.core import Binding, Emulator, Filter +from seedemu.layers import Base, Ebgp, Ibgp, Mpls, Ospf, PeerRelationship, Routing +from seedemu.layers._bgp_metadata import get_bgp_sessions, get_ospf_interface_intents +from seedemu.services import ExaBgpService + + +def _file_content(node, path: str) -> str: + for file in node.getFiles(): + file_path, content = file.get() + if file_path == path: + return content + return "" + + +def test_router_backend_default_and_legacy_rejection(): + as2 = Base().createAutonomousSystem(2) + assert as2.createRouter("default").getRoutingBackend() == "bird" + assert as2.createRouter("frr", routingBackend="frr").getRoutingBackend() == "frr" + + with pytest.raises(AssertionError, match="unsupported routing backend"): + as2.createRouter("exabgp", routingBackend="exabgp") + with pytest.raises(AssertionError, match="unsupported routing backend"): + as2.createRouter("external", routingBackend="external") + + +def test_as_level_control_plane_modes_are_explicit_intent(): + as2 = Base().createAutonomousSystem(2) + + assert as2.getIbgpMode() == "full-mesh" + assert not as2.hasIbgpMode() + assert as2.getOspfMode() == "legacy" + assert not as2.hasOspfMode() + + as2.setIbgpMode("route-reflector") + as2.setOspfMode("router-transit-only") + assert as2.getIbgpMode() == "route-reflector" + assert as2.hasIbgpMode() + assert as2.getOspfMode() == "router-transit-only" + assert as2.hasOspfMode() + + with pytest.raises(AssertionError, match="unsupported iBGP mode"): + as2.setIbgpMode("rr") + with pytest.raises(AssertionError, match="unsupported OSPF mode"): + as2.setOspfMode("host-facing") + + +def test_frr_bgp_layer_shim_is_not_exported(): + with pytest.raises(ModuleNotFoundError): + importlib.import_module("seedemu.layers.FrrBgp") + assert not hasattr(importlib.import_module("seedemu.layers"), "FrrBgp") + + +def test_route_server_peering_renders_from_intent_in_routing_layer(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + as150 = base.createAutonomousSystem(150) + as150.createNetwork("net0") + as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp.addRsPeer(100, 150) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + route_server = emu.getRegistry().get("ix", "rs", "ix100") + router = emu.getRegistry().get("150", "rnode", "router0") + + rs_conf = _file_content(route_server, "/etc/bird/bird.conf") + router_conf = _file_content(router, "/etc/bird/bird.conf") + + assert "protocol bgp p_as150" in rs_conf + assert "rs client" in rs_conf + assert "neighbor 10.100.0.150 as 150" in rs_conf + assert "protocol bgp p_rs100" in router_conf + assert "neighbor 10.100.0.100 as 100" in router_conf + assert "bgp_large_community.add(PEER_COMM)" in router_conf + + +def test_frr_route_server_backend_renders_route_server_client_config(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + ix = base.createInternetExchange(100) + ix.getRouteServerNode().setRoutingBackend("frr") + as150 = base.createAutonomousSystem(150) + as150.createNetwork("net0") + as150.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + ebgp.addRsPeer(100, 150) + ebgp.addRsPeer(100, 151) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + route_server = emu.getRegistry().get("ix", "rs", "ix100") + as150_router = emu.getRegistry().get("150", "rnode", "router0") + frr_conf = _file_content(route_server, "/etc/frr/frr.conf") + + assert _file_content(route_server, "/etc/bird/bird.conf") == "" + assert "router bgp 100" in frr_conf + assert " bgp router-id 10.100.0.100" in frr_conf + assert "neighbor 10.100.0.150 remote-as 150" in frr_conf + assert "neighbor 10.100.0.151 remote-as 151" in frr_conf + assert "neighbor 10.100.0.150 route-server-client" in frr_conf + assert "neighbor 10.100.0.151 route-server-client" in frr_conf + assert "redistribute connected" not in frr_conf + assert "protocol bgp p_rs100" in _file_content(as150_router, "/etc/bird/bird.conf") + + +def test_frr_backend_renders_frr_config_for_selected_router(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + ebgp = Ebgp() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") + as2.createRouter("r2", routingBackend="frr").joinNetwork("net0").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.addLayer(ebgp) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + r2 = emu.getRegistry().get("2", "rnode", "r2") + + assert any(cmd == "bird -d" for cmd, _ in r1.getStartCommands()) + assert not any(cmd == "bird -d" for cmd, _ in r2.getStartCommands()) + assert _file_content(r2, "/etc/bird/bird.conf") == "" + + r1_bird_conf = _file_content(r1, "/etc/bird/bird.conf") + assert r1_bird_conf.index("ipv4 table t_ospf") < r1_bird_conf.index("protocol bgp ibgp1") + + frr_conf = _file_content(r2, "/etc/frr/frr.conf") + assert "router bgp 2" in frr_conf + assert "neighbor 10.101.0.152 remote-as 152" in frr_conf + assert "RM_CONNECTED_TO_BGP" in frr_conf + assert "LC_LOCAL_OR_CUSTOMER" in frr_conf + assert "router ospf" in frr_conf + assert "interface net0" in frr_conf + + +def _build_three_router_ibgp_emulator(ibgp: Ibgp, configure_as=None): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("west") + as2.createNetwork("east") + as2.createRouter("edge_west").joinNetwork("west") + as2.createRouter("core").joinNetwork("west").joinNetwork("east") + as2.createRouter("edge_east", routingBackend="frr").joinNetwork("east") + if configure_as is not None: + configure_as(as2) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + return ( + emu.getRegistry().get("2", "rnode", "edge_west"), + emu.getRegistry().get("2", "rnode", "core"), + emu.getRegistry().get("2", "rnode", "edge_east"), + ) + + +def test_default_ibgp_mode_preserves_legacy_full_mesh(): + edge_west, core, edge_east = _build_three_router_ibgp_emulator(Ibgp()) + + assert len([session for session in get_bgp_sessions(edge_west) if session["kind"] == "ibgp"]) == 2 + assert len([session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"]) == 2 + assert len([session for session in get_bgp_sessions(edge_east) if session["kind"] == "ibgp"]) == 2 + + +def test_edge_only_full_mesh_ibgp_scope_excludes_core_router(): + def configure_edge_only(as2): + as2.setIbgpMode("full-mesh") + as2.setBgpScope("edge-only") + as2.getRouter("edge_west").setBgpRole("edge") + as2.getRouter("edge_east").setBgpRole("edge") + as2.getRouter("core").setBgpRole("core") + + edge_west, core, edge_east = _build_three_router_ibgp_emulator( + Ibgp(), + configure_as=configure_edge_only, + ) + + west_sessions = [session for session in get_bgp_sessions(edge_west) if session["kind"] == "ibgp"] + east_sessions = [session for session in get_bgp_sessions(edge_east) if session["kind"] == "ibgp"] + assert len(west_sessions) == 1 + assert len(east_sessions) == 1 + assert west_sessions[0]["peer_address"] == str(edge_east.getLoopbackAddress()) + assert east_sessions[0]["peer_address"] == str(edge_west.getLoopbackAddress()) + assert [session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"] == [] + + +def test_router_bgp_roles_drive_edge_only_full_mesh_scope(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.setIbgpMode("full-mesh") + as2.setBgpScope("edge-only") + as2.createNetwork("west") + as2.createNetwork("east") + edge_west = as2.createRouter("edge_west").joinNetwork("west") + core = as2.createRouter("core").joinNetwork("west").joinNetwork("east") + edge_east = as2.createRouter("edge_east").joinNetwork("east") + edge_west.setBgpRole("edge") + core.setBgpRole("core") + edge_east.setBgpRole("edge") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + edge_west = emu.getRegistry().get("2", "rnode", "edge_west") + core = emu.getRegistry().get("2", "rnode", "core") + edge_east = emu.getRegistry().get("2", "rnode", "edge_east") + + west_sessions = [session for session in get_bgp_sessions(edge_west) if session["kind"] == "ibgp"] + east_sessions = [session for session in get_bgp_sessions(edge_east) if session["kind"] == "ibgp"] + assert edge_west.getBgpRole() == "edge" + assert edge_west.getLabel()["seedemu_bgp_role"] == "edge" + assert core.getBgpRole() == "core" + assert len(west_sessions) == 1 + assert len(east_sessions) == 1 + assert west_sessions[0]["peer_address"] == str(edge_east.getLoopbackAddress()) + assert east_sessions[0]["peer_address"] == str(edge_west.getLoopbackAddress()) + assert [session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"] == [] + + +def test_router_bgp_roles_do_not_change_legacy_ibgp_default(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("west") + as2.createNetwork("east") + as2.createRouter("edge_west").joinNetwork("west").setBgpRole("edge") + as2.createRouter("core").joinNetwork("west").joinNetwork("east").setBgpRole("core") + as2.createRouter("edge_east").joinNetwork("east").setBgpRole("edge") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + core = emu.getRegistry().get("2", "rnode", "core") + assert len([session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"]) == 2 + + +def test_router_bgp_role_and_disable_validation(): + router = Base().createAutonomousSystem(2).createRouter("r1") + + with pytest.raises(AssertionError, match="unsupported BGP role"): + router.setBgpRole("rr") + with pytest.raises(AssertionError, match="unsupported router control-plane disable flag"): + router.disableControlPlane("ospf") + + router.disableControlPlane("ibgp") + assert router.isControlPlaneDisabled("ibgp") + assert router.getDisabledControlPlanes() == {"ibgp"} + assert router.getLabel()["seedemu_control_plane_disabled_ibgp"] == "true" + + +def test_router_ibgp_disable_removes_local_and_remote_sessions(): + edge_west, core, edge_east = _build_three_router_ibgp_emulator( + Ibgp(), + configure_as=lambda as2: as2.getRouter("core").disableControlPlane("ibgp"), + ) + + west_sessions = [session for session in get_bgp_sessions(edge_west) if session["kind"] == "ibgp"] + east_sessions = [session for session in get_bgp_sessions(edge_east) if session["kind"] == "ibgp"] + assert len(west_sessions) == 1 + assert len(east_sessions) == 1 + assert west_sessions[0]["peer_address"] == str(edge_east.getLoopbackAddress()) + assert east_sessions[0]["peer_address"] == str(edge_west.getLoopbackAddress()) + assert [session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"] == [] + +def test_disabled_ibgp_mode_matches_as_masking_semantics(): + ibgp = Ibgp() + edge_west, core, edge_east = _build_three_router_ibgp_emulator( + ibgp, + configure_as=lambda as2: as2.setIbgpMode("disabled"), + ) + + assert get_bgp_sessions(edge_west) == [] + assert get_bgp_sessions(core) == [] + assert get_bgp_sessions(edge_east) == [] + +def test_route_reflector_mode_completes_default_cluster_membership(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.setIbgpMode("route-reflector") + as2.createNetwork("net0") + as2.createBgpCluster("10.2.0.1") + as2.createRouter("rr").joinNetwork("net0").joinBgpCluster("10.2.0.1").makeRouteReflector() + as2.createRouter("client").joinNetwork("net0").joinBgpCluster("10.2.0.1") + as2.createRouter("core").joinNetwork("net0") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + rr = emu.getRegistry().get("2", "rnode", "rr") + client = emu.getRegistry().get("2", "rnode", "client") + core = emu.getRegistry().get("2", "rnode", "core") + + assert len([session for session in get_bgp_sessions(rr) if session["route_reflector_client"]]) == 2 + assert len([session for session in get_bgp_sessions(client) if session["kind"] == "ibgp"]) == 1 + assert len([session for session in get_bgp_sessions(core) if session["kind"] == "ibgp"]) == 1 + assert core.getBgpClusterId() == "10.2.0.1" + + +def test_edge_router_can_also_be_route_reflector(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.setIbgpMode("route-reflector") + as2.createNetwork("net0") + as2.createBgpCluster("10.2.0.1") + rr = as2.createRouter("edge_rr").joinNetwork("net0").joinBgpCluster("10.2.0.1") + rr.setBgpRole("edge").makeRouteReflector() + as2.createRouter("client").joinNetwork("net0").joinBgpCluster("10.2.0.1") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + edge_rr = emu.getRegistry().get("2", "rnode", "edge_rr") + assert edge_rr.getBgpRole() == "edge" + assert edge_rr.isRouteReflector() + assert len([session for session in get_bgp_sessions(edge_rr) if session["route_reflector_client"]]) == 1 + + +def test_ospf_legacy_mode_keeps_local_networks_active_by_default(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("transit") + as2.createNetwork("hostnet") + as2.createRouter("r1").joinNetwork("transit").joinNetwork("hostnet") + as2.createRouter("r2").joinNetwork("transit") + as2.createHost("host").joinNetwork("hostnet") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + intents = get_ospf_interface_intents(r1) + assert "transit" in intents["active"] + assert "hostnet" in intents["active"] + + +def test_ospf_router_transit_only_mode_keeps_host_network_passive(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + + as2 = base.createAutonomousSystem(2) + as2.setOspfMode("router-transit-only") + as2.createNetwork("transit") + as2.createNetwork("hostnet") + as2.createNetwork("hostnet2") + as2.createRouter("r1").joinNetwork("transit").joinNetwork("hostnet") + as2.createRouter("r2", routingBackend="frr").joinNetwork("transit").joinNetwork("hostnet2") + as2.createHost("host").joinNetwork("hostnet") + as2.createHost("host2").joinNetwork("hostnet2") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + intents = get_ospf_interface_intents(r1) + assert "transit" in intents["active"] + assert "hostnet" in intents["passive"] + assert "hostnet" not in intents["active"] + + bird_conf = _file_content(r1, "/etc/bird/bird.conf") + frr_conf = _file_content(emu.getRegistry().get("2", "rnode", "r2"), "/etc/frr/frr.conf") + assert 'interface "transit" { hello 1; dead count 2; }' in bird_conf + assert 'interface "hostnet" { stub; }' in bird_conf + assert "interface transit\n ip ospf area 0" in frr_conf + assert "interface hostnet2\n ip ospf area 0\n ip ospf passive" in frr_conf + + +def test_ospf_router_transit_only_respects_explicit_stub_and_mask(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ospf.markAsStub(2, "stubbed") + ospf.maskNetwork(2, "masked") + + as2 = base.createAutonomousSystem(2) + as2.setOspfMode("router-transit-only") + as2.createNetwork("transit") + as2.createNetwork("stubbed") + as2.createNetwork("masked") + as2.createRouter("r1").joinNetwork("transit").joinNetwork("stubbed").joinNetwork("masked") + as2.createRouter("r2").joinNetwork("transit").joinNetwork("stubbed").joinNetwork("masked") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + intents = get_ospf_interface_intents(r1) + assert "transit" in intents["active"] + assert "stubbed" in intents["passive"] + assert "stubbed" not in intents["active"] + assert "masked" not in intents["active"] + assert "masked" not in intents["passive"] + + +def test_duplicate_bgp_session_names_are_preserved_with_unique_render_names(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createRouter("router0").joinNetwork("ix100").joinNetwork("ix101") + + as151 = base.createAutonomousSystem(151) + as151.createRouter("router0").joinNetwork("ix100").joinNetwork("ix101") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Peer) + ebgp.addPrivatePeering(101, 2, 151, abRelationship=PeerRelationship.Peer) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + router = emu.getRegistry().get("2", "rnode", "router0") + sessions = get_bgp_sessions(router) + assert len([session for session in sessions if session["name"] == "p_as151"]) == 2 + assert len({session["render_name"] for session in sessions}) == len(sessions) + + bird_conf = _file_content(router, "/etc/bird/bird.conf") + assert bird_conf.count("neighbor 10.100.0.151 as 151") == 1 + assert bird_conf.count("neighbor 10.101.0.151 as 151") == 1 + assert bird_conf.count("protocol bgp p_as151") == 2 + + +def test_private_peering_can_select_explicit_ix_routers(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createRouter("edge_west").joinNetwork("ix100", address="10.100.0.20") + as2.createRouter("edge_east").joinNetwork("ix100", address="10.100.0.21") + + as151 = base.createAutonomousSystem(151) + as151.createRouter("router0").joinNetwork("ix100", address="10.100.0.151") + + ebgp.addPrivatePeeringByRouters( + 100, + 2, + "edge_east", + 151, + "router0", + abRelationship=PeerRelationship.Provider, + ) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + edge_west = emu.getRegistry().get("2", "rnode", "edge_west") + edge_east = emu.getRegistry().get("2", "rnode", "edge_east") + customer = emu.getRegistry().get("151", "rnode", "router0") + + assert [session for session in get_bgp_sessions(edge_west) if session["kind"] == "ebgp"] == [] + east_sessions = [session for session in get_bgp_sessions(edge_east) if session["kind"] == "ebgp"] + assert len(east_sessions) == 1 + assert east_sessions[0]["local_address"] == "10.100.0.21" + assert east_sessions[0]["peer_address"] == "10.100.0.151" + assert east_sessions[0]["peer_asn"] == 151 + + customer_conf = _file_content(customer, "/etc/bird/bird.conf") + assert "neighbor 10.100.0.21 as 2" in customer_conf + assert "neighbor 10.100.0.20 as 2" not in customer_conf + + +def test_route_server_peer_can_select_explicit_ix_router(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createRouter("edge_west").joinNetwork("ix100", address="10.100.0.20") + as2.createRouter("edge_east").joinNetwork("ix100", address="10.100.0.21") + + ebgp.addRsPeerByRouter(100, 2, "edge_east") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.render() + + route_server = emu.getRegistry().get("ix", "rs", "ix100") + edge_west = emu.getRegistry().get("2", "rnode", "edge_west") + edge_east = emu.getRegistry().get("2", "rnode", "edge_east") + + assert [session for session in get_bgp_sessions(edge_west) if session["kind"] == "ebgp"] == [] + east_sessions = [session for session in get_bgp_sessions(edge_east) if session["kind"] == "ebgp"] + assert len(east_sessions) == 1 + assert east_sessions[0]["local_address"] == "10.100.0.21" + assert east_sessions[0]["peer_asn"] == 100 + + rs_conf = _file_content(route_server, "/etc/bird/bird.conf") + assert "neighbor 10.100.0.21 as 2" in rs_conf + assert "neighbor 10.100.0.20 as 2" not in rs_conf + + +def test_explicit_ix_router_selection_rejects_non_attached_router(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("edge").joinNetwork("ix100", address="10.100.0.20") + as2.createRouter("core").joinNetwork("net0") + + as151 = base.createAutonomousSystem(151) + as151.createRouter("router0").joinNetwork("ix100", address="10.100.0.151") + + ebgp.addPrivatePeeringByRouters(100, 2, "core", 151, "router0") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + + with pytest.raises(AssertionError, match="explicit peering router as2/core is not connected to ix100"): + emu.render() + + +def test_route_reflector_intent_renders_without_direct_ibgp_bird_writes(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createBgpCluster("10.2.0.1") + as2.createRouter("rr").joinNetwork("net0").joinBgpCluster("10.2.0.1").makeRouteReflector() + as2.createRouter("client").joinNetwork("net0").joinBgpCluster("10.2.0.1") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + rr = emu.getRegistry().get("2", "rnode", "rr") + client = emu.getRegistry().get("2", "rnode", "client") + + rr_conf = _file_content(rr, "/etc/bird/bird.conf") + client_conf = _file_content(client, "/etc/bird/bird.conf") + + assert "protocol bgp Ibgp_rr_client_client" in rr_conf + assert "passive yes" in rr_conf + assert "rr client" in rr_conf + assert "rr cluster id 10.2.0.1" in rr_conf + assert "protocol bgp Ibgp_rr_rr" in client_conf + assert "next hop self" in client_conf + assert rr_conf.index("ipv4 table t_ospf") < rr_conf.index("protocol bgp Ibgp_rr_client_client") + + +def test_frr_route_reflector_renders_cluster_id_and_client(): + emu = Emulator() + base = Base() + routing = Routing() + ospf = Ospf() + ibgp = Ibgp() + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createBgpCluster("10.2.0.1") + as2.createRouter("rr", routingBackend="frr").joinNetwork("net0").joinBgpCluster("10.2.0.1").makeRouteReflector() + as2.createRouter("client").joinNetwork("net0").joinBgpCluster("10.2.0.1") + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ospf) + emu.addLayer(ibgp) + emu.render() + + rr = emu.getRegistry().get("2", "rnode", "rr") + frr_conf = _file_content(rr, "/etc/frr/frr.conf") + + assert "router bgp 2" in frr_conf + assert " bgp cluster-id 10.2.0.1" in frr_conf + assert "neighbor 10.0.0.2 route-reflector-client" in frr_conf + assert "neighbor 10.0.0.2 passive" in frr_conf + + +def test_as_level_mpls_core_forwarding_masks_ospf_and_ibgp_before_intent_is_recorded(): + emu = Emulator() + base = Base() + routing = Routing() + ebgp = Ebgp() + ibgp = Ibgp() + ospf = Ospf() + mpls = Mpls() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createNetwork("net2") + as2.createRouter("r1").joinNetwork("net0").joinNetwork("ix100") + as2.createRouter("r2").joinNetwork("net0").joinNetwork("net1") + as2.createRouter("r3").joinNetwork("net1").joinNetwork("net2") + as2.createRouter("r4").joinNetwork("net2").joinNetwork("ix101") + as2.setCoreForwarding("mpls") + + as151 = base.createAutonomousSystem(151) + as151.createNetwork("net0") + as151.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as152 = base.createAutonomousSystem(152) + as152.createNetwork("net0") + as152.createRouter("router0").joinNetwork("net0").joinNetwork("ix101") + + ebgp.addPrivatePeering(100, 2, 151, abRelationship=PeerRelationship.Provider) + ebgp.addPrivatePeering(101, 2, 152, abRelationship=PeerRelationship.Provider) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ebgp) + emu.addLayer(ibgp) + emu.addLayer(ospf) + emu.addLayer(mpls) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + r2 = emu.getRegistry().get("2", "rnode", "r2") + r4 = emu.getRegistry().get("2", "rnode", "r4") + + assert get_ospf_interface_intents(r1) == {"active": [], "passive": []} + r1_ibgp = [session for session in get_bgp_sessions(r1) if session["kind"] == "ibgp"] + r4_ibgp = [session for session in get_bgp_sessions(r4) if session["kind"] == "ibgp"] + assert len(r1_ibgp) == 1 + assert len(r4_ibgp) == 1 + assert r1_ibgp[0]["name"] == "mpls_ibgp1" + assert r1_ibgp[0]["igp_table"] == "master4" + assert r1_ibgp[0]["peer_address"] == str(r4.getLoopbackAddress()) + assert get_bgp_sessions(r2) == [] + + r1_conf = _file_content(r1, "/etc/bird/bird.conf") + r2_conf = _file_content(r2, "/etc/bird/bird.conf") + r4_conf = _file_content(r4, "/etc/bird/bird.conf") + assert "protocol ospf ospf1" not in r1_conf + assert "protocol ospf ospf1" not in r2_conf + assert "protocol bgp ibgp" not in r2_conf + assert "protocol bgp mpls_ibgp1" in r1_conf + assert "igp table master4" in r1_conf + assert "protocol bgp mpls_ibgp1" in r4_conf + + +def test_mpls_preserves_explicit_route_reflector_ibgp_mode(): + emu = Emulator() + base = Base() + routing = Routing() + ibgp = Ibgp() + ospf = Ospf() + mpls = Mpls() + + base.createInternetExchange(100) + base.createInternetExchange(101) + + as2 = base.createAutonomousSystem(2) + as2.setIbgpMode("route-reflector") + as2.setBgpScope("edge-only") + as2.createNetwork("net0") + as2.createNetwork("net1") + as2.createBgpCluster("10.2.0.1") + as2.createRouter("r1").joinNetwork("ix100").joinNetwork("net0").joinBgpCluster("10.2.0.1").setBgpRole("edge").makeRouteReflector() + as2.createRouter("r2").joinNetwork("net0").joinNetwork("net1").setBgpRole("core") + as2.createRouter("r4").joinNetwork("net1").joinNetwork("ix101").joinBgpCluster("10.2.0.1").setBgpRole("edge") + mpls.enableOn(2) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(ibgp) + emu.addLayer(ospf) + emu.addLayer(mpls) + emu.render() + + r1 = emu.getRegistry().get("2", "rnode", "r1") + r2 = emu.getRegistry().get("2", "rnode", "r2") + r4 = emu.getRegistry().get("2", "rnode", "r4") + + r1_ibgp = [session for session in get_bgp_sessions(r1) if session["kind"] == "ibgp"] + r4_ibgp = [session for session in get_bgp_sessions(r4) if session["kind"] == "ibgp"] + assert len(r1_ibgp) == 1 + assert r1_ibgp[0]["route_reflector_client"] + assert r1_ibgp[0]["igp_table"] == "master4" + assert len(r4_ibgp) == 1 + assert r4_ibgp[0]["name"] == "Ibgp_rr_r1" + assert get_bgp_sessions(r2) == [] + + r1_conf = _file_content(r1, "/etc/bird/bird.conf") + r2_conf = _file_content(r2, "/etc/bird/bird.conf") + r2_frr = _file_content(r2, "/etc/frr/frr.conf") + assert "protocol bgp Ibgp_rr_client_r4" in r1_conf + assert "igp table master4" in r1_conf + assert "protocol bgp mpls_ibgp" not in r1_conf + assert "protocol bgp" not in r2_conf + assert "mpls ldp" in r2_frr + + +def test_exabgp_service_renders_speaker_and_router_peer(): + emu = Emulator() + base = Base() + routing = Routing() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0").joinNetwork("net0").joinNetwork("ix100") + + as180 = base.createAutonomousSystem(180) + as180.createHost("exabgp").joinNetwork("ix100", address="10.100.0.180") + + exabgp.install("as180_exabgp") \ + .setLocalAsn(180) \ + .addPeer("router0", router_asn=2, router_relationship="customer") \ + .addAnnouncement("198.51.100.0/24") + emu.addBinding(Binding("as180_exabgp", filter=Filter(asn=180, nodeName="exabgp"))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(exabgp) + emu.render() + + speaker = emu.getRegistry().get("180", "hnode", "exabgp") + router = emu.getRegistry().get("2", "rnode", "router0") + + exabgp_conf = _file_content(speaker, "/etc/exabgp/exabgp.conf") + assert "neighbor 10.100.0.2" in exabgp_conf + assert "local-as 180" in exabgp_conf + assert "peer-as 2" in exabgp_conf + assert "198.51.100.0/24" in exabgp_conf + + bird_conf = _file_content(router, "/etc/bird/bird.conf") + assert "protocol bgp exabgp_180" in bird_conf + assert "neighbor 10.100.0.180 as 180" in bird_conf + assert "bgp_large_community.add(CUSTOMER_COMM)" in bird_conf + assert "bgp_local_pref = 30" in bird_conf + + +def test_exabgp_service_renders_frr_router_peer_without_bird(): + emu = Emulator() + base = Base() + routing = Routing() + exabgp = ExaBgpService() + + base.createInternetExchange(100) + + as2 = base.createAutonomousSystem(2) + as2.createNetwork("net0") + as2.createRouter("router0", routingBackend="frr").joinNetwork("net0").joinNetwork("ix100") + + as180 = base.createAutonomousSystem(180) + as180.createHost("exabgp").joinNetwork("ix100", address="10.100.0.180") + + exabgp.install("as180_exabgp") \ + .setLocalAsn(180) \ + .addPeer("router0", router_asn=2, router_relationship="customer") \ + .addAnnouncement("198.51.100.0/24") + emu.addBinding(Binding("as180_exabgp", filter=Filter(asn=180, nodeName="exabgp"))) + + emu.addLayer(base) + emu.addLayer(routing) + emu.addLayer(exabgp) + emu.render() + + speaker = emu.getRegistry().get("180", "hnode", "exabgp") + router = emu.getRegistry().get("2", "rnode", "router0") + + assert _file_content(router, "/etc/bird/bird.conf") == "" + frr_conf = _file_content(router, "/etc/frr/frr.conf") + assert "router bgp 2" in frr_conf + assert "neighbor 10.100.0.180 remote-as 180" in frr_conf + assert "neighbor 10.100.0.180 description exabgp_180" in frr_conf + assert "neighbor 10.100.0.180 next-hop-self" in frr_conf + + exabgp_conf = _file_content(speaker, "/etc/exabgp/exabgp.conf") + assert "peer-as 2" in exabgp_conf + assert "198.51.100.0/24" in exabgp_conf diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 000000000..35053a67d --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,42 @@ +# Feature-Oriented CI + +The pull-request workflow is intentionally organized around SEED features rather +than broad example buckets. The manifest in `feature_manifest.json` is the source +of truth for which features are covered by unit tests, representative example +compilation, selected Docker builds, and optional runtime integration probes. + +The CI runner writes both human-readable logs and machine-readable artifacts: + +- `ci-summary.json` records every command, return code, duration, and log path. +- `junit.xml` records the same stage in a format GitHub and review tooling can + ingest. +- `feature-coverage.json` records the manifest-derived coverage state, including + covered features and declared gaps that have not landed on this integration + line. + +The static stage compiles importable Python source plus representative examples. +It intentionally excludes embedded payload templates under +`seedemu/services/EthereumService/EthTemplates/`, where some historical `.py` +filenames contain shell script content copied into containers. + +Run stages locally from the repository root: + +```bash +python3 tests/ci/run_ci.py static --artifact-dir ci-artifacts/static +python3 tests/ci/run_ci.py unit --artifact-dir ci-artifacts/unit +python3 tests/ci/run_ci.py example-compile --artifact-dir ci-artifacts/example-compile +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +``` + +Docker image builds and runtime integration are available as explicit entry +points, but they are not default pull-request gates in this PR: + +```bash +python3 tests/ci/run_ci.py example-build --artifact-dir ci-artifacts/example-build +python3 tests/ci/run_ci.py runtime-integration --artifact-dir ci-artifacts/runtime-integration +``` + +The runtime integration stage is kept as an explicit entry point for future +Docker runtime probes. The current manifest does not enable any runtime group on +this branch, so the stage is a reserved hook unless new groups are added to +`feature_manifest.json`. diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 000000000..2829e92c3 --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +"""Feature-oriented CI helpers for SEED Emulator.""" diff --git a/tests/ci/feature_manifest.json b/tests/ci/feature_manifest.json new file mode 100644 index 000000000..38667cb12 --- /dev/null +++ b/tests/ci/feature_manifest.json @@ -0,0 +1,110 @@ +{ + "schema": 1, + "features": { + "ipv4-default": { + "status": "covered", + "description": "Legacy IPv4 examples continue to compile and build without requiring any new control-plane capability.", + "unit_groups": [], + "compile_examples": [ + "basic-a00-simple-as" + ], + "build_examples": [ + "basic-a00-simple-as" + ], + "runtime_groups": [] + }, + "ipv6-core": { + "status": "declared-gap", + "description": "Repository-level IPv6 readiness is tracked here, but this K8s integration branch does not currently carry a representative IPv6 example or runtime probe.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until native IPv6 topology/API coverage or another representative IPv6 example is added." + }, + "routing-bird-frr": { + "status": "declared-gap", + "description": "BIRD/FRR interoperability coverage from the source CI framework is not currently carried by this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the branch includes a maintained BIRD/FRR example and matching runtime probe." + }, + "exabgp-service": { + "status": "declared-gap", + "description": "ExaBGP coverage exists on the source CI framework branch, but this K8s integration branch does not carry ExaBgpService or its examples.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until ExaBGP service support is present on this branch." + }, + "looking-glass": { + "status": "declared-gap", + "description": "Looking Glass service support is present, but the source CI framework examples are not present on this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until the representative Looking Glass example is merged here." + }, + "dns-endpoint": { + "status": "covered", + "description": "DNS endpoint behavior is represented by the mini Internet plus DNS example.", + "unit_groups": [], + "compile_examples": [ + "internet-b02-mini-internet-with-dns" + ], + "build_examples": [ + "internet-b02-mini-internet-with-dns" + ], + "runtime_groups": [] + }, + "legacy-guard": { + "status": "declared-gap", + "description": "Removed legacy control-plane API guards exist on the source CI framework branch, but those tests are not present on this K8s integration branch.", + "unit_groups": [], + "compile_examples": [], + "build_examples": [], + "runtime_groups": [], + "notes": "Keep declared as a gap until control-plane guard tests are merged here." + } + }, + "unit_groups": {}, + "runtime_groups": {}, + "examples": { + "basic-a00-simple-as": { + "description": "Small IPv4 default behavior smoke example.", + "script": "examples/basic/A00_simple_as/simple_as.py", + "args": [ + "amd" + ], + "clean": [ + "output" + ], + "expected": [ + "output/docker-compose.yml" + ], + "compile": true, + "build": true + }, + "internet-b02-mini-internet-with-dns": { + "description": "Mini Internet with DNS component and local DNS endpoints.", + "script": "examples/internet/B02_mini_internet_with_dns/mini_internet_with_dns.py", + "args": [ + "amd" + ], + "clean": [ + "output", + "base_internet.bin", + "dns_component.bin" + ], + "expected": [ + "output/docker-compose.yml" + ], + "compile": true, + "build": true + } + } +} diff --git a/tests/ci/run_ci.py b/tests/ci/run_ci.py new file mode 100644 index 000000000..0594b0b49 --- /dev/null +++ b/tests/ci/run_ci.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import time +import traceback +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Iterable + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = Path(__file__).with_name("feature_manifest.json") + + +def _rel(path: Path) -> str: + try: + return str(path.resolve().relative_to(REPO_ROOT)) + except ValueError: + return str(path) + + +def _slug(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "check" + + +def _json_dump(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _tail(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[-limit:] + + +def load_manifest() -> dict[str, Any]: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def _validate_manifest(manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if manifest.get("schema") != 1: + errors.append("manifest schema must be 1") + + features = manifest.get("features", {}) + unit_groups = manifest.get("unit_groups", {}) + runtime_groups = manifest.get("runtime_groups", {}) + examples = manifest.get("examples", {}) + required_features = { + "ipv4-default", + "ipv6-core", + "routing-bird-frr", + "exabgp-service", + "looking-glass", + "dns-endpoint", + "legacy-guard", + } + missing = sorted(required_features.difference(features)) + if missing: + errors.append(f"missing required feature declarations: {', '.join(missing)}") + + for feature_id, feature in features.items(): + for group_id in feature.get("unit_groups", []): + if group_id not in unit_groups: + errors.append(f"feature {feature_id} references missing unit group {group_id}") + for group_id in feature.get("runtime_groups", []): + if group_id not in runtime_groups: + errors.append(f"feature {feature_id} references missing runtime group {group_id}") + for example_id in feature.get("compile_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing compile example {example_id}") + for example_id in feature.get("build_examples", []): + if example_id not in examples: + errors.append(f"feature {feature_id} references missing build example {example_id}") + + for example_id, example in examples.items(): + script = REPO_ROOT / example.get("script", "") + if not script.is_file(): + errors.append(f"example {example_id} script does not exist: {_rel(script)}") + for expected in example.get("expected", []): + if Path(expected).is_absolute(): + errors.append(f"example {example_id} expected path must be relative: {expected}") + for clean in example.get("clean", []): + if Path(clean).is_absolute(): + errors.append(f"example {example_id} clean path must be relative: {clean}") + return errors + + +def feature_coverage(manifest: dict[str, Any]) -> dict[str, Any]: + features = {} + for feature_id, feature in sorted(manifest["features"].items()): + features[feature_id] = { + "status": feature["status"], + "description": feature.get("description", ""), + "unit_groups": feature.get("unit_groups", []), + "compile_examples": feature.get("compile_examples", []), + "build_examples": feature.get("build_examples", []), + "runtime_groups": feature.get("runtime_groups", []), + "notes": feature.get("notes", ""), + } + return { + "schema": manifest["schema"], + "generated_by": "tests/ci/run_ci.py", + "features": features, + } + + +def _write_junit(stage: str, checks: list[dict[str, Any]], path: Path) -> None: + tests = len(checks) + failures = sum(1 for check in checks if check["status"] == "failed") + skipped = sum(1 for check in checks if check["status"] == "skipped") + suite = ET.Element( + "testsuite", + { + "name": f"seedemu-ci-{stage}", + "tests": str(tests), + "failures": str(failures), + "errors": "0", + "skipped": str(skipped), + "time": f"{sum(float(check.get('duration_s', 0.0)) for check in checks):.3f}", + }, + ) + for check in checks: + case = ET.SubElement( + suite, + "testcase", + { + "classname": f"seedemu.ci.{stage}", + "name": check["name"], + "time": f"{float(check.get('duration_s', 0.0)):.3f}", + }, + ) + if check["status"] == "failed": + failure = ET.SubElement( + case, + "failure", + { + "message": check.get("message", "check failed"), + "type": "CommandFailure", + }, + ) + failure.text = check.get("details", "") + elif check["status"] == "skipped": + skipped_node = ET.SubElement(case, "skipped", {"message": check.get("message", "skipped")}) + skipped_node.text = check.get("details", "") + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) + + +def _stage_result(stage: str, checks: list[dict[str, Any]], artifact_dir: Path) -> int: + summary = { + "stage": stage, + "status": "failed" if any(check["status"] == "failed" for check in checks) else "passed", + "checks": checks, + } + _json_dump(artifact_dir / "ci-summary.json", summary) + _write_junit(stage, checks, artifact_dir / "junit.xml") + + print(f"[seed-ci] stage={stage} status={summary['status']} artifact_dir={_rel(artifact_dir)}") + for check in checks: + print( + "[seed-ci] " + f"{check['status']}: {check['name']} " + f"log={check.get('log_path', '')}" + ) + if check["status"] == "failed": + command = check.get("command") + if command: + print(f"[seed-ci] failed-command: {' '.join(command)}") + details = (check.get("stderr_tail") or check.get("details") or "").strip() + if details: + print("[seed-ci] failure-tail:") + print(_tail(details, limit=1200)) + return 1 if summary["status"] == "failed" else 0 + + +def _run_command( + name: str, + cmd: list[str], + artifact_dir: Path, + *, + cwd: Path = REPO_ROOT, + env: dict[str, str] | None = None, + timeout: int | None = None, +) -> dict[str, Any]: + start = time.monotonic() + log_path = artifact_dir / "logs" / f"{_slug(name)}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + try: + result = subprocess.run( + cmd, + cwd=str(cwd), + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + duration = time.monotonic() - start + log_path.write_text( + "COMMAND: {}\nCWD: {}\nEXIT: {}\n\nSTDOUT:\n{}\n\nSTDERR:\n{}\n".format( + " ".join(cmd), + _rel(cwd), + result.returncode, + result.stdout, + result.stderr, + ), + encoding="utf-8", + ) + status = "passed" if result.returncode == 0 else "failed" + return { + "name": name, + "status": status, + "command": cmd, + "cwd": _rel(cwd), + "returncode": result.returncode, + "duration_s": duration, + "log_path": _rel(log_path), + "stdout_tail": _tail(result.stdout), + "stderr_tail": _tail(result.stderr), + "message": "command failed" if status == "failed" else "command passed", + "details": _tail(result.stdout + "\n" + result.stderr), + } + except Exception as exc: # pragma: no cover - defensive diagnostics for CI infrastructure failures. + duration = time.monotonic() - start + details = traceback.format_exc() + log_path.write_text(details, encoding="utf-8") + return { + "name": name, + "status": "failed", + "command": cmd, + "cwd": _rel(cwd), + "returncode": None, + "duration_s": duration, + "log_path": _rel(log_path), + "message": str(exc), + "details": details, + } + + +def _git_diff_check_command() -> list[str]: + base_ref = os.environ.get("GITHUB_BASE_REF") + if base_ref: + candidates = [f"origin/{base_ref}", base_ref] + for candidate in candidates: + probe = subprocess.run( + ["git", "rev-parse", "--verify", candidate], + cwd=str(REPO_ROOT), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if probe.returncode == 0: + return ["git", "diff", "--check", f"{candidate}...HEAD"] + return ["git", "diff", "--check"] + + +def _example_ids(manifest: dict[str, Any], key: str) -> list[str]: + ids: list[str] = [] + for example_id, example in manifest["examples"].items(): + if example.get(key): + ids.append(example_id) + return ids + + +def _pythonpath_env(extra: dict[str, str] | None = None) -> dict[str, str]: + env = dict(os.environ) + existing = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = str(REPO_ROOT) if not existing else f"{REPO_ROOT}:{existing}" + env.setdefault("DOCKER_BUILDKIT", "0") + env.setdefault("COMPOSE_BAKE", "false") + env.setdefault("COMPOSE_PARALLEL_LIMIT", "1") + if extra: + env.update(extra) + return env + + +def _pytest_env() -> dict[str, str]: + return _pythonpath_env({"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}) + + +def _safe_clean(example_dir: Path, relative_paths: Iterable[str]) -> None: + base = example_dir.resolve() + for item in relative_paths: + target = (example_dir / item).resolve() + if target == base or base not in target.parents: + raise ValueError(f"refusing to clean path outside example directory: {target}") + if target.is_dir(): + shutil.rmtree(target) + elif target.exists(): + target.unlink() + + +def _compile_example( + example_id: str, + example: dict[str, Any], + artifact_dir: Path, + *, + clean: bool, + name_prefix: str = "compile", +) -> dict[str, Any]: + script = REPO_ROOT / example["script"] + example_dir = script.parent + if clean: + _safe_clean(example_dir, example.get("clean", [])) + cmd = [sys.executable, script.name] + list(example.get("args", [])) + check = _run_command( + f"{name_prefix}:{example_id}", + cmd, + artifact_dir, + cwd=example_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("timeout", 900)), + ) + if check["status"] != "passed": + return check + + missing = [item for item in example.get("expected", []) if not (example_dir / item).exists()] + if missing: + check["status"] = "failed" + check["message"] = "expected compile outputs are missing" + check["details"] = "Missing outputs: " + ", ".join(missing) + return check + + +def run_static(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + + errors = _validate_manifest(manifest) + coverage = feature_coverage(manifest) + _json_dump(artifact_dir / "feature-coverage.json", coverage) + checks.append( + { + "name": "manifest", + "status": "failed" if errors else "passed", + "duration_s": 0.0, + "message": "manifest validation failed" if errors else "manifest validation passed", + "details": "\n".join(errors), + "log_path": _rel(artifact_dir / "feature-coverage.json"), + } + ) + + checks.append(_run_command("whitespace", _git_diff_check_command(), artifact_dir)) + + compile_targets = [ + target + for target in [ + "seedemu", + "tests/control_plane", + "tests/ci", + ] + if (REPO_ROOT / target).exists() + ] + example_dirs = sorted( + { + str(Path(manifest["examples"][example_id]["script"]).parent) + for example_id in _example_ids(manifest, "compile") + } + ) + compile_targets.extend(example_dirs) + checks.append( + _run_command( + "compileall", + [ + sys.executable, + "-m", + "compileall", + "-q", + "-x", + "seedemu/services/EthereumService/EthTemplates/", + ] + + compile_targets, + artifact_dir, + ) + ) + + smoke = ( + "import seedemu; " + "from seedemu.layers import Base, Routing, Ebgp, Ibgp, Ospf; " + "from seedemu.services import BgpLookingGlassService; " + "from seedemu.compiler import Docker; " + "import tests.ci.run_ci" + ) + checks.append(_run_command("import-smoke", [sys.executable, "-c", smoke], artifact_dir, env=_pythonpath_env())) + return _stage_result("static", checks, artifact_dir) + + +def run_unit(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["unit_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"pytest:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=900)) + return _stage_result("unit", checks, artifact_dir) + + +def run_example_compile(artifact_dir: Path) -> int: + manifest = load_manifest() + checks = [ + _compile_example(example_id, manifest["examples"][example_id], artifact_dir, clean=True) + for example_id in _example_ids(manifest, "compile") + ] + return _stage_result("example-compile", checks, artifact_dir) + + +def run_example_build(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for example_id in _example_ids(manifest, "build"): + example = manifest["examples"][example_id] + compile_check = _compile_example( + example_id, + example, + artifact_dir, + clean=True, + name_prefix="compile-before-build", + ) + checks.append(compile_check) + if compile_check["status"] != "passed": + continue + + script = REPO_ROOT / example["script"] + compose_file = script.parent / "output" / "docker-compose.yml" + checks.append( + _run_command( + f"docker-build:{example_id}", + ["docker", "compose", "-f", str(compose_file), "build"], + artifact_dir, + env=_pythonpath_env(example.get("env", {})), + timeout=int(example.get("build_timeout", 1800)), + ) + ) + return _stage_result("example-build", checks, artifact_dir) + + +def run_runtime_integration(artifact_dir: Path) -> int: + manifest = load_manifest() + checks: list[dict[str, Any]] = [] + for group_id, group in manifest["runtime_groups"].items(): + junit_path = artifact_dir / f"pytest-{_slug(group_id)}.xml" + cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + f"--junitxml={junit_path}", + ] + list(group["pytest_args"]) + checks.append(_run_command(f"runtime:{group_id}", cmd, artifact_dir, env=_pytest_env(), timeout=7200)) + return _stage_result("runtime-integration", checks, artifact_dir) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run feature-oriented SEED Emulator CI stages.") + parser.add_argument( + "stage", + choices=["static", "unit", "example-compile", "example-build", "runtime-integration"], + ) + parser.add_argument("--artifact-dir", default="ci-artifacts", help="Directory for logs, JSON, and JUnit output.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + artifact_dir = (REPO_ROOT / args.artifact_dir).resolve() + artifact_dir.mkdir(parents=True, exist_ok=True) + if args.stage == "static": + return run_static(artifact_dir) + if args.stage == "unit": + return run_unit(artifact_dir) + if args.stage == "example-compile": + return run_example_compile(artifact_dir) + if args.stage == "example-build": + return run_example_build(artifact_dir) + if args.stage == "runtime-integration": + return run_runtime_integration(artifact_dir) + raise AssertionError(f"unknown stage: {args.stage}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/compile-and-build-test/README.md b/tests/compile-and-build-test/README.md index 1a0b4e7fc..2e45cf19c 100644 --- a/tests/compile-and-build-test/README.md +++ b/tests/compile-and-build-test/README.md @@ -6,8 +6,9 @@ The `compile-and-build-test.py` script runs all examples listed in the `test_lis cls.test_list = { "basic/A00_simple_as": (["simple_as.py"], ["output"]), "basic/A01_transit_as": (["transit_as.py"], ["output"]), - "basic/A02_transit_as_mpls": (["transit_as_mpls.py"], ["output"]), - "basic/A03_real_world": (["real_world.py"], ["output"]), + "basic/A02a_transit_as_mpls": (["transit_as_mpls.py"], ["output"]), + "basic/A03a_out_to_real_world": (["out_to_real_world.py"], ["output"]), + "basic/A03b_from_real_world": (["from_real_world.py"], ["output"]), "basic/A04_visualization": (["visualization.py"], ["output", "base_component.bin"]), "basic/A05_components": (["components.py"], ["output", "base_component.bin"]), "basic/A06_merge_emulation": (["merge_emulation.py"], ["output"]), @@ -22,4 +23,4 @@ The script also executes `docker compose build` if any expected outcome includes ## Usage ```python ./compile-and-build-test.py amd|arm -``` \ No newline at end of file +``` diff --git a/tests/compile-and-build-test/compile-and-build-test.py b/tests/compile-and-build-test/compile-and-build-test.py index d306f2881..896acbf3b 100755 --- a/tests/compile-and-build-test/compile-and-build-test.py +++ b/tests/compile-and-build-test/compile-and-build-test.py @@ -31,8 +31,9 @@ def setUpClass(cls) -> None: basic_test_list = { "basic/A00_simple_as": (["simple_as.py"] , ["output"]), "basic/A01_transit_as" : (["transit_as.py"], ["output"]), - "basic/A02_transit_as_mpls" : (["transit_as_mpls.py"], ["output"]), - "basic/A03_real_world" : (["real_world.py"], ["output"]), + "basic/A02a_transit_as_mpls" : (["transit_as_mpls.py"], ["output"]), + "basic/A03a_out_to_real_world" : (["out_to_real_world.py"], ["output"]), + "basic/A03b_from_real_world" : (["from_real_world.py"], ["output"]), "basic/A04_visualization" : (["visualization.py"], ["output", "base_component.bin"]), "basic/A05_components" : (["components.py"], ["output", "base_component.bin"]), "basic/A06_merge_emulation" : (["merge_emulation.py"], ["output"]), @@ -52,11 +53,11 @@ def setUpClass(cls) -> None: "internet/B05_hybrid_internet_with_dns" : (["hybrid_internet_with_dns.py"], ["output", "base_hybrid_component.bin", "hybrid_dns_component.bin"]), "internet/B20_dhcp" : (["dhcp.py"], ["output", "base_internet.bin"]), "internet/B21_etc_hosts": (["etc_hosts.py"], ["output", "base_internet.bin"]), - "internet/B22_botnet": (["botnet_basic.py"], ["output", "base_internet.bin"]), + #"internet/B22_botnet": (["botnet_basic.py"], ["output", "base_internet.bin"]), #"internet/B23_darknet_tor": (["darknet_tor.py"], ["output", "base_internet.bin"]), "internet/B24_ip_anycast": (["ip_anycast.py"], ["output", "base_internet.bin"]), "internet/B25_pki": (["pki.py", "pki_with_dns.py"], ["output", "base_internet.bin", "base_internet_dns.bin"]), - "internet/B26_ipfs_kubo": (["kubo.py"], ["output", "base_component.bin"]), + #"internet/B26_ipfs_kubo": (["kubo.py"], ["output", "base_component.bin"]), #"internet/B27_ipfs_kubo_dapp": (["kubo-eth.py"], ["output", "base_component.bin"]), "internet/B28_traffic_generator/0-iperf-traffic-generator": (["iperf-traffic-generator.py"], ["output", "base_internet.bin"]), "internet/B28_traffic_generator/1-ditg-traffic-generator": (["ditg-traffic-generator.py"], ["output", "base_internet.bin"]), diff --git a/tests/ethereum/POS/EthereumPOSTestCase.py b/tests/ethereum/POS/EthereumPOSTestCase.py index 9a1da9ec7..0e073c448 100755 --- a/tests/ethereum/POS/EthereumPOSTestCase.py +++ b/tests/ethereum/POS/EthereumPOSTestCase.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 # encoding: utf-8 -import time import unittest as ut from .SEEDBlockchain import Wallet from seedemu import * import time -import docker +import requests from tests import SeedEmuTestCase class EthereumPOSTestCase(SeedEmuTestCase): @@ -14,22 +13,18 @@ class EthereumPOSTestCase(SeedEmuTestCase): def setUpClass(cls) -> None: super().setUpClass() + cls.rpc_url = 'http://10.150.0.72:8545' + cls.beacon_api_url = 'http://10.152.0.71:8000' + # Target block delta for POS cls.wallet1 = Wallet(chain_id=1337) for name in ['Alice', 'Bob', 'Charlie', 'David', 'Eve']: cls.wallet1.createAccount(name) - - cls.wallet2 = Wallet(chain_id=1337) - cls.wallet3 = Wallet(chain_id=1337) - cls.result = True return - - + # Test POS geth connection def test_pos_geth_connection(self): - url_1 = 'http://10.151.0.72:8545' - url_2 = 'http://10.152.0.72:8545' - url_3 = 'http://10.153.0.72:8545' + url = self.rpc_url i = 1 current_time = time.time() while True: @@ -37,84 +32,345 @@ def test_pos_geth_connection(self): if time.time() - current_time > 600: self.printLog("TimeExhausted: 600 sec") try: - self.wallet1.connectToBlockchain(url_1, isPOA=True) - self.printLog("Connection Succeed: ", url_1) - self.wallet2.connectToBlockchain(url_2, isPOA=True) - self.printLog("Connection Succeed: ", url_2) - self.wallet3.connectToBlockchain(url_3, isPOA=True) - self.printLog("Connection Succeed: ", url_3) + self.wallet1.connectToBlockchain(url, isPOA=True) + self.printLog("Connection Succeed: ", url) break except Exception as e: self.printLog(e) time.sleep(20) i += 1 self.assertTrue(self.wallet1._web3.isConnected()) - self.assertTrue(self.wallet2._web3.isConnected()) - self.assertTrue(self.wallet3._web3.isConnected()) - - - def test_pos_contract_deployment(self): - client = docker.from_env() - all_containers = client.containers.list() - beacon_setup_container:container - contract_deployed = False - for container in all_containers: - labels = container.attrs['Config']['Labels'] - if 'BeaconSetup' in labels.get('org.seedsecuritylabs.seedemu.meta.displayname', ''): - beacon_setup_container = container - web3_list = [(self.wallet1._url, self.wallet1._web3), - (self.wallet2._url, self.wallet2._web3), - (self.wallet3._url, self.wallet3._web3) ] - + + # Test POS beacon connection + def test_pos_beacon_connection(self): + base_url = self.beacon_api_url + + i = 1 + current_time = time.time() while True: - latestBlockNumber = self.wallet1._web3.eth.getBlock('latest').number - self.printLog("\n----------------------------------------") - self.printLog("Waiting for contract deployment...") - self.printLog("current blockNumber : ", latestBlockNumber) - for ip, web3 in web3_list: - self.printLog("Total in txpool: {} ({})".format(len(web3.geth.txpool.content().pending), ip)) - if latestBlockNumber == 20: - break - result = str(beacon_setup_container.exec_run("cat contract_address.txt").output, 'utf-8') - if 'Deposit contract address:' in result: - self.printLog("Deposit Succeed") - self.printLog(result) - contract_deployed = True + self.printLog("\n----------Trial {}----------".format(i)) + if time.time() - current_time > 600: + self.fail("TimeExhausted: 600 sec") + try: + version_url = "{}/eth/v1/node/version".format(base_url) + + version_resp = requests.get(version_url, timeout=5) + version_data = version_resp.json() + version_str = str(version_data.get("data", {}).get("version", "")) + self.assertIn("lighthouse", version_str.lower()) + self.printLog("Beacon API reachable: {}".format(base_url)) break - time.sleep(10) - self.assertTrue(contract_deployed) + except Exception as e: + self.printLog(e) + time.sleep(20) + i += 1 + return + + def test_block_number_increases(self): + url = self.rpc_url + try: + if (not hasattr(self.wallet1, "_web3")) or (not self.wallet1._web3.isConnected()): + self.wallet1.connectToBlockchain(url, isPOA=True) + except Exception as e: + self.fail("failed to connect execution RPC {}: {}".format(url, e)) + + start_bn = self.wallet1._web3.eth.block_number + #wait for blocknumber increase 5 blocks + target_bn = start_bn + 5 + self.printLog("Waiting for blocks: {} -> {}".format(start_bn, target_bn)) - def test_pos_chain_merged(self): start_time = time.time() - isMerged = False - self.printLog("\n----------------------------------------") - self.printLog("Terminal total difficulty is set to 30.") - self.printLog("The blockNumber will not increase from 15, if the merge is failed.") - self.printLog("So we will assume that the blockNumber is over 17, the merge is succeed.") - self.printLog("----------------------------------------") + last_bn = start_bn + while time.time() - start_time < 300: + time.sleep(6) + cur_bn = self.wallet1._web3.eth.block_number + if cur_bn != last_bn: + self.printLog("block number: {} (+ {})".format(cur_bn, cur_bn - start_bn)) + last_bn = cur_bn + if cur_bn >= target_bn: + self.assertGreaterEqual(cur_bn, target_bn) + return + self.fail("block number did not reach target delta") + + def test_peer_counts(self): + peer_counts = len(self.wallet1._web3.geth.admin.peers()) + self.assertEqual(peer_counts, 4) + + def test_validators_status(self): + base_url = self.beacon_api_url + url = "{}/eth/v1/beacon/states/head/validators".format(base_url) + + i = 1 + current_time = time.time() while True: - latestBlockNumber = self.wallet1._web3.eth.getBlock('latest').number - self.printLog("current blockNumber : ", latestBlockNumber) - if latestBlockNumber > 17: - isMerged = True - break - if time.time() - start_time > 600: - break - time.sleep(20) - self.assertTrue(isMerged) + self.printLog("\n----------Trial {}----------".format(i)) + if time.time() - current_time > 600: + self.fail("TimeExhausted: 600 sec") + try: + resp = requests.get(url, params={"status": "active"}, timeout=10) + self.assertTrue( + 200 <= resp.status_code < 300, + "Validators endpoint failed: {} {}".format(resp.status_code, resp.text), + ) + payload = resp.json() + validators = payload.get("data", []) + self.assertEqual(len(validators), 15) + balances_1 = {} + for v in validators: + status = str(v.get("status", "")).lower() + self.assertTrue(status.startswith("active")) + vid = str(v.get("index", "")) + bal = v.get("balance", None) + self.assertTrue(bal is not None) + balances_1[vid] = int(bal) + self.printLog("Active validators: {}".format(len(validators))) + + time.sleep(12) + resp2 = requests.get(url, params={"status": "active"}, timeout=10) + self.assertTrue( + 200 <= resp2.status_code < 300, + "Validators endpoint failed: {} {}".format(resp2.status_code, resp2.text), + ) + payload2 = resp2.json() + validators2 = payload2.get("data", []) + self.assertEqual(len(validators2), 15) + balances_2 = {} + for v in validators2: + status = str(v.get("status", "")).lower() + self.assertTrue(status.startswith("active")) + vid = str(v.get("index", "")) + bal = v.get("balance", None) + self.assertTrue(bal is not None) + balances_2[vid] = int(bal) + + changed = False + for vid, bal1 in balances_1.items(): + bal2 = balances_2.get(vid) + if bal2 is None: + continue + if bal1 != bal2: + changed = True + self.printLog("validator balance changed: {} -> {}".format(bal1, bal2)) + break + + if changed: + return + raise Exception("validator balances did not change yet") + except Exception as e: + self.printLog(e) + time.sleep(20) + i += 1 def test_pos_send_transaction(self): + time.sleep(10) + balance_rpc_url = "http://10.151.0.72:8545" + balance_wallet = Wallet(chain_id=1337) + balance_wallet.connectToBlockchain(balance_rpc_url, isPOA=True) + recipient = self.wallet1.getAccountAddressByName('Bob') + + recipient_before = balance_wallet._web3.eth.get_balance(recipient) + + value_wei = 100000000000000000 txhash = self.wallet1.sendTransaction(recipient, 0.1, sender_name='David', wait=True, verbose=False) - self.assertTrue(self.wallet1.getTransactionReceipt(txhash)["status"], 1) + receipt = self.wallet1.getTransactionReceipt(txhash) + self.assertEqual(receipt.get("status"), 1) + + start_time = time.time() + while True: + recipient_after = balance_wallet._web3.eth.get_balance(recipient) + if recipient_after - recipient_before == value_wei: + break + if time.time() - start_time > 60: + break + time.sleep(3) + + recipient_after = balance_wallet._web3.eth.get_balance(recipient) + + self.assertEqual(recipient_after - recipient_before, value_wei) + + def test_beacon_slot_increases(self): + base_url = self.beacon_api_url + url = "{}/eth/v1/beacon/headers/head".format(base_url) + + def _get_slot() -> int: + resp = requests.get(url, timeout=10) + self.assertTrue( + 200 <= resp.status_code < 300, + "Beacon headers endpoint failed: {} {}".format(resp.status_code, resp.text), + ) + payload = resp.json() + slot_str = ( + payload.get("data", {}) + .get("header", {}) + .get("message", {}) + .get("slot", "0") + ) + return int(slot_str) + + i = 1 + current_time = time.time() + while True: + self.printLog("\n----------Trial {}----------".format(i)) + if time.time() - current_time > 600: + self.fail("TimeExhausted: 600 sec") + try: + start_slot = _get_slot() + target_slot = start_slot + 5 + self.printLog("Waiting for slot: {} -> {}".format(start_slot, target_slot)) + + start_time = time.time() + last_slot = start_slot + while time.time() - start_time < 180: + time.sleep(6) + cur_slot = _get_slot() + if cur_slot != last_slot: + self.printLog("slot: {} (+ {})".format(cur_slot, cur_slot - start_slot)) + last_slot = cur_slot + if cur_slot >= target_slot: + self.assertGreaterEqual(cur_slot, target_slot) + return + self.fail("slot did not increase") + except Exception as e: + self.printLog(e) + time.sleep(20) + i += 1 + + def test_faucet_static_fund(self): + fund_address = "0x40e38EF94ab2bC9506167D478821ffd55ff2d88d" + # Set maximum number of attempts + max_attempts = 10 + attempts = 0 + + while attempts < max_attempts: + if self.wallet1._web3.eth.getBalance(fund_address) >= 2*EthUnit.ETHER.value: + break + else: + # Increment the attempts counter + attempts += 1 + # Wait for a short duration before trying again (e.g., 5 seconds) + time.sleep(10) + self.assertTrue(self.wallet1._web3.eth.getBalance(fund_address) >= 2*EthUnit.ETHER.value) + + def test_faucet_dynamic_fund(self): + max_attempts = 20 + attempts = 0 + + while attempts < max_attempts: + try: + # Send the POST request + response = requests.get('http://10.164.0.71:80/') + # Check if the request was successful (status code 200) + if response.status_code == 200: + print("POST request successful") + break # Exit the loop if successful + else: + print(f"POST request failed with status code {response.status_code}") + except requests.exceptions.RequestException as e: + print(f"Error: {e}") + + # Increment the attempts counter + attempts += 1 + # Wait for a short duration before trying again (e.g., 5 seconds) + time.sleep(10) + fund_address = "0x9e4f73dE97FEB05FE4e3c0d42B92585C9A0c0E91" + + # Define the URL of faucet API endpoint + url = 'http://10.164.0.71:80/fundme' + # Define the parameters to send in the POST request + params = {'address': fund_address, 'amount': 5} + # Send the POST request + response = requests.post(url, data=params) + print(response) + while attempts < max_attempts: + if self.wallet1._web3.eth.getBalance(fund_address) >= 5*EthUnit.ETHER.value: + break + else: + # Increment the attempts counter + attempts += 1 + # Wait for a short duration before trying again (e.g., 5 seconds) + time.sleep(10) + + self.assertTrue(self.wallet1._web3.eth.getBalance(fund_address) >= 5*EthUnit.ETHER.value) + + def test_beacon_peers(self): + ''' + Test the connection status of beacon peers; it is considered normal + when the number of consensus-layer peers is greater than or equal to 1. + ''' + base_url = self.beacon_api_url + url = "{}/eth/v1/node/peers".format(base_url) + + i = 1 + current_time = time.time() + while True: + self.printLog("\n----------Trial {}----------".format(i)) + if time.time() - current_time > 600: + self.fail("TimeExhausted: 600 sec") + try: + resp = requests.get(url, timeout=10) + self.assertTrue( + 200 <= resp.status_code < 300, + "Peers endpoint failed: {} {}".format(resp.status_code, resp.text), + ) + payload = resp.json() + peers = payload.get("data", []) + self.assertGreaterEqual(len(peers), 1) + connected_peers = 0 + for p in peers: + state = str(p.get("state", "")).lower() + if state == "connected": + connected_peers += 1 + self.assertGreaterEqual(connected_peers, 1) + self.printLog("Beacon peers: total={}, connected={}".format(len(peers), connected_peers)) + return + except Exception as e: + self.printLog(e) + time.sleep(20) + i += 1 + + def test_utility(self): + + max_attempts = 20 + utility_url = "http://10.164.0.72:5000" + + contract_data = { + "contract_name": "test999", + "contract_address": "0xc0ffee254729296a45a3885639AC7E10F9d54979" + } + + + register_resp = requests.post("{}/register_contract".format(utility_url), json=contract_data, timeout=10) + self.assertTrue(200 <= register_resp.status_code < 300, "Failed to register contract: {} {}".format(register_resp.status_code, register_resp.text)) + + expected_address = contract_data["contract_address"] + attempts = 0 + while attempts < max_attempts: + list_resp = requests.get("{}/contracts_info".format(utility_url), timeout=10) + if 200 <= list_resp.status_code < 300: + contracts_info = list_resp.json() + if contracts_info.get("test999") == expected_address: + self.printLog("Contract registration and retrieval successful.") + return + time.sleep(3) + attempts += 1 + + self.fail("Registered contract not found in /contracts_info at {}".format(utility_url)) @classmethod def get_test_suite(cls): test_suite = ut.TestSuite() test_suite.addTest(cls('test_pos_geth_connection')) - test_suite.addTest(cls('test_pos_contract_deployment')) - test_suite.addTest(cls('test_pos_chain_merged')) + test_suite.addTest(cls('test_pos_beacon_connection')) + test_suite.addTest(cls('test_validators_status')) + test_suite.addTest(cls('test_block_number_increases')) + test_suite.addTest(cls('test_beacon_slot_increases')) + test_suite.addTest(cls('test_beacon_peers')) + test_suite.addTest(cls('test_peer_counts')) test_suite.addTest(cls('test_pos_send_transaction')) + test_suite.addTest(cls('test_faucet_static_fund')) + test_suite.addTest(cls('test_faucet_dynamic_fund')) + test_suite.addTest(cls('test_utility')) return test_suite if __name__ == "__main__": test_suite = EthereumPOSTestCase.get_test_suite() diff --git a/tests/ethereum/POS/emulator-code/test-emulator.py b/tests/ethereum/POS/emulator-code/test-emulator.py index d44ca9d07..c5044c6d9 100755 --- a/tests/ethereum/POS/emulator-code/test-emulator.py +++ b/tests/ethereum/POS/emulator-code/test-emulator.py @@ -2,94 +2,140 @@ # encoding: utf-8 from seedemu import * -import os - -# Create Emulator Base with 10 Stub AS (150-154, 160-164) using Makers utility method. -# hosts_per_stub_as=3 : create 3 hosts per one stub AS. -# It will create hosts(physical node) named `host_{}`.format(counter), counter starts from 0. -hosts_per_stub_as = 2 -emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as = hosts_per_stub_as) - -# Create the Ethereum layer -eth = EthereumService() - -# Create the Blockchain layer which is a sub-layer of Ethereum layer. -# chainName="pos": set the blockchain name as "pos" -# consensus="ConsensusMechnaism.POS" : set the consensus of the blockchain as "ConsensusMechanism.POS". -# supported consensus option: ConsensusMechanism.POA, ConsensusMechanism.POW, ConsensusMechanism.POS -blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) - -# set `terminal_total_difficulty`, which is the value to designate when the Merge is happen. -blockchain.setTerminalTotalDifficulty(30) - -asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] - -################################################### -# Ethereum Node - -i = 1 -for asn in asns: - for id in range(hosts_per_stub_as): - # Create a blockchain virtual node named "eth{}".format(i) - e:EthereumServer = blockchain.createNode("eth{}".format(i)) - - # Create Docker Container Label named 'Ethereum-POS-i' - e.appendClassName('Ethereum-POS-{}'.format(i)) - - # Enable Geth to communicate with geth node via http - e.enableGethHttp() - - # Set host in asn 150 with id 0 (ip : 10.150.0.71) as BeaconSetupNode. - if asn == 150 and id == 0: - e.setBeaconSetupNode() - - # Set host in asn 150 with id 1 (ip : 10.150.0.72) as BootNode. - # This node will serve as a BootNode in both execution layer (geth) and consensus layer (lighthouse). - if asn == 150 and id == 1: - e.setBootNode(True) - - # Set hosts in asn 152 and 162 with id 0 and 1 as validator node. - # Validator is added by deposit 32 Ethereum and is activated in realtime after the Merge. - # isManual=True : deposit 32 Ethereum by manual. - # Other than deposit part, create validator key and running a validator node is done by codes. - if asn in [151]: - if id == 0: - e.enablePOSValidatorAtRunning() - if id == 1: - e.enablePOSValidatorAtRunning(is_manual=True) - - # Set hosts in asn 152, 153, 154, and 160 as validator node. - # These validators are activated by default from genesis status. - # Before the Merge, when the consensus in this blockchain is still POA, - # these hosts will be the signer nodes. - if asn in [152,153,154,160,161,162,163,164]: - e.enablePOSValidatorAtGenesis() - e.startMiner() - - # Customizing the display names (for visualiztion purpose) - if e.isBeaconSetupNode(): - emu.getVirtualNode('eth{}'.format(i)).setDisplayName('Ethereum-BeaconSetup') +import sys, os +import math +from eth_account import Account + +DOMAIN = ".net" + +def run(dumpfile = None, total_beacon_nodes=5, vc_per_beacon=3): + ############################################################################### + # Set the platform information + if dumpfile is None: + script_name = os.path.basename(__file__) + if len(sys.argv) == 1: + platform = Platform.AMD64 + elif len(sys.argv) == 2: + if sys.argv[1].lower() == 'amd': + platform = Platform.AMD64 + elif sys.argv[1].lower() == 'arm': + platform = Platform.ARM64 + else: + print(f"Usage: {script_name} amd|arm") + sys.exit(1) else: - emu.getVirtualNode('eth{}'.format(i)).setDisplayName('Ethereum-POS-{}'.format(i)) - - # Binding the virtual node to the physical node. - emu.addBinding(Binding('eth{}'.format(i), filter=Filter(asn=asn, nodeName='host_{}'.format(id)))) - - i = i+1 - - -# Add layer to the emulator -emu.addLayer(eth) - -emu.render() - -# Access an environment variable -platform = os.environ.get('platform') - -platform_mapping = { - "amd": Platform.AMD64, - "arm": Platform.ARM64 -} -docker = Docker(platform=platform_mapping[platform]) - -emu.compile(docker, './output', override = True) + print(f"Usage: {script_name} amd|arm") + sys.exit(1) + + # Configure the number of nodes + geth_node_number = total_beacon_nodes + beacon_node_number = total_beacon_nodes + vc_node_number = vc_per_beacon * beacon_node_number + # Create Emulator Base with 10 Stub AS (150-154, 160-164) using Makers utility method. + # hosts_per_stub_as=3 : create 3 hosts per one stub AS. + # It will create hosts(physical node) named `host_{}`.format(counter), counter starts from 0. + hosts_per_stub_as = 3 + emu = Makers.makeEmulatorBaseWith10StubASAndHosts(hosts_per_stub_as = hosts_per_stub_as) + + # Create the Ethereum layer + eth = EthereumService() + # Create the Blockchain layer which is a sub-layer of Ethereum layer. + # chainName="pos": set the blockchain name as "pos" + # consensus="ConsensusMechnaism.POS" : set the consensus of the blockchain as "ConsensusMechanism.POS". + # supported consensus option: ConsensusMechanism.POA, ConsensusMechanism.POW, ConsensusMechanism.POS + blockchain = eth.createBlockchain(chainName="pos", consensus=ConsensusMechanism.POS) + # Generate a list of accounts and prefund them + + + asns = [150, 151, 152, 153, 154, 160, 161, 162, 163, 164] + geth_nodes: List[PoSGethServer] = [] + beacon_nodes: List[PoSBeaconServer] = [] + vc_nodes: List[PoSVcServer] =[] + # Create the BeaconSetupNode + beaconsetupServer: PoSBeaconSetupServer = blockchain.createBeaconSetupNode(f"BeaconSetupNode") + emu.getVirtualNode(f'BeaconSetupNode').setDisplayName('Ethereum-POS-BeaconSetup') + for i in range(geth_node_number): + gethServer: PoSGethServer = blockchain.createGethNode(f"gethnode{i}") + gethServer.enableGethHttp() + gethServer.appendClassName(f'Ethereum-POS-Geth-{i+1}') + gethServer.addHostName(f"gethnode{i}" + DOMAIN) + geth_nodes.append(gethServer) + emu.getVirtualNode(f'gethnode{i}').setDisplayName(f'Ethereum-POS-Geth-{i+1}') + + # Create Beacon nodes and connect to Geth nodes + for i in range(beacon_node_number): + beaconServer: PoSBeaconServer = blockchain.createBeaconNode(f"beaconnode{i}") + beaconServer.appendClassName(f'Ethereum-POS-Beacon-{i+1}') + beaconServer.connectToGethNode(f"gethnode{(i+1)%len(geth_nodes)}") + beacon_nodes.append(beaconServer) + emu.getVirtualNode(f'beaconnode{i}').setDisplayName(f'Ethereum-POS-Beacon-{i+1}') + + # Set boot nodes + geth_nodes[0].setBootNode(True) + beacon_nodes[0].setBootNode(True) + + # Create Validator nodes and connect to Beacon nodes + for i in range(vc_node_number): + VcServer: PoSVcServer=blockchain.createVcNode(f"vcnode{i}") + VcServer.appendClassName(f'Ethereum-POS-Validator-{i+1}') + VcServer.connectToBeaconNode(f"beaconnode{(i+1)%len(beacon_nodes)}") + VcServer.enablePOSValidatorAtGenesis() + vc_nodes.append(VcServer) + emu.getVirtualNode(f'vcnode{i}').setDisplayName(f'Ethereum-POS-Validator-{i+1}') + # Create the Validator node and enablePOSValidatorAtRunning + # and connect to the Beacon node + vc_at_running_1: PoSVcServer = blockchain.createVcNode("vcnodeAtRunning") + vc_at_running_1.appendClassName("Ethereum-POS-Validator-AtRunning") + vc_at_running_1.connectToBeaconNode("beaconnode0") + vc_at_running_1.enablePOSValidatorAtRunning() + vc_nodes.append(vc_at_running_1) + emu.getVirtualNode("vcnodeAtRunning").setDisplayName("Ethereum-POS-Validator-AtRunning-1") + # Create the Faucet server + faucet:FaucetServer = blockchain.createFaucetServer( + vnode='faucet', + port=80, + linked_eth_node='gethnode1', + balance=10000, + max_fund_amount=10) + faucet.fund('0x40e38EF94ab2bC9506167D478821ffd55ff2d88d',2) + + faucet.setDisplayName('Faucet') + faucet.addHostName('faucet' + DOMAIN) + # Create the Utility server + util_server:EthUtilityServer = blockchain.createEthUtilityServer( + vnode='utility', + port=5000, + linked_eth_node='gethnode2', + linked_faucet_node='faucet') + util_server.setDisplayName('UtilityServer') + util_server.addHostName('utility' + DOMAIN) + + # Add Ethereum service to the emulator + emu.addLayer(eth) + + # Bind all virtual servers (including faucet/utility) to physical hosts + for _, servers in blockchain.getAllServerNames().items(): + for server in servers: + emu.addBinding(Binding(server, filter=Filter(nodeName="host_*"), + action=Action.FIRST)) + + # Add /etc/hosts layer + emu.addLayer(EtcHosts()) + + # Configure IP range for each AS + base_layer = emu.getLayer('Base') + for asn in asns: + as_obj = base_layer.getAutonomousSystem(asn) + net = as_obj.getNetwork('net0') + net.setHostIpRange(hostStart=71, hostEnd=199, hostStep=1) + + # Generate the emulator output + if dumpfile is not None: + emu.dump(dumpfile) + else: + emu.render() + docker = Docker(internetMapEnabled=True, etherViewEnabled=True, platform=platform) + emu.compile(docker, './output', override=True) + +if __name__ == "__main__": + run() diff --git a/tests/internet/mini_internet/MiniInternetTestCase.py b/tests/internet/mini_internet/MiniInternetTestCase.py index 3f67e1e8c..a2b7bec51 100755 --- a/tests/internet/mini_internet/MiniInternetTestCase.py +++ b/tests/internet/mini_internet/MiniInternetTestCase.py @@ -2,6 +2,7 @@ # encoding: utf-8 import unittest as ut +import os from tests import SeedEmuTestCase class MiniInternetTestCase(SeedEmuTestCase): @@ -30,6 +31,11 @@ def test_customized_ip_address(self): self.assertTrue(self.ping_test(self.source_host, "10.154.0.129")) def test_real_world_as(self): + if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("SEED_EMU_RUN_REAL_WORLD_TESTS") != "true": + self.printLog("\n-------- real world as test --------") + self.printLog("skipping real world AS HTTP test in CI; set SEED_EMU_RUN_REAL_WORLD_TESTS=true to enable") + self.skipTest("real external network reachability is not a stable PR gate") + self.printLog("\n-------- real world as test --------") self.printLog("real world as 11872") self.printLog("check real world ip : 128.230.18.63") diff --git a/tests/internet/mini_internet/README.md b/tests/internet/mini_internet/README.md index 6766a3c29..748d24d7f 100644 --- a/tests/internet/mini_internet/README.md +++ b/tests/internet/mini_internet/README.md @@ -33,4 +33,9 @@ In this test script, it comprises 8 test cases: (1) `test_internet_connection`, Testcase (1) `test_internet_connection` ping to all other containers and test it is not failed. Testcase (2) `test_customized_ip_address` ping to cutomized ip and test if a ip customization works. -Testcase (3) `test_real_world_as` ping to a real world ip that belongs to the enable real world AS (11872). \ No newline at end of file +Testcase (3) `test_real_world_as` ping to a real world ip that belongs to the enable real world AS (11872). + +In GitHub Actions, `test_real_world_as` is skipped by default because it +depends on external network reachability and a live remote HTTP response. To +include it in a manual CI run, enable the `run-real-world-tests` workflow +dispatch option or set `SEED_EMU_RUN_REAL_WORLD_TESTS=true`. diff --git a/tests/run-tests.py b/tests/run-tests.py index 996ff9caf..adac9d6b8 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -30,10 +30,10 @@ blockchain_tests = [ EthereumPOATestCase, - EthereumPOSTestCase, - EthereumPOWTestCase, - ChainlinkPOATestCase, - EthUtilityPOATestCase + EthereumPOSTestCase + # EthereumPOWTestCase, + # ChainlinkPOATestCase, + # EthUtilityPOATestCase ] scion_tests = [ diff --git a/tools/DemoSystem/Dockerfile b/tools/DemoSystem/Dockerfile index 79565897d..1a364f945 100644 --- a/tools/DemoSystem/Dockerfile +++ b/tools/DemoSystem/Dockerfile @@ -1,20 +1,19 @@ FROM node:22-slim AS builder -WORKDIR /build -COPY . . +RUN npm install -g pnpm@9.15.9 WORKDIR /build/frontend -RUN npm install -g pnpm +COPY frontend/package.json frontend/pnpm-lock.yaml* ./ RUN pnpm i +COPY frontend/ . RUN pnpm run build-dev FROM node:22-slim -WORKDIR /usr/src/app -COPY . . -COPY --from=builder /build/frontend/dist ./frontend/dist +COPY start.sh / +RUN chmod +x /start.sh WORKDIR /usr/src/app/backend +COPY backend/package.json backend/pnpm-lock.yaml* ./ RUN npm install -RUN npm install -D typescript @types/tar-fs js-yaml @types/js-yaml +COPY backend/ . +COPY --from=builder /build/frontend/dist ../frontend/dist RUN ./node_modules/.bin/tsc -COPY start.sh / -RUN chmod +x /start.sh ENTRYPOINT ["sh", "/start.sh"] \ No newline at end of file diff --git a/tools/DemoSystem/backend/package-lock.json b/tools/DemoSystem/backend/package-lock.json index 1ddbaff2b..f35d470ee 100644 --- a/tools/DemoSystem/backend/package-lock.json +++ b/tools/DemoSystem/backend/package-lock.json @@ -12,12 +12,17 @@ "@types/dockerode": "^3.2.2", "@types/express": "^4.17.11", "@types/express-ws": "^3.0.0", + "@types/js-yaml": "^4.0.9", "@types/node": "^14.10.1", + "@types/tar-fs": "^2.0.4", "@types/ws": "^7.2.6", "dockerode": "^3.2.1", "express": "^4.17.1", + "express-async-errors": "^3.1.1", "express-ws": "^4.0.0", + "js-yaml": "^4.1.1", "tslog": "^3.0.5", + "typescript": "^5.9.3", "ws": "^7.3.1" } }, @@ -77,6 +82,12 @@ "@types/ws": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmmirror.com/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", @@ -106,6 +117,25 @@ "@types/node": "*" } }, + "node_modules/@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, + "node_modules/@types/tar-stream": { + "version": "3.1.4", + "resolved": "https://registry.npmmirror.com/@types/tar-stream/-/tar-stream-3.1.4.tgz", + "integrity": "sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/ws": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.0.tgz", @@ -126,6 +156,12 @@ "node": ">= 0.6" } }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -388,6 +424,15 @@ "node": ">= 0.10.0" } }, + "node_modules/express-async-errors": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/express-async-errors/-/express-async-errors-3.1.1.tgz", + "integrity": "sha512-h6aK1da4tpqWSbyCa3FxB/V6Ehd4EEB15zyQq9qe75OZBp0krinNKuH4rAY+S/U/2I36vdLAUFSjQJ+TFmODng==", + "license": "ISC", + "peerDependencies": { + "express": "^4.16.2" + } + }, "node_modules/express-ws": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-4.0.0.tgz", @@ -525,6 +570,18 @@ "node": ">= 0.10" } }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -892,6 +949,19 @@ "node": ">= 0.6" } }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -992,6 +1062,11 @@ "@types/ws": "*" } }, + "@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmmirror.com/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==" + }, "@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", @@ -1021,6 +1096,23 @@ "@types/node": "*" } }, + "@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "requires": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, + "@types/tar-stream": { + "version": "3.1.4", + "resolved": "https://registry.npmmirror.com/@types/tar-stream/-/tar-stream-3.1.4.tgz", + "integrity": "sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==", + "requires": { + "@types/node": "*" + } + }, "@types/ws": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.0.tgz", @@ -1038,6 +1130,11 @@ "negotiator": "0.6.2" } }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1288,6 +1385,12 @@ } } }, + "express-async-errors": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/express-async-errors/-/express-async-errors-3.1.1.tgz", + "integrity": "sha512-h6aK1da4tpqWSbyCa3FxB/V6Ehd4EEB15zyQq9qe75OZBp0krinNKuH4rAY+S/U/2I36vdLAUFSjQJ+TFmODng==", + "requires": {} + }, "express-ws": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-4.0.0.tgz", @@ -1392,6 +1495,14 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "requires": { + "argparse": "^2.0.1" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -1688,6 +1799,11 @@ "mime-types": "~2.1.24" } }, + "typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1719,4 +1835,4 @@ "integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==" } } -} +} \ No newline at end of file diff --git a/tools/DemoSystem/backend/package.json b/tools/DemoSystem/backend/package.json index 50b525d94..e6f121c58 100644 --- a/tools/DemoSystem/backend/package.json +++ b/tools/DemoSystem/backend/package.json @@ -12,12 +12,17 @@ "@types/dockerode": "^3.2.2", "@types/express": "^4.17.11", "@types/express-ws": "^3.0.0", - "@types/node": "^14.10.1", + "@types/js-yaml": "^4.0.9", + "@types/node": "^22.0.0", + "@types/tar-fs": "^2.0.4", "@types/ws": "^7.2.6", "dockerode": "^3.2.1", "express": "^4.17.1", + "express-async-errors": "^3.1.1", "express-ws": "^4.0.0", + "js-yaml": "^4.1.1", "tslog": "^3.0.5", + "typescript": "^5.9.3", "ws": "^7.3.1" } -} +} \ No newline at end of file diff --git a/tools/DemoSystem/backend/src/api/v1/docker/index.ts b/tools/DemoSystem/backend/src/api/v1/docker/index.ts index 3bc795d7f..0add872e3 100644 --- a/tools/DemoSystem/backend/src/api/v1/docker/index.ts +++ b/tools/DemoSystem/backend/src/api/v1/docker/index.ts @@ -42,11 +42,7 @@ router.post('/exec', async (req, res) => { )).filter(id => id !== "") } if (!containerIds.length) { - dockerOperation.execDockerCommand(cmd).then(r => { - res.json({ok: true, result: r.stdout}) - }).catch(err => { - res.json({ok: false, result: err.message}) - }) + res.json({ok: false, result: `容器未找到 ${containerNames}`}) return } await Promise.all( @@ -76,36 +72,6 @@ router.post('/exec', async (req, res) => { } res.json({ok: true, result: JSON.stringify({succeeded, failed})}); }); -router.post('/compose/exec', async (req, res) => { - let {host, port, composePath, cmd} = req.body as { - host: string; - port: string, - composePath: string, - cmd: string, - }; - - const docker = await dockerOperation.getDocker(); - if (!docker) { - res.json({ok: false, result: "getDocker failed"}); - return - } - try { - composePath = path.join(basePath, composePath) - switch (cmd) { - case "composeUp": - await dockerOperation.composeUp(composePath) - break - case "composeDown": - await dockerOperation.composeDown(composePath) - break - default: - throw new Error(`暂不支持的命令: ${cmd}`) - } - res.json({ok: true, result: ''}); - } catch (e) { - res.json({ok: false, result: e.message as string}) - } -}); router.post('/host/exec', async (req, res) => { let {cmd} = req.body as { cmd: string } const result = await dockerOperation.execOnHost(cmd) diff --git a/tools/DemoSystem/backend/src/api/v1/main.ts b/tools/DemoSystem/backend/src/api/v1/main.ts index 4d13c4704..46026165a 100644 --- a/tools/DemoSystem/backend/src/api/v1/main.ts +++ b/tools/DemoSystem/backend/src/api/v1/main.ts @@ -12,151 +12,5 @@ router.get('/env.js', (req, res, next) => { next(); }); -router.post('/run', async (req, res) => { - const {cmd} = req.body; - - // 更严格的白名单检查 - // const whitelist = ['ls', 'cat', 'pwd', 'whoami', 'docker', 'docker-compose', 'cd', 'echo']; - // const cmdParts = cmd.trim().split(/\s+/); - // const cmdName = cmdParts[0]; - - // // 检查命令是否在白名单中 - // if (!whitelist.includes(cmdName)) { - // return res.status(400).json({ - // ok: false, - // result: `不允许的命令: ${cmdName}` - // }); - // } - - // // 安全检查：防止命令注入 - // const allowedDockerCommands = ['ps', 'images', 'version', 'info', 'compose']; - // if (cmdName === 'docker' || cmdName === 'docker-compose') { - // const subCommand = cmdParts[1]; - // if (!allowedDockerCommands.includes(subCommand) && - // !subCommand?.startsWith('compose')) { - // return res.status(400).json({ - // ok: false, - // result: `不允许的Docker子命令: ${subCommand}` - // }); - // } - // } - - try { - // 使用 spawn 而不是 exec，支持长时间执行 - const childProcess = spawn('sh', ['-c', cmd], { - cwd: process.cwd(), - timeout: 5 * 60 * 1000, // 5分钟超时 - stdio: ['pipe', 'pipe', 'pipe'] // 标准输入、输出、错误 - }); - - // 收集输出 - let stdout = ''; - let stderr = ''; - - childProcess.stdout.on('data', (data) => { - stdout += data.toString(); - // 可选：实时流式输出 - // 如果需要实时推送，可以在这里使用 WebSocket 或 Server-Sent Events - }); - - childProcess.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - // 监听进程退出 - const exitCode = await new Promise((resolve, reject) => { - childProcess.on('close', (code) => { - resolve(code || 0); - }); - - childProcess.on('error', (error) => { - reject(error); - }); - - // 可选：设置超时 - setTimeout(() => { - childProcess.kill('SIGTERM'); // 发送终止信号 - reject(new Error('命令执行超时')); - }, 10 * 60 * 1000); // 10分钟超时 - }); - - if (exitCode === 0) { - return res.json({ - ok: true, - result: stdout.trim(), - // exitCode - }); - } else { - return res.status(500).json({ - ok: false, - result: `命令执行失败: ${stderr.trim() || '未知错误'}`, - }); - } - - } catch (error: any) { - return res.status(500).json({ - ok: false, - result: error.message || '命令执行异常' - }); - } -}); -router.post('/ssh/run', (req, res) => { - const {host, port = 22, user, keyPath, remoteCmd} = req.body; - // 示例白名单，仅允许 ls、cat、df 等安全命令 - const allowed = ['ls', 'cat', 'df', 'uptime']; - const remoteCmdName = remoteCmd.split(' ')[0]; - if (!allowed.includes(remoteCmdName)) { - return res.status(400).json({error: '不允许的远程命令'}); - } - - // 构造 ssh 参数 - const sshArgs = [ - '-i', keyPath, // 私钥路径 - '-p', String(port), - `${user}@${host}`, - remoteCmd - ]; - - const ssh = spawn('ssh', sshArgs, {cwd: process.cwd()}); - - let stdout = ''; - let stderr = ''; - - ssh.stdout.on('data', data => (stdout += data.toString())); - ssh.stderr.on('data', data => (stderr += data.toString())); - - ssh.on('close', code => { - res.json({code, stdout, stderr}); - }); - - ssh.on('error', err => { - res.status(500).json({error: err.message}); - }); -}); -router.post('/stream', (req, res) => { - const {cmd, args = []} = req.body; // 例如：{ "cmd": "ping", "args": ["-c", "4", "8.8.8.8"] } - - const child = spawn(cmd, args, {cwd: process.cwd()}); - - let stdout = ''; - let stderr = ''; - - child.stdout.on('data', data => { - stdout += data.toString(); - // 实时推送给前端（可使用 SSE / WebSocket） - }); - - child.stderr.on('data', data => { - stderr += data.toString(); - }); - - child.on('close', code => { - res.json({code, stdout, stderr}); - }); - - child.on('error', err => { - res.status(500).json({error: err.message}); - }); -}); router.use('/docker', dockerRouter); export default router; diff --git a/tools/DemoSystem/backend/src/main.ts b/tools/DemoSystem/backend/src/main.ts index af80eb953..f3366f842 100644 --- a/tools/DemoSystem/backend/src/main.ts +++ b/tools/DemoSystem/backend/src/main.ts @@ -1,31 +1,23 @@ -import express, { Request, Response } from 'express'; +import 'express-async-errors' +import express, {Request, Response} from 'express'; import path from 'path'; -import expressWs from 'express-ws'; import {errorHandler} from "./utils/tool"; - -const app = express(); -expressWs(app); - import apiV1Router from './api/v1/main'; + const frontendPath = path.resolve(__dirname, '../../frontend/dist/frontend'); +const app = express(); app.use(express.json()); - -app.use(express.urlencoded({ extended: true })); - +app.use(express.urlencoded({extended: true})); app.use('/static', express.static(path.join(frontendPath))); - app.use('/api/v1', apiV1Router); - app.get('/pro/*', (req: Request, res: Response) => { - res.sendFile(path.join(frontendPath, 'index.html')); + res.sendFile(path.join(frontendPath, 'index.html')); }); - app.get('/', (req: Request, res: Response) => { - res.redirect('/pro/home'); + res.redirect('/pro/dashboard/home'); }); - app.use(errorHandler) app.listen(5050, '0.0.0.0'); \ No newline at end of file diff --git a/tools/DemoSystem/backend/src/utils/docker-operation.ts b/tools/DemoSystem/backend/src/utils/docker-operation.ts index 6644acb86..45deec6bb 100644 --- a/tools/DemoSystem/backend/src/utils/docker-operation.ts +++ b/tools/DemoSystem/backend/src/utils/docker-operation.ts @@ -26,16 +26,18 @@ interface ExecResult { } export interface SeedemuConf { - condaPath: string; demoSystem: { hostProjectPath: string - envName: string } } +export interface HostConf { + seedemuConf: SeedemuConf +} + export class DockerOperation implements LogProducer { private readonly _logger: Logger; - public seedemuConf: SeedemuConf | null = null; + public hostConf: HostConf | null = null; constructor() { this._logger = new Logger({name: 'Docker'}); @@ -64,8 +66,7 @@ export class DockerOperation implements LogProducer { return containerInfo!.Id } catch (error) { - this._logger.error('查找容器失败:', error) - this._logger.error(`查找容器失败: ${error}`,) + this._logger.error(`查找容器失败: ${error}`) return '' } } @@ -295,7 +296,7 @@ APPEND_EOF` ret.stdout = output.stdout; this._logger.info(`containerId: ${containerId} Successfully exec cmd: (${cmd}), detach: ${detach}`); } catch (error) { - this._logger.warn(`Error exec cmd: (${cmd}), error: ${error}`); + this._logger.warn(`Warning exec cmd: (${cmd}), error: ${error}`); ret.ok = false ret.stderr = `${error}` } @@ -619,94 +620,11 @@ APPEND_EOF` } } - /** - * 执行原生 Docker 命令（类似 docker ps） - */ - async execDockerCommand(command: string): Promise<{ stdout: string; stderr: string }> { - try { - return await execAsync(`docker ${command}`); - } catch (error: any) { - if (error.stdout || error.stderr) { - return { - stdout: error.stdout || '', - stderr: error.stderr || '' - }; - } - throw new Error(`执行 Docker 命令失败: ${error.message}`); - } - } - - /** - * 执行 Docker Compose 命令 - */ - async execComposeCommand( - composePath: string, - command: string, - serviceName?: string - ): Promise<{ stdout: string; stderr: string }> { - let cleanedStdout, cleanedStderr - try { - let cmd = `cd ${composePath} && ${command}`; - if (serviceName) { - cmd += ` ${serviceName}`; - } - const {stdout, stderr} = await execAsync(cmd, { - timeout: 60000 * 30, - maxBuffer: 1024 * 1024 * 10 // 10MB buffer - }); - cleanedStdout = stdout.trim(); - cleanedStderr = stderr.trim(); - } catch (error: any) { - if (error.stdout || error.stderr) { - cleanedStdout = error.stdout || '' - cleanedStderr = error.stderr || '' - } else { - throw new Error(`执行 Docker Compose 命令失败: ${error.message}`); - } - } - if (!cleanedStderr.includes("level=warning")) { - throw new Error(cleanedStderr) - } - return {stdout: cleanedStdout, stderr: ''} - } - - /** - * 启动指定路径的 Docker Compose 服务 - */ - async composeUp(composePath: string, serviceName?: string): Promise { - // let command = "DOCKER_BUILDKIT=0 docker compose build && docker compose up -d" - let command = "docker compose up -d" - command = serviceName ? `${command} ${serviceName}` : command; - const result = await this.execComposeCommand(composePath, command); - - if (result.stderr && !result.stderr.includes('Creating')) { - throw new Error(result.stderr); - } - - return result.stdout || '服务启动成功'; - } - - /** - * 停止 Docker Compose 服务 - */ - async composeDown(composePath: string): Promise { - const command = "docker compose down" - const result = await this.execComposeCommand(composePath, command); - - if (result.stderr) { - throw new Error(result.stderr); - } - - return result.stdout || '服务停止成功'; - } - async execOnHost(command: string): Promise { - const seedemuConf = await this.getHostSeedemuConf() + const hostConf = await this.getHostConf() command = replaceRelativePath( command, - seedemuConf.demoSystem.hostProjectPath, - seedemuConf.condaPath, - seedemuConf.demoSystem.envName, + hostConf.seedemuConf.demoSystem.hostProjectPath, ) command = `nsenter -t ${HOST_PID} -m -u -i -n -p /bin/sh -c "${command}"`; this._logger.info(`宿主机执行命令: ${command}`); @@ -729,13 +647,14 @@ APPEND_EOF` } } - async getHostSeedemuConf(): Promise { - if (this.seedemuConf === null) { - const command = `nsenter -t ${HOST_PID} -m -u -i -n -p -- cat ${JSON.stringify(SEEDEMU_CONF_FILE_PATH)}`; - const {stdout} = await execAsync(command, {timeout: 6000}) - this.seedemuConf = yaml.load(stdout) as SeedemuConf; + async getHostConf(): Promise { + if (this.hostConf === null) { + let command = `nsenter -t ${HOST_PID} -m -u -i -n -p -- cat ${JSON.stringify(SEEDEMU_CONF_FILE_PATH)}`; + let {stdout} = await execAsync(command, {timeout: 6000}) + this.hostConf = {} as HostConf + this.hostConf.seedemuConf = yaml.load(stdout) as SeedemuConf; } - return this.seedemuConf; + return this.hostConf; } getLoggers(): Logger[] { diff --git a/tools/DemoSystem/backend/src/utils/tool.ts b/tools/DemoSystem/backend/src/utils/tool.ts index 7eaf1b90c..0049f403c 100644 --- a/tools/DemoSystem/backend/src/utils/tool.ts +++ b/tools/DemoSystem/backend/src/utils/tool.ts @@ -11,14 +11,13 @@ export const getBaseDir = (): string => { return path.resolve(baseDir) } -export const replaceRelativePath = (command: string, hostProjectPath: string, condaPath: string, envName: string): string => { +export const replaceRelativePath = (command: string, hostProjectPath: string): string => { return command.replace("$hostProjectPath", hostProjectPath) - .replace("$condaPath", condaPath) - .replace("$envName", envName) } export function errorHandler(err: any, _req: Request, res: Response, _next: NextFunction): void { - const statusCode = err.status || 500; + // const statusCode = err.status || 500; + const statusCode = 200; const message = err.message || '内部服务器错误'; res.status(statusCode).json({ diff --git a/tools/DemoSystem/extensions/assets/img/bgp_exploration.png b/tools/DemoSystem/extensions/assets/img/bgp_exploration.png new file mode 100644 index 000000000..6370f35c6 Binary files /dev/null and b/tools/DemoSystem/extensions/assets/img/bgp_exploration.png differ diff --git a/tools/DemoSystem/extensions/assets/img/large_blockchain.png b/tools/DemoSystem/extensions/assets/img/large_blockchain.png new file mode 100644 index 000000000..70e4d7760 Binary files /dev/null and b/tools/DemoSystem/extensions/assets/img/large_blockchain.png differ diff --git a/tools/DemoSystem/extensions/assets/img/mirai.png b/tools/DemoSystem/extensions/assets/img/mirai.png new file mode 100644 index 000000000..ebc71327b Binary files /dev/null and b/tools/DemoSystem/extensions/assets/img/mirai.png differ diff --git a/tools/DemoSystem/extensions/assets/img/worm.png b/tools/DemoSystem/extensions/assets/img/worm.png new file mode 100644 index 000000000..eb935d075 Binary files /dev/null and b/tools/DemoSystem/extensions/assets/img/worm.png differ diff --git a/tools/DemoSystem/extensions/assets/img/yesterday_once_more.png b/tools/DemoSystem/extensions/assets/img/yesterday_once_more.png new file mode 100644 index 000000000..8c12e2090 Binary files /dev/null and b/tools/DemoSystem/extensions/assets/img/yesterday_once_more.png differ diff --git a/tools/DemoSystem/extensions/install.py b/tools/DemoSystem/extensions/install.py index f698b6467..7fec8053f 100644 --- a/tools/DemoSystem/extensions/install.py +++ b/tools/DemoSystem/extensions/install.py @@ -3,6 +3,7 @@ import json import os import glob +import shutil from pathlib import Path @@ -228,17 +229,161 @@ def find_all_yml_files(base_dir): return yml_files -def generate_overview_index(component_record, output_file): - """生成概览index.ts文件""" - # 生成TypeScript代码 - 使用format()而不是f-string避免反斜杠问题 +def copy_assets(source_base_dir, target_base_dir, menus_data): + """复制资源文件（图片、视频等）""" + assets_copied = { + 'img': [], + 'video': [] + } + + def process_menu_item(item): + # 复制图片 + if 'meta' in item and 'img' in item['meta'] and item['meta']['img']: + img_path = item['meta']['img'] + if img_path: # 非空字符串 + # 构建源文件路径 + source_img = os.path.join(source_base_dir, img_path) + # 构建目标文件路径 - 修正：直接使用assets_target_dir + target_img = os.path.join( + os.path.join(target_base_dir, 'img'), os.path.basename(os.path.basename(img_path)) + ) + + # 确保目标目录存在 + os.makedirs(os.path.dirname(target_img), exist_ok=True) + + # 复制文件 + if os.path.exists(source_img): + shutil.copy2(source_img, target_img) + assets_copied['img'].append(img_path) + print(f" 复制图片: {img_path}") + print(f" 源路径: {source_img}") + print(f" 目标路径: {target_img}") + else: + print(f" ⚠️ 图片不存在: {source_img}") + + # 复制视频 + if 'meta' in item and 'video' in item['meta'] and 'src' in item['meta']['video']: + video_src = item['meta']['video']['src'] + if video_src: # 非空字符串 + # 构建源文件路径 + source_video = os.path.join(source_base_dir, video_src) + target_video = os.path.join( + os.path.join(target_base_dir, 'video'), os.path.basename(os.path.basename(video_src)) + ) + + # 确保目标目录存在 + os.makedirs(os.path.dirname(target_video), exist_ok=True) + + # 复制文件 + if os.path.exists(source_video): + shutil.copy2(source_video, target_video) + assets_copied['video'].append(video_src) + print(f" 复制视频: {video_src}") + print(f" 源路径: {source_video}") + print(f" 目标路径: {target_video}") + else: + print(f" ⚠️ 视频不存在: {source_video}") + + # 处理子菜单 + if 'children' in item: + for child in item['children']: + process_menu_item(child) + + # 处理所有菜单项 + for menu in menus_data: + process_menu_item(menu) + + return assets_copied + + +def convert_menus_for_json(menus_data): + """转换menus数据，准备生成JSON""" + + def convert_item(item): + # 创建新对象的副本 + new_item = item.copy() + + if 'meta' in new_item: + new_item['meta'] = new_item['meta'].copy() + + # 处理图片路径 - 使用占位符 + if 'img' in new_item['meta'] and new_item['meta']['img']: + img_path = new_item['meta']['img'] + if img_path and img_path.strip(): + # 使用特殊占位符，后续会替换 + new_item['meta']['img'] = f'__IMG_PLACEHOLDER__{img_path}__' + + # 处理视频路径 + if 'video' in new_item['meta'] and new_item['meta']['video']: + new_item['meta']['video'] = new_item['meta']['video'].copy() + video_src = new_item['meta']['video'].get('src', '') + if video_src and video_src.strip(): + # 使用特殊占位符 + new_item['meta']['video']['src'] = f'__VIDEO_PLACEHOLDER__{video_src}__' + + # 递归处理子菜单 + if 'children' in new_item: + new_item['children'] = [convert_item(child) for child in new_item['children']] + + return new_item + + return [convert_item(menu) for menu in menus_data] + + +def replace_placeholders_with_new_url(json_str): + """将JSON字符串中的占位符替换为new URL()表达式""" + + import re + + # 替换img占位符 + def replace_img_placeholder(match): + path = match.group(1) + # 确保路径以 @/ 开头 + if not path.startswith('@/'): + path = '@/{}'.format(path) + return 'new URL(\'{}\', import.meta.url).href'.format(path) + + # 替换video.src占位符 + def replace_video_placeholder(match): + path = match.group(1) + # 确保路径以 @/ 开头 + if not path.startswith('@/'): + path = '@/{}'.format(path) + return 'new URL(\'{}\', import.meta.url).href'.format(path) + + # 先替换video.src占位符 + json_str = re.sub(r'"__VIDEO_PLACEHOLDER__([^"]+)__"', replace_video_placeholder, json_str) + + # 再替换img占位符 + json_str = re.sub(r'"__IMG_PLACEHOLDER__([^"]+)__"', replace_img_placeholder, json_str) + + return json_str + + +def generate_extensions_index(component_record, menus_data, output_file, target_base_dir): + """生成扩展的概览index.ts文件，包含componentRecord和menus""" + + # 转换menus数据，使用占位符 + converted_menus = convert_menus_for_json(menus_data) + + # 生成TypeScript代码 entries = [] for key, value in component_record.items(): - entries.append(f' {key}: \'{value}\'') + entries.append(' {}: \'{}\''.format(key, value)) + + # 生成menus的JSON字符串 + menus_json = json.dumps(converted_menus, ensure_ascii=False, indent=2) + # 将占位符替换为new URL()表达式 + menus_ts_code = replace_placeholders_with_new_url(menus_json) + + # 生成完整的TypeScript代码 ts_code = """export const componentRecord = {{ -{} +{entries} }} -""".format(',\n'.join(entries)) + +export const menus = {menus} +""".format(entries=',\n'.join(entries), menus=menus_ts_code) # 确保输出目录存在 os.makedirs(os.path.dirname(output_file), exist_ok=True) @@ -247,7 +392,24 @@ def generate_overview_index(component_record, output_file): with open(output_file, 'w', encoding='utf-8') as f: f.write(ts_code) - print(f"\n已生成概览文件: {output_file}") + print(f"\n已生成扩展概览文件: {output_file}") + + +def load_menus_config(menus_file): + """加载menus.yml配置文件""" + try: + with open(menus_file, 'r', encoding='utf-8') as f: + config_data = yaml.safe_load(f) + + if 'menus' not in config_data: + print(f" 警告: menus.yml文件缺少menus字段") + return [] + + return config_data['menus'] + + except Exception as e: + print(f" 错误: 无法读取menus.yml文件 {menus_file}: {e}") + return [] def main(): @@ -257,8 +419,12 @@ def main(): # 输出目录（frontend/src/extensions/） output_base_dir = os.path.join(os.path.dirname(script_dir), 'frontend', 'src', 'extensions') + # 前端资源目录（frontend/src/assets/） + assets_target_dir = os.path.join(os.path.dirname(script_dir), 'frontend', 'src', 'assets') + print(f"脚本目录: {script_dir}") print(f"输出目录: {output_base_dir}") + print(f"资源目录: {assets_target_dir}") # 查找所有YAML文件 yml_files = find_all_yml_files(script_dir) @@ -272,8 +438,13 @@ def main(): component_record = {} successful_count = 0 + # 先处理所有实验配置文件 for yml_file, rel_path in yml_files: - print(f"\n处理文件: {rel_path}") + # 跳过menus.yml文件，单独处理 + if os.path.basename(yml_file) == 'menus.yml': + continue + + print(f"\n处理实验文件: {rel_path}") # 验证YAML结构 if not validate_yaml_structure(yml_file): @@ -315,22 +486,46 @@ def main(): else: print(f" ❌ 失败: {rel_path}") - # 生成概览index.ts文件 + # 处理menus.yml文件 + menus_file = os.path.join(script_dir, 'menus.yml') + menus_data = [] + + if os.path.exists(menus_file): + print(f"\n处理菜单文件: menus.yml") + menus_data = load_menus_config(menus_file) + if menus_data: + print(f" ✅ 成功加载: {len(menus_data)} 个菜单项") + + # 统计子菜单项 + child_count = sum(len(menu.get('children', [])) for menu in menus_data) + print(f" 包含: {child_count} 个子菜单项") + + # 复制资源文件 + print(f"\n复制资源文件:") + assets_copied = copy_assets(script_dir, assets_target_dir, menus_data) + + print(f" 图片: {len(assets_copied['img'])} 个") + print(f" 视频: {len(assets_copied['video'])} 个") + else: + print(f"\n⚠️ 未找到menus.yml文件，将只生成componentRecord") + + # 生成扩展的概览index.ts文件（包含componentRecord和menus） overview_file = os.path.join(output_base_dir, 'index.ts') - generate_overview_index(component_record, overview_file) + generate_extensions_index(component_record, menus_data, overview_file, assets_target_dir) # 打印总结 print(f"\n{'=' * 50}") print(f"处理完成:") print(f" 总共找到: {len(yml_files)} 个YML文件") - print(f" 成功生成: {successful_count} 个组件") - print(f" 失败: {len(yml_files) - successful_count} 个") + print(f" 成功生成: {successful_count} 个实验组件") + print(f" 加载菜单: {len(menus_data)} 个主菜单项") + print(f" 失败: {len(yml_files) - successful_count - (1 if os.path.exists(menus_file) else 0)} 个") if component_record: - print(f"\n生成的组件:") + print(f"\n生成的实验组件:") for key, value in sorted(component_record.items()): print(f" {key}: '{value}'") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/DemoSystem/extensions/menus.yml b/tools/DemoSystem/extensions/menus.yml new file mode 100644 index 000000000..4cb19efb6 --- /dev/null +++ b/tools/DemoSystem/extensions/menus.yml @@ -0,0 +1,67 @@ +menus: + - name: 'home' + path: '/dashboard/home' + meta: + title: "首页" + img: '' + description: "" + video: + src: "" + title: '' + description: '' + + - name: 'yesterdayOnceMore' + path: '/dashboard/yesterdayOnceMore' + meta: + title: "昨日重现" + img: "assets/img/yesterday_once_more.png" + description: "通过WEB页面，将历史场景、经典案例或特定状态进行高保真还原的模拟技术。它不仅是一个技术实现的过程，更是一种科研与教学相结合的教学范式" + video: + src: "" + title: '' + description: '' + children: + - name: 'bgp' + path: '/dashboard/yesterdayOnceMore/simulation/bgp' + meta: + title: "BGP 前缀劫持" + img: "assets/img/bgp_exploration.png" + description: "边界网关协议（BGP）是用于在互联网上的自治系统（AS）之间交换路由和可达性信息的标准外部网关协议。它是互联网的'粘合剂'，是互联网基础设施的重要组成部分，也是主要的攻击目标之一。如果攻击者能够控制 BGP，则可以断开互联网并重定向流量。本实验的目标是帮助学生了解 BGP 如何将互联网连接在一起以及互联网是如何连接的。我们构建了一个互联网仿真器，并将使用此仿真器作为实验活动的基础。" + video: + src: "" + title: '' + description: '' + + - name: 'morris' + path: '/dashboard/yesterdayOnceMore/simulation/morris' + meta: + title: "Morris worm 蠕虫" + img: "assets/img/worm.png" + description: "莫里斯蠕虫（1988年11月）是通过互联网传播的最古老的计算机蠕虫之一。虽然它很古老，但今天大多数蠕虫使用的技术仍然是相同的。它们包括两个主要部分：攻击和自我复制。攻击部分利用一个漏洞（或几个漏洞），因此蠕虫可以进入另一台计算机。自我复制部分是将自己的副本发送到受感染的机器，然后从那里发动攻击。" + video: + src: "" + title: '' + description: '' + + - name: 'mirai' + path: '/dashboard/yesterdayOnceMore/simulation/mirai' + meta: + title: "Mirai 僵尸网络攻击" + img: "assets/img/mirai.png" + description: "近年来物联网（IoT）安全领域最经典、最具代表性的威胁之一。它主要通过暴力破解（Brute Force）手段入侵具有弱密码的网络设备（如路由器、DVR、摄像头等），将其转变为“僵尸”（Bot），并在命令与控制（C&C）服务器的指挥下发起大规模的分布式拒绝服务（DDoS）攻击。" + video: + src: "" + title: '' + description: '' + + - name: 'blockchain' + path: '/dashboard/blockchain' + meta: + title: "大规模区块链仿真" + img: "assets/img/large_blockchain.png" + description: "跨学科、综合性的科研与教学课题。既要具备区块链理论（共识、加密）的深度，又要有系统工程（网络拓扑、分布式系统）的广度，熟练运用现代工具（Docker、Python 脚本、仿真框架）进行可重复性实验的构建与分析" + video: + src: "" + title: '' + description: '' + children: [] \ No newline at end of file diff --git a/tools/DemoSystem/extensions/yesterday_once_more/mirai.yml b/tools/DemoSystem/extensions/yesterday_once_more/mirai.yml new file mode 100644 index 000000000..c95e22239 --- /dev/null +++ b/tools/DemoSystem/extensions/yesterday_once_more/mirai.yml @@ -0,0 +1,158 @@ +baseInfo: + name: "Mirai 僵尸网络攻击" + path: "./yesterday_once_more/03_mirai" + +sections: + - headerTitle: "启动靶场" + steps: + - type: "command_only" + action: host_exec + cmd: "cd $hostProjectPath/yesterday_once_more/03_mirai/demo/demo_output && docker compose up -d" + - headerTitle: "环境设置" + steps: + - type: "description_with_command" + shortText: "环境设置" + description: + - type: "paragraph" + content: "首先，我们需要启动仿真器。" + - type: "paragraph" + content: "在本实验中，我们需要安装`submit_event`插件。" + - type: "code_block" + title: "命令" + content: | + curl -X POST -H "Content-Type: application/json" -d '{"name": "submit_event"}' http://:8080/api/v1/install + commands: + - action: host_exec + cmd: | + /usr/bin/curl -X POST -H \"Content-Type: application/json\" -d '{\"name\": \"submit_event\"}' \"http://$hostname:8080/api/v1/install\" + + - headerTitle: "构建僵尸网络" + steps: + - type: "description_with_command" + shortText: "启动 C2 服务器" + description: + - type: "paragraph" + content: "C2 服务器是僵尸网络的控制中心，用于接收`bot`的连接并发布指令。本示例使用开源工具 BYOB 作为 C2 框架。" + - type: "code_block" + title: "命令" + content: | + ./yesterday_once_more/03_mirai/scripts/start_c2_server.sh + + commands: + - action: host_exec + cmd: "cd $hostProjectPath/yesterday_once_more/03_mirai/scripts/ && /bin/bash start_c2_server.sh" + - type: "description_with_command_group" + shortText: "蠕虫控制" + description: + - type: "paragraph" + content: "C2 服务准备就绪后，即可释放蠕虫。我们将从 C2 服务器本身开始，运行 mirai.py 脚本。它将作为第一个被感染的节点，扫描网络中其他具有弱 Telnet 凭据的设备，并植入自身的副本" + - type: "paragraph" + content: "现在，执行下面的代码单元以启动蠕虫。" + - type: "code_block" + title: "命令" + content: | + print("正在 C2 服务器上释放 'mirai.py' 蠕虫...") + c2_container_id = !docker ps -f "name=C2_server" -q + if c2_container_id: + !docker exec -d {c2_container_id[0]} sh -c 'cd /var/www/html/ && python3 mirai.py > /tmp/mirai_worm.log 2>&1' + print("蠕虫已在后台运行，将开始扫描并感染网络中的其他主机。") + else: + print("错误：未找到 C2_server 容器。") + - type: "paragraph" + content: "执行上一步后，蠕虫已开始传播。请切换到 Internet map 进行观察（短暂闪烁后高亮）。" + - type: "paragraph" + content: "您将看到新的节点不断发起与 C2 服务器的连接，这表明这些节点正在下载 mirai.py 蠕虫脚本。整个传播过程的流量动态将直观地呈现在您面前。" + - type: "paragraph" + content: "同时，在运行 BYOB 的终端中，您会看到新的会话连接消息出现。当网络被完全感染时，会话总数将超过 90 个，可作为传播进度的参考。" + cmdGroups: + - title: "release" + tooltip: "释放蠕虫" + commands: + - action: docker_exec + containerNames: [ "as170h-C2_server-10.170.0.100" ] + cmd: "cd /var/www/html/ && python3 mirai.py > /tmp/mirai_worm.log 2>&1" + detach: true + - title: "stop" + tooltip: "停止蠕虫" + commands: + - action: host_exec + cmd: "cd $hostProjectPath/yesterday_once_more/03_mirai/scripts && /bin/bash ./control_worm.sh stop" + + - headerTitle: "Botnet" + steps: + - type: "description_with_command_group" + shortText: "Botnet 控制" + description: + - type: "paragraph" + content: "使用 `broadcast` 控制 Botnet" + - type: "paragraph" + content: "切换到运行 BYOB 控制台的终端窗口，键入以下命令并回车。此命令将被广播给所有在线的 `bot`。" + - type: "paragraph" + content: "请切换到 Internet map 进行观察（闪烁）。" + - type: "code_block" + title: "命令" + content: | + # 在 BYOB 控制台中输入命令： + + # display + broadcast timeout 300s bash -c ' + /usr/bin/echo "{ \"flash\": { \"dynamic\": { \"borderWidth\": 8, \"size\": 50 }, \"duration\": 1000 } }" >/map-plugins/style.json + && /bin/bash /map-plugins/submit_event.sh -a flash -s /map-plugins/style.json > /dev/null 2>&1' + + # restore + broadcast timeout 30s bash -c '/bin/bash /map-plugins/submit_event.sh + -a restore > /dev/null 2>&1' + cmdGroups: + - title: "display" + tooltip: "展示" + commands: + - action: terminal_exec + containerName: "as170h-C2_server-10.170.0.100" + cmd: | + broadcast timeout 60s bash -c '/usr/bin/echo "{ \"flash\": { \"dynamic\": { \"borderWidth\": 8, \"size\": 50 }, \"duration\": 1000 } }" >/map-plugins/style.json && /bin/bash /map-plugins/submit_event.sh -a flash -s /map-plugins/style.json && sleep 5 && /bin/bash /map-plugins/submit_event.sh -a highlight > /dev/null 2>&1' + - title: "restore" + tooltip: "恢复" + commands: + - action: terminal_exec + containerName: "as170h-C2_server-10.170.0.100" + cmd: | + broadcast timeout 30s bash -c '/bin/bash /map-plugins/submit_event.sh -a restore > /dev/null 2>&1' + - type: "description_with_command_group" + shortText: "Botnet 攻击" + description: + - type: "paragraph" + content: "切换到运行 BYOB 控制台的终端窗口，键入以下命令并回车。此命令将被广播给所有在线的 `bot`。" + - type: "paragraph" + content: "请切换到 Internet map 进行观察（数据包数量统计）。" + - type: "code_block" + title: "命令" + content: | + # 在 BYOB 控制台中输入此命令： + broadcast timeout 30s bash -c 'ping 10.170.0.99 > /dev/null 2>&1' + cmdGroups: + - title: "attack" + tooltip: "发起攻击" + commands: + - action: terminal_exec + containerName: "as170h-C2_server-10.170.0.100" + cmd: | + broadcast timeout 30s bash -c 'ping 10.170.0.99 > /dev/null 2>&1' + - action: host_exec + cmd: | + /usr/bin/curl -X POST -H \"Content-Type: application/json\" -d '{\"nodeName\": \"/as170h-victim-10.170.0.99\", \"filter\": \"dst 10.170.0.99\"}' \"http://$hostname:8080/api/v1/packet\" + - title: "restore" + tooltip: "恢复" + commands: + - action: terminal_exec + containerName: "as170h-C2_server-10.170.0.100" + cmd: | + broadcast bash -c 'pkill -f "ping 10.170.0.99"' + - action: host_exec + cmd: | + /usr/bin/curl -X POST -H \"Content-Type: application/json\" -d '{\"nodeName\": \"/as170h-victim-10.170.0.99\", \"action\": \"restore\"}' \"http://$hostname:8080/api/v1/packet\" + + - headerTitle: "关闭靶场" + steps: + - type: "command_only" + action: host_exec + cmd: "cd $hostProjectPath/yesterday_once_more/03_mirai/demo/demo_output && docker compose down" \ No newline at end of file diff --git a/tools/DemoSystem/extensions/yesterday_once_more/morris.yml b/tools/DemoSystem/extensions/yesterday_once_more/morris.yml index d1d3077f0..7fb678523 100644 --- a/tools/DemoSystem/extensions/yesterday_once_more/morris.yml +++ b/tools/DemoSystem/extensions/yesterday_once_more/morris.yml @@ -39,7 +39,7 @@ sections: content: "执行 ./yesterday_once_more/02_morris_worm/worm/first_attack.py 发起攻击" commands: - action: host_exec - cmd: "cd $hostProjectPath/yesterday_once_more/02_morris_worm/worm && $condaPath run -n $envName python first_attack.py" + cmd: "cd $hostProjectPath/yesterday_once_more/02_morris_worm/worm && /usr/bin/python3 first_attack.py" - headerTitle: "蠕虫控制" steps: diff --git a/tools/DemoSystem/frontend/index.html b/tools/DemoSystem/frontend/index.html index 6774ac768..9e6419c95 100644 --- a/tools/DemoSystem/frontend/index.html +++ b/tools/DemoSystem/frontend/index.html @@ -4,7 +4,7 @@ - demo-system + DemoSystem
diff --git a/tools/DemoSystem/frontend/src/assets/img/large_blockchain.png b/tools/DemoSystem/frontend/src/assets/img/large_blockchain.png new file mode 100644 index 000000000..70e4d7760 Binary files /dev/null and b/tools/DemoSystem/frontend/src/assets/img/large_blockchain.png differ diff --git a/tools/DemoSystem/frontend/src/assets/img/mirai.png b/tools/DemoSystem/frontend/src/assets/img/mirai.png new file mode 100644 index 000000000..ebc71327b Binary files /dev/null and b/tools/DemoSystem/frontend/src/assets/img/mirai.png differ diff --git a/tools/DemoSystem/frontend/src/assets/img/yesterday_once_more.png b/tools/DemoSystem/frontend/src/assets/img/yesterday_once_more.png new file mode 100644 index 000000000..8c12e2090 Binary files /dev/null and b/tools/DemoSystem/frontend/src/assets/img/yesterday_once_more.png differ diff --git a/tools/DemoSystem/frontend/src/assets/video/morris-demo-new.mp4 b/tools/DemoSystem/frontend/src/assets/video/morris-demo-new.mp4 deleted file mode 100644 index 2c008a296..000000000 Binary files a/tools/DemoSystem/frontend/src/assets/video/morris-demo-new.mp4 and /dev/null differ diff --git a/tools/DemoSystem/frontend/src/components/BaseMap/index.vue b/tools/DemoSystem/frontend/src/components/BaseMap/index.vue index 2669f131e..63be59b7e 100644 --- a/tools/DemoSystem/frontend/src/components/BaseMap/index.vue +++ b/tools/DemoSystem/frontend/src/components/BaseMap/index.vue @@ -1,6 +1,7 @@