test(nemo-agents): add docker-mode agent deployment e2e test

.github/workflows/ci.yaml

@@ -1189,6 +1189,76 @@ jobs:
             report.xml
             ${{ runner.temp }}/e2e-services-logs/
 
+  # E2E tests that run on the subprocess harness but need a prebuilt platform
+  # image available in the local Docker daemon (e.g. docker-mode agent
+  # deployments, which deploy the nmp-api image as a sibling container). Marked
+  # `needs_nmp_api_image`; the plain python-e2e-test job skips them because
+  # NMP_E2E_IMAGE_REGISTRY / NMP_E2E_IMAGE_TAG are unset there.
+  python-e2e-image-test:
+    name: Python e2e tests (prebuilt image)
+    needs: [policy-wasm, build-cpu-smoke-images]
+    if: >
+      !cancelled() &&
+      needs.build-cpu-smoke-images.result == 'success' &&
+      needs.build-cpu-smoke-images.outputs.publish_images == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      packages: read
+    env:
+      NMP_E2E_IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
+      NMP_E2E_IMAGE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - name: Download policy WASM
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: policy-wasm
+          path: services/core/auth/src/nmp/core/auth/assets
+      - name: Install uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          python-version: "3.13"
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Log in to GHCR
+        shell: bash
+        env:
+          GHCR_TOKEN: ${{ github.token }}
+        run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+      - name: Pull nmp-api image (timed)
+        shell: bash
+        run: |
+          image="${NMP_E2E_IMAGE_REGISTRY}/nmp-api:${NMP_E2E_IMAGE_TAG}"
+          echo "Pulling ${image}"
+          start=$(date +%s)
+          docker pull "$image"
+          echo "nmp-api pull took $(( $(date +%s) - start ))s"
+      - name: Run e2e tests (needs_nmp_api_image)
+        run: make test-e2e PYTEST_EXTRA="-m needs_nmp_api_image"
+        env:
+          _TYPER_FORCE_DISABLE_TERMINAL: "1"
+          E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
+      - name: Dump server logs
+        if: always()
+        run: |
+          echo "::group::Server logs"
+          for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do
+            [ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" || echo "No server logs found"
+          done
+          echo "::endgroup::"
+      - name: Upload test artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: python-e2e-image-test-results
+          retention-days: 30
+          path: |
+            report.xml
+            ${{ runner.temp }}/e2e-services-logs/
+
   web-typecheck:
     name: Web typecheck
     needs: [changes]

conftest.py

@@ -316,6 +316,9 @@ def pytest_runtest_setup(item):
     if "container_only" in [marker.name for marker in item.iter_markers()]:
         if not os.environ.get("NMP_BASE_URL"):
             skip_test("Skipping container-only test (requires NMP_BASE_URL)")
+    if "needs_nmp_api_image" in [marker.name for marker in item.iter_markers()]:
+        if not (os.environ.get("NMP_E2E_IMAGE_REGISTRY") and os.environ.get("NMP_E2E_IMAGE_TAG")):
+            skip_test("Skipping nmp-api-image test (set NMP_E2E_IMAGE_REGISTRY and NMP_E2E_IMAGE_TAG)")
     if "requires_gpu" in [marker.name for marker in item.iter_markers()]:
         if "gpu" not in _e2e_features_enabled(item.config):
             skip_test("Skipping GPU container e2e (pass --feature gpu)")

e2e/configs/local-docker-agents.yaml

@@ -0,0 +1,68 @@
+# E2E config for docker-mode agent deployments.
+#
+# The platform runs as a normal local process (subprocess harness backend), but
+# it is wired with a nemo-deployments Docker executor so an agent deployed with
+# deployment_mode=docker runs as a real Docker container on the host daemon.
+#
+# platform.base_url below is a placeholder: the e2e harness rewrites it to
+# http://<container_base_url_host>:<the free port it picked> right before launch
+# (see harness={"container_base_url_host": ...} on the test module). That makes
+# the Inference Gateway URL the platform injects into the deployed agent
+# container reachable from *inside* that container (the docker bridge address),
+# instead of a loopback the container cannot reach. The runner seeds NMP_BASE_URL
+# from this platform.base_url host (paired with the actual bind port), so the
+# configured host takes effect instead of the bind-derived loopback default.
+
+platform:
+  runtime: "docker"
+  base_url: "http://0.0.0.0:8080"
+
+service: {}
+
+auth:
+  enabled: false
+  allow_unsigned_jwt: true
+  policy_decision_point_provider: embedded
+  policy_decision_point_base_url: "http://localhost:8080"
+  policy_data_refresh_interval: 2
+  bundle_cache_seconds: 15
+  admin_email: "admin@example.com"
+
+entities: {}
+
+agents:
+  deployments:
+    # Names below must match a deployments.executors[].name.
+    default_executor: local-docker
+    docker_executor: local-docker
+    # Container port the NAT server binds inside the agent container (and the
+    # readiness-probe target).
+    container_port: 8000
+
+deployments:
+  default_executor: local-docker
+  executors:
+    - name: local-docker
+      backend: docker
+      config:
+        # The agent runs from the prebuilt nmp-api image, already present in the
+        # local Docker daemon (pulled by the test's CI job), so disable the
+        # per-run pull and use the image as-is.
+        pull_images: false
+        port_range_start: 9000
+        port_range_end: 9100
+
+models:
+  controller:
+    interval_seconds: 5
+    model_deployment_garbage_collection_ttl_seconds: 30
+
+inference_gateway: {}
+
+secrets:
+  allow_key_creation: true
+
+files:
+  default_storage_config:
+    type: local
+    path: .tmp/e2e/files

e2e/services_pool.py

@@ -61,6 +61,19 @@ class E2EHarnessConfig(TypedDict, total=False):
     lifecycle: Literal["fresh", "reuse"]
     compose_project_prefix: str
     env: dict[str, str]
+    # Subprocess backend only. When set, the harness (a) binds the platform on all
+    # interfaces (``--host 0.0.0.0``) and (b) rewrites ``platform.base_url`` in the
+    # rendered config to ``http://<this host>:<the free port it chose>`` right
+    # before launch. The platform still runs as a normal local process, but the
+    # base URL it injects into container-mode agent deployments (docker/k8s) then
+    # points at a host reachable from *inside* those containers (e.g. the docker
+    # bridge ``172.17.0.1``) instead of a loopback the container can't reach.
+    # Binding all interfaces is required so the platform's own in-process clients,
+    # which also use ``NMP_BASE_URL``, can still reach it at that host. The runner
+    # seeds ``NMP_BASE_URL`` from the config ``platform.base_url`` host (paired
+    # with the real bind port), so the rewritten host takes effect instead of the
+    # bind-derived loopback default.
+    container_base_url_host: str
 
 
 @dataclass
@@ -345,6 +358,8 @@ def _resolve_e2e_harness_config_from_node(node: Node) -> E2EHarnessConfig:
     if backend not in {"subprocess", "docker", "docker_compose"}:
         raise pytest.UsageError(f"unsupported e2e harness backend: {backend}")
     normalized["backend"] = backend
+    if "container_base_url_host" in normalized and backend != "subprocess":
+        raise pytest.UsageError("container_base_url_host is only supported with the 'subprocess' harness backend")
     if backend == "docker_compose":
         required = {"compose_file", "service_url"}
         missing = sorted(required - set(normalized))
@@ -544,6 +559,14 @@ def _find_free_port() -> int:
         return sock.getsockname()[1]
 
 
+def _set_platform_base_url(config_path: Path, base_url: str) -> None:
+    """Rewrite ``platform.base_url`` in an already-materialized config file."""
+    config_data = yaml.safe_load(config_path.read_text()) or {}
+    platform = config_data.setdefault("platform", {})
+    platform["base_url"] = base_url
+    config_path.write_text(yaml.safe_dump(config_data, default_flow_style=False, sort_keys=True))
+
+
 def _process_exited(proc: subprocess.Popen[Any]) -> bool:
     return proc.poll() is not None
 
@@ -631,11 +654,15 @@ def _start_services(
         return _start_services_docker(config_path, config_data, config_hash)
     if backend == "docker_compose":
         return _start_services_docker_compose(config_path, config_data, harness_config, config_hash, log_path)
-    return _start_services_subprocess(config_path, config_data, config_hash, log_path)
+    return _start_services_subprocess(config_path, config_data, harness_config, config_hash, log_path)
 
 
 def _start_services_subprocess(
-    config_path: Path, config_data: dict[str, Any], config_hash: str, log_path: Path
+    config_path: Path,
+    config_data: dict[str, Any],
+    harness_config: E2EHarnessConfig,
+    config_hash: str,
+    log_path: Path,
 ) -> RunningServices:
     port = _find_free_port()
     url = f"http://127.0.0.1:{port}"
@@ -652,6 +679,22 @@ def _start_services_subprocess(
         "--port",
         str(port),
     ]
+
+    # For container-mode agent deployments the platform must advertise a base URL
+    # reachable from inside the deployed agent container, not the loopback the
+    # platform binds by default. Two coordinated changes make that work:
+    #   1. Bind all interfaces (--host 0.0.0.0) instead of the CLI default
+    #      127.0.0.1, so the platform is reachable on the container-facing host
+    #      (e.g. the docker bridge 172.17.0.1) as well as loopback.
+    #   2. Rewrite platform.base_url on disk to http://<host>:<this port>. The
+    #      free port is only known here, after the config file was materialized.
+    #      The platform seeds NMP_BASE_URL from this, so both the platform's own
+    #      in-process clients and the injected agent LLM base_url point at a
+    #      host the agent container can reach.
+    container_host = harness_config.get("container_base_url_host")
+    if container_host:
+        args += ["--host", "0.0.0.0"]
+        _set_platform_base_url(config_path, f"http://{container_host}:{port}")
     data_dir = e2e_services_data_dir(log_path.parent, config_hash)
     data_dir.mkdir(parents=True, exist_ok=True)
     env = e2e_services_env(config_path, data_dir)