diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ac42b6d37c..277e2d1141 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1189,6 +1189,76 @@ jobs: report.xml ${{ runner.temp }}/e2e-services-logs/ + # E2E tests that run on the subprocess harness but need a prebuilt platform + # image available in the local Docker daemon (e.g. docker-mode agent + # deployments, which deploy the nmp-api image as a sibling container). Marked + # `needs_nmp_api_image`; the plain python-e2e-test job skips them because + # NMP_E2E_IMAGE_REGISTRY / NMP_E2E_IMAGE_TAG are unset there. + python-e2e-image-test: + name: Python e2e tests (prebuilt image) + needs: [policy-wasm, build-cpu-smoke-images] + if: > + !cancelled() && + needs.build-cpu-smoke-images.result == 'success' && + needs.build-cpu-smoke-images.outputs.publish_images == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + packages: read + env: + NMP_E2E_IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }} + NMP_E2E_IMAGE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Download policy WASM + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: policy-wasm + path: services/core/auth/src/nmp/core/auth/assets + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + python-version: "3.13" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Log in to GHCR + shell: bash + env: + GHCR_TOKEN: ${{ github.token }} + run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Pull nmp-api image (timed) + shell: bash + run: | + image="${NMP_E2E_IMAGE_REGISTRY}/nmp-api:${NMP_E2E_IMAGE_TAG}" + echo "Pulling ${image}" + start=$(date +%s) + docker pull "$image" + echo "nmp-api pull took $(( $(date +%s) - start ))s" + - name: Run e2e tests (needs_nmp_api_image) + run: make test-e2e PYTEST_EXTRA="-m needs_nmp_api_image" + env: + _TYPER_FORCE_DISABLE_TERMINAL: "1" + E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs + - name: Dump server logs + if: always() + run: | + echo "::group::Server logs" + for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do + [ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" || echo "No server logs found" + done + echo "::endgroup::" + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-e2e-image-test-results + retention-days: 30 + path: | + report.xml + ${{ runner.temp }}/e2e-services-logs/ + web-typecheck: name: Web typecheck needs: [changes] diff --git a/conftest.py b/conftest.py index c90bfa5af7..195ad9bac6 100644 --- a/conftest.py +++ b/conftest.py @@ -316,6 +316,9 @@ def pytest_runtest_setup(item): if "container_only" in [marker.name for marker in item.iter_markers()]: if not os.environ.get("NMP_BASE_URL"): skip_test("Skipping container-only test (requires NMP_BASE_URL)") + if "needs_nmp_api_image" in [marker.name for marker in item.iter_markers()]: + if not (os.environ.get("NMP_E2E_IMAGE_REGISTRY") and os.environ.get("NMP_E2E_IMAGE_TAG")): + skip_test("Skipping nmp-api-image test (set NMP_E2E_IMAGE_REGISTRY and NMP_E2E_IMAGE_TAG)") if "requires_gpu" in [marker.name for marker in item.iter_markers()]: if "gpu" not in _e2e_features_enabled(item.config): skip_test("Skipping GPU container e2e (pass --feature gpu)") diff --git a/e2e/configs/local-docker-agents.yaml b/e2e/configs/local-docker-agents.yaml new file mode 100644 index 0000000000..f14663bb49 --- /dev/null +++ b/e2e/configs/local-docker-agents.yaml @@ -0,0 +1,68 @@ +# E2E config for docker-mode agent deployments. +# +# The platform runs as a normal local process (subprocess harness backend), but +# it is wired with a nemo-deployments Docker executor so an agent deployed with +# deployment_mode=docker runs as a real Docker container on the host daemon. +# +# platform.base_url below is a placeholder: the e2e harness rewrites it to +# http://: right before launch +# (see harness={"container_base_url_host": ...} on the test module). That makes +# the Inference Gateway URL the platform injects into the deployed agent +# container reachable from *inside* that container (the docker bridge address), +# instead of a loopback the container cannot reach. The runner seeds NMP_BASE_URL +# from this platform.base_url host (paired with the actual bind port), so the +# configured host takes effect instead of the bind-derived loopback default. + +platform: + runtime: "docker" + base_url: "http://0.0.0.0:8080" + +service: {} + +auth: + enabled: false + allow_unsigned_jwt: true + policy_decision_point_provider: embedded + policy_decision_point_base_url: "http://localhost:8080" + policy_data_refresh_interval: 2 + bundle_cache_seconds: 15 + admin_email: "admin@example.com" + +entities: {} + +agents: + deployments: + # Names below must match a deployments.executors[].name. + default_executor: local-docker + docker_executor: local-docker + # Container port the NAT server binds inside the agent container (and the + # readiness-probe target). + container_port: 8000 + +deployments: + default_executor: local-docker + executors: + - name: local-docker + backend: docker + config: + # The agent runs from the prebuilt nmp-api image, already present in the + # local Docker daemon (pulled by the test's CI job), so disable the + # per-run pull and use the image as-is. + pull_images: false + port_range_start: 9000 + port_range_end: 9100 + +models: + controller: + interval_seconds: 5 + model_deployment_garbage_collection_ttl_seconds: 30 + +inference_gateway: {} + +secrets: + allow_key_creation: true + +files: + default_storage_config: + type: local + path: .tmp/e2e/files diff --git a/e2e/services_pool.py b/e2e/services_pool.py index 215cae3db0..2d3cb08192 100644 --- a/e2e/services_pool.py +++ b/e2e/services_pool.py @@ -61,6 +61,9 @@ class E2EHarnessConfig(TypedDict, total=False): lifecycle: Literal["fresh", "reuse"] compose_project_prefix: str env: dict[str, str] + # Subprocess backend only; container-reachable host for platform.base_url (see + # _start_services_subprocess for how it's applied). + container_base_url_host: str @dataclass @@ -345,6 +348,8 @@ def _resolve_e2e_harness_config_from_node(node: Node) -> E2EHarnessConfig: if backend not in {"subprocess", "docker", "docker_compose"}: raise pytest.UsageError(f"unsupported e2e harness backend: {backend}") normalized["backend"] = backend + if "container_base_url_host" in normalized and backend != "subprocess": + raise pytest.UsageError("container_base_url_host is only supported with the 'subprocess' harness backend") if backend == "docker_compose": required = {"compose_file", "service_url"} missing = sorted(required - set(normalized)) @@ -544,6 +549,14 @@ def _find_free_port() -> int: return sock.getsockname()[1] +def _set_platform_base_url(config_path: Path, base_url: str) -> None: + """Rewrite ``platform.base_url`` in an already-materialized config file.""" + config_data = yaml.safe_load(config_path.read_text()) or {} + platform = config_data.setdefault("platform", {}) + platform["base_url"] = base_url + config_path.write_text(yaml.safe_dump(config_data, default_flow_style=False, sort_keys=True)) + + def _process_exited(proc: subprocess.Popen[Any]) -> bool: return proc.poll() is not None @@ -631,11 +644,15 @@ def _start_services( return _start_services_docker(config_path, config_data, config_hash) if backend == "docker_compose": return _start_services_docker_compose(config_path, config_data, harness_config, config_hash, log_path) - return _start_services_subprocess(config_path, config_data, config_hash, log_path) + return _start_services_subprocess(config_path, config_data, harness_config, config_hash, log_path) def _start_services_subprocess( - config_path: Path, config_data: dict[str, Any], config_hash: str, log_path: Path + config_path: Path, + config_data: dict[str, Any], + harness_config: E2EHarnessConfig, + config_hash: str, + log_path: Path, ) -> RunningServices: port = _find_free_port() url = f"http://127.0.0.1:{port}" @@ -652,6 +669,22 @@ def _start_services_subprocess( "--port", str(port), ] + + # For container-mode agent deployments the platform must advertise a base URL + # reachable from inside the deployed agent container, not the loopback the + # platform binds by default. Two coordinated changes make that work: + # 1. Bind all interfaces (--host 0.0.0.0) instead of the CLI default + # 127.0.0.1, so the platform is reachable on the container-facing host + # (e.g. the docker bridge 172.17.0.1) as well as loopback. + # 2. Rewrite platform.base_url on disk to http://:. The + # free port is only known here, after the config file was materialized. + # The platform seeds NMP_BASE_URL from this, so both the platform's own + # in-process clients and the injected agent LLM base_url point at a + # host the agent container can reach. + container_host = harness_config.get("container_base_url_host") + if container_host: + args += ["--host", "0.0.0.0"] + _set_platform_base_url(config_path, f"http://{container_host}:{port}") data_dir = e2e_services_data_dir(log_path.parent, config_hash) data_dir.mkdir(parents=True, exist_ok=True) env = e2e_services_env(config_path, data_dir) diff --git a/e2e/test_nemo_agents_docker.py b/e2e/test_nemo_agents_docker.py new file mode 100644 index 0000000000..66ed24e6d7 --- /dev/null +++ b/e2e/test_nemo_agents_docker.py @@ -0,0 +1,332 @@ +"""E2E test for docker-mode agent deployments. + +Unlike ``test_nemo_agents.py`` (backend-agnostic API/SDK surface, plus +subprocess-mode deployment coverage), this module deploys an agent as a real +**Docker container** through the nemo-deployments plugin and invokes it through +the agents gateway. + +What it proves — the container-mode chain end to end:: + + sdk.agents.invoke (gateway proxy, container-mode endpoint resolution) + -> docker agent container (nat start fastapi) + -> Inference Gateway /openai (base_url injected at deploy time) + -> mock provider short-circuit (no real upstream / no API key) + -> response back through the gateway + +How it runs, and where: + +- The deployed agent runs from the platform's own ``nmp-api`` image, which + already ships the NAT runtime (``nvidia-nat-core`` / ``nvidia-nat-langchain``, + via ``nemo-agents-plugin``) — so ``nat`` is on ``PATH`` and the + ``chat_completion`` workflow / ``openai`` LLM the agent uses resolve. The + deployments docker executor overrides the image entrypoint with + ``nat start fastapi``, so the image's own entrypoint is irrelevant. No + agent-specific image is built. +- The image is supplied prebuilt via ``NMP_E2E_IMAGE_REGISTRY`` / + ``NMP_E2E_IMAGE_TAG`` (the existing e2e convention). Its dedicated CI job + (``python-e2e-image-test``) builds/pulls ``nmp-api`` and sets these; the + ``needs_nmp_api_image`` marker skips the test everywhere they are unset (the + plain subprocess e2e job, the kind cluster job, and local runs without them). +- The agent is registered with a deterministic single-LLM ``chat_completion`` + workflow served by the e2e mock inference provider, so no ``NVIDIA_API_KEY`` + or model egress is needed; we assert the exact mocked completion round-trips. +- The platform runs as a normal local process (subprocess harness). The + ``container_base_url_host`` harness option makes the harness bind the platform + on all interfaces (``--host 0.0.0.0``) and rewrite ``platform.base_url`` to the + docker bridge address; the runner seeds ``NMP_BASE_URL`` from that host paired + with the actual bind port, so the Inference Gateway URL injected into the agent + container is reachable from *inside* the container while the platform's own + in-process clients still reach it. This requires the Linux docker bridge, which + is reachable from both the container and the host process; Docker Desktop's + host alias is not host-resolvable, so the module skips on non-Linux. +""" + +import os +import platform +import time +import uuid +from contextlib import suppress +from typing import Any + +import httpx +import pytest +from nemo_platform import NeMoPlatform +from nmp.testing import MockProviderResponse, add_mock_provider + +# The docker bridge gateway. On Linux (incl. GitHub Actions ubuntu runners) this +# address is reachable both from inside a container AND by the platform process +# itself (it is a local host interface). That dual reachability is required: +# platform.base_url is used both as the Inference Gateway URL injected into the +# agent container *and* by the platform's own in-process service clients. +_DOCKER_BRIDGE_HOST = "172.17.0.1" + +# Platform image name to deploy the agent from. The nmp-api image already ships +# the NAT runtime and the agent components (see module docstring), so it doubles +# as the agent runtime image. Registry and tag come from NMP_E2E_IMAGE_REGISTRY / +# NMP_E2E_IMAGE_TAG (the existing e2e image convention). +_AGENT_IMAGE_NAME = "nmp-api" + +# Runs the platform as a local process wired with a docker deployments executor +# (see the config), and deploys the agent as a real docker container. +# +# Markers: +# - ``needs_nmp_api_image``: skips unless NMP_E2E_IMAGE_REGISTRY + NMP_E2E_IMAGE_TAG +# are set (its dedicated CI job builds/pulls nmp-api and sets them). This also +# keeps the test out of the plain subprocess e2e job and the kind cluster job. +# - ``subprocess_only``: this test drives its own subprocess-harness platform +# configured with a docker deployments executor. It must NOT run against an +# external cluster (``NMP_BASE_URL`` set, e.g. the Kind CPU e2e job), where the +# deployed Helm platform has no docker executor and the module's own +# ``e2e_config``/harness are ignored. +pytestmark = [ + pytest.mark.needs_nmp_api_image, + pytest.mark.subprocess_only, + pytest.mark.skipif( + platform.system() != "Linux", + reason="Docker-mode agent deployment e2e requires a Linux docker bridge reachable by both the " + "platform process and the agent container (Docker Desktop's host alias is not host-resolvable).", + ), + pytest.mark.e2e_config( + "e2e/configs/local-docker-agents.yaml", + harness={"backend": "subprocess", "container_base_url_host": _DOCKER_BRIDGE_HOST}, + ), +] + +_TEST_AGENT_RESPONSE = "The answer to your question is 42." + + +def _unique_name(prefix: str) -> str: + return f"e2e-{prefix}-{uuid.uuid4().hex[:8]}" + + +def _chat_completion_response(content: str, model: str) -> dict[str, Any]: + return { + "id": "chatcmpl-agents-docker-e2e", + "object": "chat.completion", + "model": model, + "choices": [ + { + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + "index": 0, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +def _mock_backed_agent_config(model_name: str) -> dict[str, Any]: + """A deterministic single-LLM workflow pointed at the mock model. + + ``base_url``/``api_key`` are intentionally omitted so the deployment injects + the Inference Gateway URL (and the mock provider short-circuits the call). + """ + return { + "llms": { + "main": { + "_type": "openai", + "model_name": model_name, + } + }, + "workflow": { + "_type": "chat_completion", + "llm_name": "main", + }, + } + + +def _page_data(page: Any) -> list[dict[str, Any]]: + if isinstance(page, dict): + data = page.get("data", []) + else: + data = getattr(page, "data", []) + assert isinstance(data, list) + return data + + +def _delete_agent_if_exists(sdk: NeMoPlatform, *, workspace: str, name: str) -> None: + try: + sdk.agents.delete(name, workspace=workspace) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + + +def _delete_deployment_if_exists(sdk: NeMoPlatform, *, workspace: str, name: str) -> None: + try: + sdk.agents.deployments.delete(name, workspace=workspace) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + + +def _get_deployment_log_text(sdk: NeMoPlatform, *, workspace: str, name: str) -> str: + try: + response = sdk._client.get( + f"/apis/agents/v2/workspaces/{workspace}/deployments/{name}/logs", + params={"tail": 100}, + ) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + return exc.response.text + + payload = response.json() + lines = _page_data(payload) + return "\n".join(str(line.get("message", line)) for line in lines) + + +def _wait_for_deployment_deleted( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 120, +) -> None: + deadline = time.monotonic() + timeout_seconds + last_status: str | None = None + while time.monotonic() < deadline: + try: + deployment = sdk.agents.deployments.get(name, workspace=workspace) + last_status = deployment.get("status") + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return + raise + time.sleep(2) + pytest.fail(f"Deployment {name!r} was not deleted within {timeout_seconds}s; last status={last_status!r}") + + +def _wait_for_deployment_running( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 300, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + last_deployment: dict[str, Any] | None = None + while time.monotonic() < deadline: + deployment = sdk.agents.deployments.get(name, workspace=workspace) + last_deployment = deployment + status = deployment["status"] + if status == "running": + return deployment + if status == "failed": + logs = _get_deployment_log_text(sdk, workspace=workspace, name=name) + pytest.fail(f"Deployment {name!r} failed: {deployment.get('error', '')}\n{logs}") + time.sleep(2) + pytest.fail(f"Deployment {name!r} did not reach running within {timeout_seconds}s: {last_deployment}") + + +@pytest.fixture(scope="session") +def agent_deployment_image() -> str: + """Return the prebuilt platform image ref to deploy the agent from. + + Composed from the e2e image convention (``NMP_E2E_IMAGE_REGISTRY`` / + ``NMP_E2E_IMAGE_TAG``) as ``{registry}/nmp-api:{tag}``. The nmp-api image + already ships the NAT runtime and agent components, so no agent-specific + image is built here. The ``needs_nmp_api_image`` marker guarantees both env + vars are set before this test runs; assert defensively. + """ + registry = os.environ.get("NMP_E2E_IMAGE_REGISTRY") + tag = os.environ.get("NMP_E2E_IMAGE_TAG") + assert registry and tag, "needs_nmp_api_image marker should have skipped when registry/tag are unset" + return f"{registry.rstrip('/')}/{_AGENT_IMAGE_NAME}:{tag}" + + +def _remove_agent_container_if_present(deployment_name: str) -> None: + """Best-effort removal of a leaked agent container after teardown.""" + try: + from docker.errors import NotFound + + import docker + except Exception: + return + try: + client = docker.from_env() + except Exception: + return + # The deployments docker backend names containers after the deployment; match + # loosely so a naming-scheme change does not silently leak containers. + for container in client.containers.list(all=True): + if deployment_name in container.name: + try: + container.remove(force=True) + except NotFound: + pass + except Exception: + pass + + +def test_docker_agent_deploys_and_invokes_through_gateway( + sdk: NeMoPlatform, workspace: str, agent_deployment_image: str +) -> None: + """Deploy an agent as a docker container from the nmp-api image and invoke it. + + Asserts the deployment reaches ``running`` with the container-mode endpoint + shape (empty scalar ``endpoint``, populated ``endpoints``), then invokes + through the gateway and asserts the mocked completion round-trips from inside + the container back to the caller. + """ + agent_name = _unique_name("calc-agent") + deployment_name = _unique_name("calc-deployment") + model_name = _unique_name("calc-model") + + add_mock_provider( + sdk, + workspace=workspace, + name=_unique_name("calc-provider"), + mock_response_body_by_model={ + f"{workspace}/{model_name}": [ + MockProviderResponse(response_body=_chat_completion_response(_TEST_AGENT_RESPONSE, model_name)), + ], + }, + served_models={model_name: model_name}, + ) + + sdk.agents.create( + workspace=workspace, + name=agent_name, + config=_mock_backed_agent_config(f"{workspace}/{model_name}"), + ) + + try: + created = sdk.agents.deployments.create( + workspace=workspace, + agent=agent_name, + name=deployment_name, + deployment_mode="docker", + image=agent_deployment_image, + ) + assert created["deployment_mode"] == "docker" + + deployment = _wait_for_deployment_running(sdk, workspace=workspace, name=deployment_name) + assert deployment["agent"] == agent_name + assert deployment["deployment_mode"] == "docker" + + # Container-mode addressing: the loopback scalar ``endpoint`` is empty and + # the routable address lives in ``endpoints`` (this is what the gateway's + # container-mode resolution reads). Guarding this shape ensures a pass can + # only come through the container path, not a subprocess fallback. + assert deployment.get("endpoint", "") == "" + endpoints = deployment.get("endpoints") or [] + assert endpoints and endpoints[0]["url"], deployment + + response = sdk.agents.invoke( + workspace=workspace, + agent=agent_name, + input="What is 12 multiplied by 8?", + ) + content = response["choices"][0]["message"]["content"] + assert _TEST_AGENT_RESPONSE in content, response + finally: + # Each step is isolated so a failure (e.g. a deployment-delete timeout) + # doesn't skip the remaining cleanup and leak resources. + with suppress(Exception): + _delete_deployment_if_exists(sdk, workspace=workspace, name=deployment_name) + with suppress(Exception): + _wait_for_deployment_deleted(sdk, workspace=workspace, name=deployment_name) + with suppress(Exception): + _remove_agent_container_if_present(deployment_name) + with suppress(Exception): + _delete_agent_if_exists(sdk, workspace=workspace, name=agent_name) diff --git a/pytest.ini b/pytest.ini index 0980e96dfc..a542b7dcba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -68,6 +68,7 @@ markers = e2e_config(*layers, harness=...): Ordered list of repo-root-relative config paths and/or inline dict overlays; harness config stays separate from platform config subprocess_only: Test only works in subprocess mode (not on Kubernetes); skipped when NMP_BASE_URL is set container_only: Test requires a container backend (Docker or Kubernetes); skipped unless NMP_BASE_URL is set + needs_nmp_api_image: Test runs on the subprocess harness with a Docker deployments executor and needs the nmp-api image available as the deployment image; skipped unless NMP_E2E_PLATFORM_IMAGE is set (its own CI job builds/pulls the image and sets it) requires_gpu: Container e2e requiring GPU job scheduling; skipped unless --feature gpu is passed regression: Regression tests - test individual functional microservices for baseline functionality infrastructure: Infrastructure tests - ensure services are compatible with customer infrastructure