NVIDIA-NeMo · ironcommit · Jul 23, 2026 · Jul 23, 2026
@@ -0,0 +1,61 @@
+name: Finalize Kind e2e
+description: Collects diagnostics, uploads artifacts, and deletes a Kind cluster.
+
+inputs:
+  kind-cluster-name:
+    description: Kind cluster name to inspect and delete.
+    required: true
+  artifact-name:
+    description: Uploaded artifact name.
+    required: true
+  junit-report:
+    description: JUnit XML report path to include in uploaded artifacts.
+    required: true
+  services-log-dir:
+    description: E2E services log directory to include in uploaded artifacts.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Collect Kubernetes logs
+      if: always()
+      shell: bash
+      run: e2e/k8s/scripts/collect_k8s_logs.sh
+
+    - name: Disk usage summary
+      if: always()
+      shell: bash
+      env:
+        KIND_CLUSTER_NAME: ${{ inputs.kind-cluster-name }}
+      run: |
+        echo "=== Host disk ==="
+        df -h /
+        echo "=== Docker system ==="
+        docker system df
+        echo "=== kind node storage ==="
+        for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do
+          echo "--- ${node} ---"
+          docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true
+        done
+
+    - name: Upload Kubernetes artifacts
+      if: always()
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: ${{ inputs.artifact-name }}
+        retention-days: 7
+        if-no-files-found: ignore
+        path: |
+          k8s-logs/
+          ${{ inputs.junit-report }}
+          ${{ inputs.services-log-dir }}
+
+    - name: Delete kind cluster
+      if: always()
+      shell: bash
+      env:
+        KIND_CLUSTER_NAME: ${{ inputs.kind-cluster-name }}
+      run: |
+        docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true
+        kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
@@ -453,43 +453,14 @@ jobs:
             --no-cov \
             --junitxml=report-kubernetes-smoke.xml
 
-      - name: Collect Kubernetes logs
-        if: always()
-        shell: bash
-        run: e2e/k8s/scripts/collect_k8s_logs.sh
-
-      - name: Disk usage summary
+      - name: Finalize Kind e2e
         if: always()
-        shell: bash
-        run: |
-          echo "=== Host disk ==="
-          df -h /
-          echo "=== Docker system ==="
-          docker system df
-          echo "=== kind node storage ==="
-          for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do
-            echo "--- ${node} ---"
-            docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true
-          done
-
-      - name: Upload Kubernetes artifacts
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: ./.github/actions/finalize-kind-e2e
         with:
-          name: kind-smoke-kubernetes-artifacts
-          retention-days: 7
-          if-no-files-found: ignore
-          path: |
-            k8s-logs/
-            report-kubernetes-smoke.xml
-            ${{ runner.temp }}/e2e-services-logs/
-
-      - name: Delete kind cluster
-        if: always()
-        shell: bash
-        run: |
-          docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true
-          kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
+          kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
+          artifact-name: kind-smoke-kubernetes-artifacts
+          junit-report: report-kubernetes-smoke.xml
+          services-log-dir: ${{ runner.temp }}/e2e-services-logs
 
   kind-cpu-e2e:
     name: Kind CPU e2e
@@ -543,43 +514,14 @@ jobs:
             --no-cov \
             --junitxml=report-kubernetes-e2e.xml
 
-      - name: Collect Kubernetes logs
-        if: always()
-        shell: bash
-        run: e2e/k8s/scripts/collect_k8s_logs.sh
-
-      - name: Disk usage summary
+      - name: Finalize Kind e2e
         if: always()
-        shell: bash
-        run: |
-          echo "=== Host disk ==="
-          df -h /
-          echo "=== Docker system ==="
-          docker system df
-          echo "=== kind node storage ==="
-          for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do
-            echo "--- ${node} ---"
-            docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true
-          done
-
-      - name: Upload Kubernetes artifacts
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: ./.github/actions/finalize-kind-e2e
         with:
-          name: kind-e2e-kubernetes-artifacts
-          retention-days: 7
-          if-no-files-found: ignore
-          path: |
-            k8s-logs/
-            report-kubernetes-e2e.xml
-            ${{ runner.temp }}/e2e-services-logs/
-
-      - name: Delete kind cluster
-        if: always()
-        shell: bash
-        run: |
-          docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true
-          kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
+          kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
+          artifact-name: kind-e2e-kubernetes-artifacts
+          junit-report: report-kubernetes-e2e.xml
+          services-log-dir: ${{ runner.temp }}/e2e-services-logs
 
   helm-lint:
     name: Helm lint

@@ -65,22 +65,25 @@
 stay focused on pytest hooks and fixtures.
 """
 
-import logging
 import os
-import tempfile
 import uuid
 from collections.abc import Iterator
-from pathlib import Path
 
 import pytest
 from nemo_platform import NeMoPlatform
 from nemo_platform_plugin.client.adapter import client_from_platform
 from nemo_platform_plugin.files.client import FilesClient
 
-from e2e.services_pool import E2EServicesPool, RunningServices, admin_headers
-
-_services_pool_manager_key = pytest.StashKey[E2EServicesPool]()
-_services_metadata_key = pytest.StashKey[dict[str, str]]()
+from e2e.services_pool_fixtures import (  # noqa: F401
+    _services,
+    _services_instance,
+    _services_log_key,
+    _services_pool_manager,
+    append_services_pool_report_sections,
+    configure_services_pool,
+    register_services_pool_items,
+    services_pool_sdk,
+)
 
 
 def pytest_configure(config: pytest.Config) -> None:
@@ -91,28 +94,13 @@ def pytest_configure(config: pytest.Config) -> None:
     cleared because the config module evaluates ``get_service_config()``
     at import time, which may run before this hook.
     """
-    os.environ.setdefault("NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX", "igw-mock-")
-
-    from nemo_platform_plugin.config import Configuration
-
-    Configuration.clear_cache()
-    config.stash[_services_pool_manager_key] = E2EServicesPool()
+    configure_services_pool(config)
 
 
 def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]) -> None:
     """Register collected E2E modules with the services pool manager."""
-    config.stash[_services_pool_manager_key].register_collected_items(items)
-
-
-logger = logging.getLogger(__name__)
-_E2E_HARNESS_DEBUG = os.environ.get("E2E_HARNESS_DEBUG") == "1"
-
-_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log")))
+    register_services_pool_items(config, items)
 
-# Number of log lines to dump from the services log on test failure.
-_TAIL_LINES_ON_FAILURE = 100
-
-_services_log_key = pytest.StashKey[Path]()
 
 NGC_API_KEY_ENV = "NGC_API_KEY"
 
@@ -155,104 +143,13 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo):  # noqa
     """
     outcome = yield
     report = outcome.get_result()
-
-    if not report.failed:
-        return
-
-    metadata: dict[str, str] | None = None
-    module = item.getparent(pytest.Module)
-    if module is not None:
-        manager = item.config.stash[_services_pool_manager_key]
-        active_metadata = manager.describe_active_module_binding(module.nodeid)
-        if active_metadata:
-            metadata = {key: str(value) for key, value in active_metadata.items() if value is not None}
-
-    log_path: Path | None = None
-    if metadata and metadata.get("service_log_path"):
-        log_path = Path(metadata["service_log_path"])
-    if log_path is None:
-        log_path = item.session.stash.get(_services_log_key, None)
-    if log_path and log_path.exists():
-        lines = log_path.read_text().splitlines(keepends=True)
-        tail = lines[-_TAIL_LINES_ON_FAILURE:]
-        if tail:
-            header = f"--- services log (last {len(tail)} lines) [{log_path}] ---"
-            report.sections.append(("Services Log", f"{header}\n{''.join(tail)}"))
-    if metadata:
-        report.sections.append(
-            (
-                "E2E Services Binding",
-                "\n".join(f"{key}: {value}" for key, value in sorted(metadata.items())),
-            )
-        )
+    append_services_pool_report_sections(item, report, metadata_section_name="E2E Services Binding")
 
 
-# ---- Fixtures --------------------------------------------------------------
-@pytest.fixture(scope="session")
-def _services_pool_manager(
-    request: pytest.FixtureRequest,
-    tmp_path_factory: pytest.TempPathFactory,
-) -> Iterator[E2EServicesPool]:
-    manager = request.config.stash[_services_pool_manager_key]
-    manager.bind_tmp_path_factory(tmp_path_factory)
-    yield manager
-    manager.shutdown_all()
-
-
-@pytest.fixture(scope="module")
-def _services_instance(
-    request: pytest.FixtureRequest,
-    _services_pool_manager: E2EServicesPool,
-) -> Iterator[RunningServices]:
-    """Return the running services instance for the current module's config.
-
-    Skipped when ``NMP_BASE_URL`` is already set (external services).
-
-    Modules do not each get a dedicated services process. Instead, the harness
-    computes the effective config hash for the module and reuses any existing
-    process already started for that hash within the pytest session. A new
-    process is started only when the module resolves to a config that no prior
-    module has used.
-    """
-    module = request.node.getparent(pytest.Module)
-    if module is None:
-        raise RuntimeError("Expected module-scoped E2E fixture to have a pytest module parent")
-    services = _services_pool_manager.acquire_for_module(module)
-    if services.log_path is not None:
-        request.session.stash[_services_log_key] = services.log_path
-    try:
-        yield services
-    finally:
-        _services_pool_manager.release_for_module(module)
-
-
-@pytest.fixture(scope="module")
-def _services(_services_instance: RunningServices) -> Iterator[str]:
-    yield _services_instance.url
-
-
-@pytest.fixture(scope="module")
-def sdk(_services: str, _services_instance: RunningServices) -> NeMoPlatform:
-    """Provide an SDK client connected to the running platform.
-
-    When connecting to an external cluster (via ``NMP_BASE_URL``), authentication
-    can be provided through:
-    - ``NMP_ACCESS_TOKEN`` env var (e.g. from ``nemo auth token``)
-    - ``NMP_CONTEXT_NAME`` env var (e.g. ``tot``) to read credentials from CLI config
-
-    For local auth-enabled deployments, admin headers are injected via
-    ``default_headers`` based on the rendered platform config.
-    """
-    access_token = os.environ.get("NMP_ACCESS_TOKEN")
-    context_name = os.environ.get("NMP_CONTEXT_NAME")
-    headers = admin_headers() if _services_instance.auth_enabled else {}
-    return NeMoPlatform(
-        base_url=_services,
-        access_token=access_token,
-        context_name=context_name,
-        max_retries=2,
-        default_headers=headers,
-    )
+@pytest.fixture(scope="module", name="sdk")
+def e2e_sdk(request: pytest.FixtureRequest) -> NeMoPlatform:
+    """Provide the conventional e2e SDK fixture name."""
+    return request.getfixturevalue("services_pool_sdk")
 
 
 @pytest.fixture(scope="module")

@@ -23,7 +23,7 @@ fi
 if [ "$#" -gt 0 ]; then
     PYTEST_TARGETS=("$@")
 else
-    PYTEST_TARGETS=("e2e/test_workspaces.py" "e2e/test_jobs_auth.py")
+    PYTEST_TARGETS=()
 fi
 
 wait_for_url() {
@@ -73,6 +73,9 @@ fi
 echo "Authenticated request check passed"
 
 export NMP_E2E_CLUSTER_URL="${BASE_URL}"
+export NMP_BASE_URL="${BASE_URL}"
 
 cd "${REPO_ROOT}"
-uv run --project "${UV_PROJECT}" --frozen pytest "${PYTEST_TARGETS[@]}" --kubernetes --feature auth -v
+if [ "${#PYTEST_TARGETS[@]}" -gt 0 ]; then
+    uv run --project "${UV_PROJECT}" --frozen pytest "${PYTEST_TARGETS[@]}" --run-e2e --feature auth -v
+fi
@@ -586,7 +586,7 @@ def with_e2e_instance_paths(config_data: dict[str, Any], data_dir: Path) -> dict
 # session, so a pool platform spawned afterward would otherwise discover them too — and
 # with the ``on_invalid_plugin=hard_fail`` default an unruled fixture aborts the whole OPA
 # bundle ("Policy data not loaded — refusing to evaluate"), 502-ing every request and
-# wedging unrelated auth tests (e.g. test_jobs_auth). Pool platforms therefore pin the
+# wedging unrelated auth job integration tests. Pool platforms therefore pin the
 # service-plugin allowlist to the real installed plugins, fencing the fixtures out.
 # (authz_oidc spawns its own platforms with their own env, opting into the fixtures with
 # deny_route/quarantine, so it is unaffected by this.)