From 4cfa4438b6783a39c1daffc79f2b158882c71ce5 Mon Sep 17 00:00:00 2001 From: Ryan S <267728323+ironcommit@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:43:59 -0700 Subject: [PATCH] chore(auth): enable skipped e2e tests Signed-off-by: Ryan S <267728323+ironcommit@users.noreply.github.com> --- .github/actions/finalize-kind-e2e/action.yaml | 61 ++++++++ .github/workflows/ci.yaml | 82 ++--------- e2e/conftest.py | 137 +++--------------- e2e/k8s/scripts/run_auth_e2e.sh | 7 +- e2e/services_pool.py | 2 +- e2e/services_pool_fixtures.py | 134 +++++++++++++++++ pytest.ini | 1 + tests/auth/integration/conftest.py | 32 ++++ tests/auth/integration/jobs_auth_helpers.py | 20 +++ .../auth/integration}/test_jobs_auth.py | 70 +++++---- .../test_jobs_auth_helpers.py} | 8 +- tests/test_e2e_services_pool.py | 17 +++ 12 files changed, 338 insertions(+), 233 deletions(-) create mode 100644 .github/actions/finalize-kind-e2e/action.yaml create mode 100644 e2e/services_pool_fixtures.py create mode 100644 tests/auth/integration/conftest.py create mode 100644 tests/auth/integration/jobs_auth_helpers.py rename {e2e => tests/auth/integration}/test_jobs_auth.py (79%) rename tests/{test_e2e_jobs_auth.py => auth/test_jobs_auth_helpers.py} (85%) diff --git a/.github/actions/finalize-kind-e2e/action.yaml b/.github/actions/finalize-kind-e2e/action.yaml new file mode 100644 index 0000000000..9d3c602aae --- /dev/null +++ b/.github/actions/finalize-kind-e2e/action.yaml @@ -0,0 +1,61 @@ +name: Finalize Kind e2e +description: Collects diagnostics, uploads artifacts, and deletes a Kind cluster. + +inputs: + kind-cluster-name: + description: Kind cluster name to inspect and delete. + required: true + artifact-name: + description: Uploaded artifact name. + required: true + junit-report: + description: JUnit XML report path to include in uploaded artifacts. + required: true + services-log-dir: + description: E2E services log directory to include in uploaded artifacts. + required: true + +runs: + using: composite + steps: + - name: Collect Kubernetes logs + if: always() + shell: bash + run: e2e/k8s/scripts/collect_k8s_logs.sh + + - name: Disk usage summary + if: always() + shell: bash + env: + KIND_CLUSTER_NAME: ${{ inputs.kind-cluster-name }} + run: | + echo "=== Host disk ===" + df -h / + echo "=== Docker system ===" + docker system df + echo "=== kind node storage ===" + for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do + echo "--- ${node} ---" + docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true + done + + - name: Upload Kubernetes artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact-name }} + retention-days: 7 + if-no-files-found: ignore + path: | + k8s-logs/ + ${{ inputs.junit-report }} + ${{ inputs.services-log-dir }} + + - name: Delete kind cluster + if: always() + shell: bash + env: + KIND_CLUSTER_NAME: ${{ inputs.kind-cluster-name }} + run: | + docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true + kind delete cluster --name "${KIND_CLUSTER_NAME}" || true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4b0e54e2fb..ce60915d51 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -453,43 +453,14 @@ jobs: --no-cov \ --junitxml=report-kubernetes-smoke.xml - - name: Collect Kubernetes logs - if: always() - shell: bash - run: e2e/k8s/scripts/collect_k8s_logs.sh - - - name: Disk usage summary + - name: Finalize Kind e2e if: always() - shell: bash - run: | - echo "=== Host disk ===" - df -h / - echo "=== Docker system ===" - docker system df - echo "=== kind node storage ===" - for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do - echo "--- ${node} ---" - docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true - done - - - name: Upload Kubernetes artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: ./.github/actions/finalize-kind-e2e with: - name: kind-smoke-kubernetes-artifacts - retention-days: 7 - if-no-files-found: ignore - path: | - k8s-logs/ - report-kubernetes-smoke.xml - ${{ runner.temp }}/e2e-services-logs/ - - - name: Delete kind cluster - if: always() - shell: bash - run: | - docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true - kind delete cluster --name "${KIND_CLUSTER_NAME}" || true + kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }} + artifact-name: kind-smoke-kubernetes-artifacts + junit-report: report-kubernetes-smoke.xml + services-log-dir: ${{ runner.temp }}/e2e-services-logs kind-cpu-e2e: name: Kind CPU e2e @@ -543,43 +514,14 @@ jobs: --no-cov \ --junitxml=report-kubernetes-e2e.xml - - name: Collect Kubernetes logs - if: always() - shell: bash - run: e2e/k8s/scripts/collect_k8s_logs.sh - - - name: Disk usage summary + - name: Finalize Kind e2e if: always() - shell: bash - run: | - echo "=== Host disk ===" - df -h / - echo "=== Docker system ===" - docker system df - echo "=== kind node storage ===" - for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}" 2>/dev/null); do - echo "--- ${node} ---" - docker exec "${node}" sh -c "du -sh /var/lib/containerd /var/lib/kubelet /var/log 2>/dev/null | sort -h" || true - done - - - name: Upload Kubernetes artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: ./.github/actions/finalize-kind-e2e with: - name: kind-e2e-kubernetes-artifacts - retention-days: 7 - if-no-files-found: ignore - path: | - k8s-logs/ - report-kubernetes-e2e.xml - ${{ runner.temp }}/e2e-services-logs/ - - - name: Delete kind cluster - if: always() - shell: bash - run: | - docker rm -f "cloud-provider-kind-${KIND_CLUSTER_NAME}" || true - kind delete cluster --name "${KIND_CLUSTER_NAME}" || true + kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }} + artifact-name: kind-e2e-kubernetes-artifacts + junit-report: report-kubernetes-e2e.xml + services-log-dir: ${{ runner.temp }}/e2e-services-logs helm-lint: name: Helm lint diff --git a/e2e/conftest.py b/e2e/conftest.py index 14938026bb..c3b928e639 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -65,22 +65,25 @@ stay focused on pytest hooks and fixtures. """ -import logging import os -import tempfile import uuid from collections.abc import Iterator -from pathlib import Path import pytest from nemo_platform import NeMoPlatform from nemo_platform_plugin.client.adapter import client_from_platform from nemo_platform_plugin.files.client import FilesClient -from e2e.services_pool import E2EServicesPool, RunningServices, admin_headers - -_services_pool_manager_key = pytest.StashKey[E2EServicesPool]() -_services_metadata_key = pytest.StashKey[dict[str, str]]() +from e2e.services_pool_fixtures import ( # noqa: F401 + _services, + _services_instance, + _services_log_key, + _services_pool_manager, + append_services_pool_report_sections, + configure_services_pool, + register_services_pool_items, + services_pool_sdk, +) def pytest_configure(config: pytest.Config) -> None: @@ -91,28 +94,13 @@ def pytest_configure(config: pytest.Config) -> None: cleared because the config module evaluates ``get_service_config()`` at import time, which may run before this hook. """ - os.environ.setdefault("NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX", "igw-mock-") - - from nemo_platform_plugin.config import Configuration - - Configuration.clear_cache() - config.stash[_services_pool_manager_key] = E2EServicesPool() + configure_services_pool(config) def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]) -> None: """Register collected E2E modules with the services pool manager.""" - config.stash[_services_pool_manager_key].register_collected_items(items) - - -logger = logging.getLogger(__name__) -_E2E_HARNESS_DEBUG = os.environ.get("E2E_HARNESS_DEBUG") == "1" - -_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log"))) + register_services_pool_items(config, items) -# Number of log lines to dump from the services log on test failure. -_TAIL_LINES_ON_FAILURE = 100 - -_services_log_key = pytest.StashKey[Path]() NGC_API_KEY_ENV = "NGC_API_KEY" @@ -155,104 +143,13 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # noqa """ outcome = yield report = outcome.get_result() - - if not report.failed: - return - - metadata: dict[str, str] | None = None - module = item.getparent(pytest.Module) - if module is not None: - manager = item.config.stash[_services_pool_manager_key] - active_metadata = manager.describe_active_module_binding(module.nodeid) - if active_metadata: - metadata = {key: str(value) for key, value in active_metadata.items() if value is not None} - - log_path: Path | None = None - if metadata and metadata.get("service_log_path"): - log_path = Path(metadata["service_log_path"]) - if log_path is None: - log_path = item.session.stash.get(_services_log_key, None) - if log_path and log_path.exists(): - lines = log_path.read_text().splitlines(keepends=True) - tail = lines[-_TAIL_LINES_ON_FAILURE:] - if tail: - header = f"--- services log (last {len(tail)} lines) [{log_path}] ---" - report.sections.append(("Services Log", f"{header}\n{''.join(tail)}")) - if metadata: - report.sections.append( - ( - "E2E Services Binding", - "\n".join(f"{key}: {value}" for key, value in sorted(metadata.items())), - ) - ) + append_services_pool_report_sections(item, report, metadata_section_name="E2E Services Binding") -# ---- Fixtures -------------------------------------------------------------- -@pytest.fixture(scope="session") -def _services_pool_manager( - request: pytest.FixtureRequest, - tmp_path_factory: pytest.TempPathFactory, -) -> Iterator[E2EServicesPool]: - manager = request.config.stash[_services_pool_manager_key] - manager.bind_tmp_path_factory(tmp_path_factory) - yield manager - manager.shutdown_all() - - -@pytest.fixture(scope="module") -def _services_instance( - request: pytest.FixtureRequest, - _services_pool_manager: E2EServicesPool, -) -> Iterator[RunningServices]: - """Return the running services instance for the current module's config. - - Skipped when ``NMP_BASE_URL`` is already set (external services). - - Modules do not each get a dedicated services process. Instead, the harness - computes the effective config hash for the module and reuses any existing - process already started for that hash within the pytest session. A new - process is started only when the module resolves to a config that no prior - module has used. - """ - module = request.node.getparent(pytest.Module) - if module is None: - raise RuntimeError("Expected module-scoped E2E fixture to have a pytest module parent") - services = _services_pool_manager.acquire_for_module(module) - if services.log_path is not None: - request.session.stash[_services_log_key] = services.log_path - try: - yield services - finally: - _services_pool_manager.release_for_module(module) - - -@pytest.fixture(scope="module") -def _services(_services_instance: RunningServices) -> Iterator[str]: - yield _services_instance.url - - -@pytest.fixture(scope="module") -def sdk(_services: str, _services_instance: RunningServices) -> NeMoPlatform: - """Provide an SDK client connected to the running platform. - - When connecting to an external cluster (via ``NMP_BASE_URL``), authentication - can be provided through: - - ``NMP_ACCESS_TOKEN`` env var (e.g. from ``nemo auth token``) - - ``NMP_CONTEXT_NAME`` env var (e.g. ``tot``) to read credentials from CLI config - - For local auth-enabled deployments, admin headers are injected via - ``default_headers`` based on the rendered platform config. - """ - access_token = os.environ.get("NMP_ACCESS_TOKEN") - context_name = os.environ.get("NMP_CONTEXT_NAME") - headers = admin_headers() if _services_instance.auth_enabled else {} - return NeMoPlatform( - base_url=_services, - access_token=access_token, - context_name=context_name, - max_retries=2, - default_headers=headers, - ) +@pytest.fixture(scope="module", name="sdk") +def e2e_sdk(request: pytest.FixtureRequest) -> NeMoPlatform: + """Provide the conventional e2e SDK fixture name.""" + return request.getfixturevalue("services_pool_sdk") @pytest.fixture(scope="module") diff --git a/e2e/k8s/scripts/run_auth_e2e.sh b/e2e/k8s/scripts/run_auth_e2e.sh index 2e5016b7b8..80f31f0257 100755 --- a/e2e/k8s/scripts/run_auth_e2e.sh +++ b/e2e/k8s/scripts/run_auth_e2e.sh @@ -23,7 +23,7 @@ fi if [ "$#" -gt 0 ]; then PYTEST_TARGETS=("$@") else - PYTEST_TARGETS=("e2e/test_workspaces.py" "e2e/test_jobs_auth.py") + PYTEST_TARGETS=() fi wait_for_url() { @@ -73,6 +73,9 @@ fi echo "Authenticated request check passed" export NMP_E2E_CLUSTER_URL="${BASE_URL}" +export NMP_BASE_URL="${BASE_URL}" cd "${REPO_ROOT}" -uv run --project "${UV_PROJECT}" --frozen pytest "${PYTEST_TARGETS[@]}" --kubernetes --feature auth -v +if [ "${#PYTEST_TARGETS[@]}" -gt 0 ]; then + uv run --project "${UV_PROJECT}" --frozen pytest "${PYTEST_TARGETS[@]}" --run-e2e --feature auth -v +fi diff --git a/e2e/services_pool.py b/e2e/services_pool.py index 444f975660..c27093c99a 100644 --- a/e2e/services_pool.py +++ b/e2e/services_pool.py @@ -586,7 +586,7 @@ def with_e2e_instance_paths(config_data: dict[str, Any], data_dir: Path) -> dict # session, so a pool platform spawned afterward would otherwise discover them too — and # with the ``on_invalid_plugin=hard_fail`` default an unruled fixture aborts the whole OPA # bundle ("Policy data not loaded — refusing to evaluate"), 502-ing every request and -# wedging unrelated auth tests (e.g. test_jobs_auth). Pool platforms therefore pin the +# wedging unrelated auth job integration tests. Pool platforms therefore pin the # service-plugin allowlist to the real installed plugins, fencing the fixtures out. # (authz_oidc spawns its own platforms with their own env, opting into the fixtures with # deny_route/quarantine, so it is unaffected by this.) diff --git a/e2e/services_pool_fixtures.py b/e2e/services_pool_fixtures.py new file mode 100644 index 0000000000..39f99c07bd --- /dev/null +++ b/e2e/services_pool_fixtures.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reusable pytest fixtures backed by the E2E services pool.""" + +import os +from collections import deque +from collections.abc import Iterator +from pathlib import Path + +import pytest +from _pytest.reports import TestReport +from nemo_platform import DefaultHttpxClient, NeMoPlatform + +from e2e.services_pool import E2EServicesPool, RunningServices, admin_headers + +_services_pool_manager_key = pytest.StashKey[E2EServicesPool]() +_services_log_key = pytest.StashKey[dict[str, Path] | Path]() + +_TAIL_LINES_ON_FAILURE = 100 + + +def _read_services_log_tail(log_path: Path) -> list[str]: + with log_path.open() as log_file: + return list(deque(log_file, maxlen=_TAIL_LINES_ON_FAILURE)) + + +def configure_services_pool(config: pytest.Config, *, configure_mock_provider: bool = True) -> None: + """Initialize the shared service-pool manager for a pytest session.""" + if configure_mock_provider: + os.environ.setdefault("NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX", "igw-mock-") + + from nemo_platform_plugin.config import Configuration + + Configuration.clear_cache() + if config.stash.get(_services_pool_manager_key, None) is None: + config.stash[_services_pool_manager_key] = E2EServicesPool() + + +def register_services_pool_items(config: pytest.Config, items: list[pytest.Item]) -> None: + """Register collected modules that may use the service pool.""" + config.stash[_services_pool_manager_key].register_collected_items(items) + + +def append_services_pool_report_sections( + item: pytest.Item, + report: TestReport, + *, + metadata_section_name: str, +) -> None: + """Append service-pool metadata and log tails to failed test reports.""" + if not report.failed: + return + + metadata: dict[str, str] | None = None + module = item.getparent(pytest.Module) + if module is not None: + manager = item.config.stash[_services_pool_manager_key] + active_metadata = manager.describe_active_module_binding(module.nodeid) + if active_metadata: + metadata = {key: str(value) for key, value in active_metadata.items() if value is not None} + + log_path: Path | None = None + if metadata and metadata.get("service_log_path"): + log_path = Path(metadata["service_log_path"]) + if log_path is None and module is not None: + log_paths_by_module = item.session.stash.get(_services_log_key, None) + if isinstance(log_paths_by_module, dict): + log_path = log_paths_by_module.get(module.nodeid) + if log_path and log_path.exists(): + tail = _read_services_log_tail(log_path) + if tail: + header = f"--- services log (last {len(tail)} lines) [{log_path}] ---" + report.sections.append(("Services Log", f"{header}\n{''.join(tail)}")) + if metadata: + report.sections.append( + ( + metadata_section_name, + "\n".join(f"{key}: {value}" for key, value in sorted(metadata.items())), + ) + ) + + +@pytest.fixture(scope="session") +def _services_pool_manager( + request: pytest.FixtureRequest, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[E2EServicesPool]: + manager = request.config.stash[_services_pool_manager_key] + manager.bind_tmp_path_factory(tmp_path_factory) + yield manager + manager.shutdown_all() + + +@pytest.fixture(scope="module") +def _services_instance( + request: pytest.FixtureRequest, + _services_pool_manager: E2EServicesPool, +) -> Iterator[RunningServices]: + module = request.node.getparent(pytest.Module) + if module is None: + raise RuntimeError("Expected module-scoped service-pool fixture to have a pytest module parent") + services = _services_pool_manager.acquire_for_module(module) + if services.log_path is not None: + log_paths_by_module = request.session.stash.get(_services_log_key, None) + if not isinstance(log_paths_by_module, dict): + log_paths_by_module = {} + request.session.stash[_services_log_key] = log_paths_by_module + log_paths_by_module[module.nodeid] = services.log_path + try: + yield services + finally: + _services_pool_manager.release_for_module(module) + + +@pytest.fixture(scope="module") +def _services(_services_instance: RunningServices) -> Iterator[str]: + yield _services_instance.url + + +@pytest.fixture(scope="module", name="services_pool_sdk") +def services_pool_sdk(_services: str, _services_instance: RunningServices) -> NeMoPlatform: + access_token = os.environ.get("NMP_ACCESS_TOKEN") + context_name = os.environ.get("NMP_CONTEXT_NAME") + headers = admin_headers() if _services_instance.auth_enabled else {} + http_client = DefaultHttpxClient(base_url=_services, verify=True) if _services_instance.proc is not None else None + return NeMoPlatform( + base_url=_services, + access_token=access_token, + context_name=context_name, + http_client=http_client, + max_retries=2, + default_headers=headers, + ) diff --git a/pytest.ini b/pytest.ini index 5131d6eced..0568982260 100644 --- a/pytest.ini +++ b/pytest.ini @@ -53,6 +53,7 @@ testpaths = services/safe-synthesizer-api/tests docs/_scripts docs/fern/scripts + tests/auth tests/integration tests/unit diff --git a/tests/auth/integration/conftest.py b/tests/auth/integration/conftest.py new file mode 100644 index 0000000000..f80164d2c0 --- /dev/null +++ b/tests/auth/integration/conftest.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Top-level auth integration fixtures backed by the E2E services pool.""" + +import pytest + +from e2e.services_pool_fixtures import ( # noqa: F401 + _services, + _services_instance, + _services_pool_manager, + append_services_pool_report_sections, + configure_services_pool, + register_services_pool_items, + services_pool_sdk, +) + + +def pytest_configure(config: pytest.Config) -> None: + configure_services_pool(config, configure_mock_provider=False) + + +def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]) -> None: + pool_items = [item for item in items if item.get_closest_marker("e2e_config") is not None] + register_services_pool_items(config, pool_items) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # noqa: ARG001 + outcome = yield + report = outcome.get_result() + append_services_pool_report_sections(item, report, metadata_section_name="Auth Integration Services Binding") diff --git a/tests/auth/integration/jobs_auth_helpers.py b/tests/auth/integration/jobs_auth_helpers.py new file mode 100644 index 0000000000..09188f4de4 --- /dev/null +++ b/tests/auth/integration/jobs_auth_helpers.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Iterator +from contextlib import contextmanager + +from nemo_platform import NeMoPlatform + + +@contextmanager +def managed_admin_workspace(admin_sdk: NeMoPlatform, workspace_name: str) -> Iterator[str]: + admin_sdk.workspaces.create(name=workspace_name) + try: + yield workspace_name + finally: + admin_sdk.workspaces.delete(workspace_name) + + +def job_exists_in_pages(jobs_page: object, job_name: str) -> bool: + return any(item.name == job_name for page in jobs_page.iter_pages() for item in page.data) diff --git a/e2e/test_jobs_auth.py b/tests/auth/integration/test_jobs_auth.py similarity index 79% rename from e2e/test_jobs_auth.py rename to tests/auth/integration/test_jobs_auth.py index 463d26d61e..cae83ca9c7 100644 --- a/e2e/test_jobs_auth.py +++ b/tests/auth/integration/test_jobs_auth.py @@ -1,13 +1,16 @@ -"""E2E tests for jobs with auth enabled. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 -Local E2E runs translate ``cpu/default`` container steps to the subprocess -backend, so these tests intentionally omit ``container.image`` and rely only on -the command shape that subprocess consumes. +"""Integration tests for jobs with auth enabled. + +These tests run against the auth-enabled subprocess services pool. The job steps +use the local cpu/default -> subprocess translation, so the suite does not +require a container runtime or Kubernetes cluster. """ import logging -from collections.abc import Iterator -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack +from uuid import uuid4 import pytest from nemo_platform import NeMoPlatform @@ -27,7 +30,9 @@ from nmp.testing import TEST_ADMIN_EMAIL, grant_workspace_role, short_unique_name, unique_email from nmp.testing.e2e import wait_for_platform_job -JOB_SOURCE = "e2e-auth-test" +from tests.auth.integration.jobs_auth_helpers import job_exists_in_pages, managed_admin_workspace + +JOB_SOURCE = "integration-auth-test" logger = logging.getLogger(__name__) pytestmark = [ @@ -40,16 +45,22 @@ def _as_bearer_user( sdk: NeMoPlatform, email: str, *, + principal_id: str | None = None, groups: list[str] | None = None, ) -> NeMoPlatform: token = generate_unsigned_jwt( - principal_id=email, + principal_id=principal_id or email, email=email, groups=groups, ) return sdk.with_options(set_default_headers={"Authorization": f"Bearer {token}"}) +def _oidc_subject() -> str: + """Return an IdP-style subject id distinct from the user's email.""" + return str(uuid4()) + + def _log_auth_job_diagnostics( sdk: NeMoPlatform, *, @@ -76,28 +87,15 @@ def _log_auth_job_diagnostics( ) -@contextmanager -def _managed_admin_workspace(admin_sdk: NeMoPlatform, workspace_name: str) -> Iterator[str]: - admin_sdk.workspaces.create(name=workspace_name) - try: - yield workspace_name - finally: - admin_sdk.workspaces.delete(workspace_name) - - -def _job_exists_in_pages(jobs_page: object, job_name: str) -> bool: - return any(item.name == job_name for page in jobs_page.iter_pages() for item in page.data) - - -def test_job_principal_propagation(sdk: NeMoPlatform): - admin_sdk = _as_bearer_user(sdk, TEST_ADMIN_EMAIL, groups=["admin"]) +def test_job_principal_propagation(services_pool_sdk: NeMoPlatform): + admin_sdk = _as_bearer_user(services_pool_sdk, TEST_ADMIN_EMAIL, groups=["admin"]) user_email = unique_email("job-creator") workspace_name = short_unique_name("job-auth-test") - with _managed_admin_workspace(admin_sdk, workspace_name): + with managed_admin_workspace(admin_sdk, workspace_name): grant_workspace_role(admin_sdk, workspace=workspace_name, principal=user_email, roles=["Editor"]) - user_sdk = _as_bearer_user(sdk, user_email) + user_sdk = _as_bearer_user(services_pool_sdk, user_email, principal_id=_oidc_subject()) job = user_sdk.jobs.create( workspace=workspace_name, source=JOB_SOURCE, @@ -136,8 +134,8 @@ def test_job_principal_propagation(sdk: NeMoPlatform): assert file_content == b"auth propagation test" -def test_job_cannot_access_unauthorized_workspace(sdk: NeMoPlatform): - admin_sdk = _as_bearer_user(sdk, TEST_ADMIN_EMAIL, groups=["admin"]) +def test_job_cannot_access_unauthorized_workspace(services_pool_sdk: NeMoPlatform): + admin_sdk = _as_bearer_user(services_pool_sdk, TEST_ADMIN_EMAIL, groups=["admin"]) owner_email = unique_email("owner") other_email = unique_email("other") @@ -145,13 +143,13 @@ def test_job_cannot_access_unauthorized_workspace(sdk: NeMoPlatform): runner_workspace = short_unique_name("runner") with ExitStack() as stack: - stack.enter_context(_managed_admin_workspace(admin_sdk, restricted_workspace)) - stack.enter_context(_managed_admin_workspace(admin_sdk, runner_workspace)) + stack.enter_context(managed_admin_workspace(admin_sdk, restricted_workspace)) + stack.enter_context(managed_admin_workspace(admin_sdk, runner_workspace)) grant_workspace_role(admin_sdk, workspace=restricted_workspace, principal=owner_email, roles=["Editor"]) grant_workspace_role(admin_sdk, workspace=runner_workspace, principal=other_email, roles=["Editor"]) - owner_sdk = _as_bearer_user(sdk, owner_email) - other_sdk = _as_bearer_user(sdk, other_email) + owner_sdk = _as_bearer_user(services_pool_sdk, owner_email, principal_id=_oidc_subject()) + other_sdk = _as_bearer_user(services_pool_sdk, other_email, principal_id=_oidc_subject()) fileset_name = "private-data" files = client_from_platform(owner_sdk, FilesClient) @@ -215,15 +213,15 @@ def test_job_cannot_access_unauthorized_workspace(sdk: NeMoPlatform): assert "403" in task.error_stack and "Forbidden" in task.error_stack -def test_job_admin_can_list_jobs_in_all_workspaces(sdk: NeMoPlatform): - admin_sdk = _as_bearer_user(sdk, TEST_ADMIN_EMAIL, groups=["admin"]) +def test_job_admin_can_list_jobs_in_all_workspaces(services_pool_sdk: NeMoPlatform): + admin_sdk = _as_bearer_user(services_pool_sdk, TEST_ADMIN_EMAIL, groups=["admin"]) user_email = unique_email("member") workspace_name = short_unique_name("admin-list-jobs") - with _managed_admin_workspace(admin_sdk, workspace_name): + with managed_admin_workspace(admin_sdk, workspace_name): grant_workspace_role(admin_sdk, workspace=workspace_name, principal=user_email, roles=["Editor"]) - user_sdk = _as_bearer_user(sdk, user_email) + user_sdk = _as_bearer_user(services_pool_sdk, user_email, principal_id=_oidc_subject()) job = user_sdk.jobs.create( workspace=workspace_name, source=JOB_SOURCE, @@ -248,4 +246,4 @@ def test_job_admin_can_list_jobs_in_all_workspaces(sdk: NeMoPlatform): jobs = admin_sdk.jobs.list(workspace=ALL_WORKSPACES) assert jobs.pagination is not None - assert _job_exists_in_pages(jobs, job.name) + assert job_exists_in_pages(jobs, job.name) diff --git a/tests/test_e2e_jobs_auth.py b/tests/auth/test_jobs_auth_helpers.py similarity index 85% rename from tests/test_e2e_jobs_auth.py rename to tests/auth/test_jobs_auth_helpers.py index da2fd4368f..18ef7d109e 100644 --- a/tests/test_e2e_jobs_auth.py +++ b/tests/auth/test_jobs_auth_helpers.py @@ -3,7 +3,7 @@ import pytest -from e2e.test_jobs_auth import _job_exists_in_pages, _managed_admin_workspace +from tests.auth.integration.jobs_auth_helpers import job_exists_in_pages, managed_admin_workspace class _StubWorkspaces: @@ -41,7 +41,7 @@ def iter_pages(self): def test_managed_admin_workspace_deletes_workspace_after_success() -> None: sdk = _StubSDK() - with _managed_admin_workspace(sdk, "workspace-a") as workspace_name: + with managed_admin_workspace(sdk, "workspace-a") as workspace_name: assert workspace_name == "workspace-a" assert sdk.workspaces.created == ["workspace-a"] @@ -52,7 +52,7 @@ def test_managed_admin_workspace_deletes_workspace_after_failure() -> None: sdk = _StubSDK() with pytest.raises(RuntimeError, match="boom"): - with _managed_admin_workspace(sdk, "workspace-b"): + with managed_admin_workspace(sdk, "workspace-b"): raise RuntimeError("boom") assert sdk.workspaces.created == ["workspace-b"] @@ -64,4 +64,4 @@ def test_job_exists_in_pages_checks_later_pages() -> None: page_one = _StubPage([], ["other-job"]) page_one._pages = [page_one, page_two] - assert _job_exists_in_pages(page_one, "target-job") is True + assert job_exists_in_pages(page_one, "target-job") is True diff --git a/tests/test_e2e_services_pool.py b/tests/test_e2e_services_pool.py index f3af52ace1..281337b967 100644 --- a/tests/test_e2e_services_pool.py +++ b/tests/test_e2e_services_pool.py @@ -6,6 +6,7 @@ import pytest import e2e.services_pool as services_pool +import e2e.services_pool_fixtures as services_pool_fixtures def test_render_e2e_config_for_docker_preserves_container_paths(tmp_path) -> None: @@ -488,3 +489,19 @@ def test_describe_active_module_binding_ignores_stale_active_key() -> None: ) assert pool.describe_active_module_binding(module_id) is None + + +def test_read_services_log_tail_streams_last_failure_lines(tmp_path, monkeypatch) -> None: + log_path = tmp_path / "services.log" + lines = [f"line {index}\n" for index in range(services_pool_fixtures._TAIL_LINES_ON_FAILURE + 3)] + log_path.write_text("".join(lines), encoding="utf-8") + + def fail_read_text(*args, **kwargs) -> str: + raise AssertionError("services log tail should not materialize the full log with read_text") + + monkeypatch.setattr(type(log_path), "read_text", fail_read_text) + + assert ( + services_pool_fixtures._read_services_log_tail(log_path) + == lines[-services_pool_fixtures._TAIL_LINES_ON_FAILURE :] + )