diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks index 2abcb75f37..15443c3464 100644 --- a/docker/Dockerfile.safe-synthesizer-tasks +++ b/docker/Dockerfile.safe-synthesizer-tasks @@ -43,6 +43,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ apt-get update && apt-get install -y --no-install-recommends \ bash \ + build-essential \ ca-certificates \ libgomp1 \ tini && \ @@ -53,6 +54,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ rm -rf /var/lib/apt/lists/* ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ + NEMO_DEPLOYMENT_TYPE=nmp \ PATH="/opt/venv/bin:${PATH}" \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ @@ -80,6 +82,7 @@ LABEL org.opencontainers.image.title="Safe Synthesizer Tasks" \ # Install the Safe Synthesizer engine/CUDA runtime before copying Platform # sources, so plugin-only edits do not invalidate the large dependency layers. +COPY plugins/nemo-safe-synthesizer/constraints.txt /tmp/safe-synthesizer-constraints.txt RUN uv venv /opt/venv RUN printf '%s\n' \ flashinfer-cubin \ @@ -121,6 +124,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --index-strategy unsafe-best-match \ --torch-backend cu129 \ --overrides /tmp/safe-synthesizer-overrides.txt \ + --constraints /tmp/safe-synthesizer-constraints.txt \ --excludes /tmp/exclude-flashinfer-torch-vllm.txt \ --requirements /tmp/safe-synthesizer-runtime.txt @@ -132,6 +136,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --index-strategy unsafe-best-match \ --torch-backend cu129 \ --overrides /tmp/safe-synthesizer-overrides.txt \ + --constraints /tmp/safe-synthesizer-constraints.txt \ --excludes /tmp/exclude-torch-vllm.txt \ --requirements /tmp/safe-synthesizer-runtime.txt @@ -143,6 +148,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --index-strategy unsafe-best-match \ --torch-backend cu129 \ --overrides /tmp/safe-synthesizer-overrides.txt \ + --constraints /tmp/safe-synthesizer-constraints.txt \ --excludes /tmp/exclude-vllm.txt \ --requirements /tmp/safe-synthesizer-runtime.txt @@ -154,6 +160,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --index-strategy unsafe-best-match \ --torch-backend cu129 \ --overrides /tmp/safe-synthesizer-overrides.txt \ + --constraints /tmp/safe-synthesizer-constraints.txt \ --requirements /tmp/safe-synthesizer-runtime.txt \ --requirements /tmp/vllm-cu129.txt diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 448a608137..a5972ec56b 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -36,9 +36,9 @@ For a PyPI install, run the following command in a **terminal (shell)**: ```shell if command -v uv &> /dev/null; then - uv pip install "nemo-platform[all]" kagglehub matplotlib + uv pip install "nemo-platform[all]" 'kagglehub==0.3.13' matplotlib else - pip install "nemo-platform[all]" kagglehub matplotlib + pip install "nemo-platform[all]" 'kagglehub==0.3.13' matplotlib fi ``` diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index 0d49c872e9..e0e3244df6 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -890,8 +890,10 @@ safe_synthesizer: job_mode: subprocess-local # default: 'default' job_executor_profile: default - # default: 'nmp-gpu-tasks' - container_image: nmp-gpu-tasks + # default: 'safe-synthesizer-tasks' + container_image: safe-synthesizer-tasks + # Optional fully qualified task image reference. When set, this bypasses platform NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs. + container_image_ref: # default: '.nemo/safe-synthesizer-runtime' runtime_venv: .nemo/safe-synthesizer-runtime # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.2' diff --git a/plugins/nemo-safe-synthesizer/constraints.txt b/plugins/nemo-safe-synthesizer/constraints.txt index 06a8e39c00..6f487f70d6 100644 --- a/plugins/nemo-safe-synthesizer/constraints.txt +++ b/plugins/nemo-safe-synthesizer/constraints.txt @@ -30,3 +30,4 @@ tornado>=6.5.5 # Runtime compatibility constraints. boto3>=1.40.46,<1.40.62 botocore>=1.40.46,<1.40.62 +gliner<0.2.27 diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index e3e91431f0..61e057bac8 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -57,6 +57,10 @@ def _runtime_job_config(job_config: SafeSynthesizerJobConfig) -> dict[str, Any]: return config +def _container_image() -> str: + return config.container_image_ref or get_qualified_image(config.container_image) + + def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[EnvironmentVariable]) -> PlatformJobStep: if config.job_mode == "subprocess-local": try: @@ -94,7 +98,7 @@ def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[Env provider="gpu", profile=config.job_executor_profile, container=ContainerSpec( - image=get_qualified_image(config.container_image), + image=_container_image(), entrypoint=config.entrypoint, ), resources=resources, diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py index 4dce9b14ac..1c69204ea4 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py @@ -22,7 +22,14 @@ class SafeSynthesizerConfig(NemoConfig): ) job_mode: Literal["subprocess-local", "container"] = "subprocess-local" job_executor_profile: str = "default" - container_image: str = "nmp-gpu-tasks" + container_image: str = "safe-synthesizer-tasks" + container_image_ref: str | None = Field( + default=None, + description=( + "Optional fully qualified task image reference. When set, this bypasses platform " + "NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs." + ), + ) runtime_venv: str = ".nemo/safe-synthesizer-runtime" runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.2" runtime_python_version: str = "3.11" diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md index 1489e4ee4f..a8544adf8e 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md @@ -12,13 +12,14 @@ Task router for agents helping a person use the NeMo Safe Synthesizer NMP plugin - The NeMo Safe Synthesizer plugin is installed in the active NeMo Platform environment. - The `nemo safe-synthesizer` CLI is available, or repo development can use `uv run nemo safe-synthesizer`. -- Host-local generation requires a Linux host with a CUDA-capable NVIDIA GPU, compatible drivers, and a runtime created by `nemo safe-synthesizer runtime setup`. - Platform jobs require workspace access to the input fileset and any `hf_token_secret` or PII classification provider. +- Container jobs require a GPU-capable Jobs backend and access to the configured Safe Synthesizer task image. +- Host-local generation requires a Linux host with a CUDA-capable NVIDIA GPU, compatible drivers, and a runtime created by `nemo safe-synthesizer runtime setup`. - Fileset references use `/#` unless a workflow states otherwise. ## Route -- Run the plugin locally or submit platform jobs: read `workflows/run.md`. +- Submit platform container jobs or run host-local development tasks: read `workflows/run.md`. - Set or override job parameters: read `workflows/config.md`. - Diagnose runtime, install, generation, OOM, validation, or fileset failures: read `workflows/diagnose.md`. - Retrieve job result artifacts: read `workflows/results.md`. @@ -26,10 +27,12 @@ Task router for agents helping a person use the NeMo Safe Synthesizer NMP plugin ## Plugin-Specific Rules -- Prefer the plugin CLI surface over upstream-only commands. -- Use `nemo safe-synthesizer run-local` for host-local CUDA/GPU development. -- Use `nemo safe-synthesizer runtime setup` to install engine/CUDA dependencies into the separate runtime venv. +- Prefer platform container jobs for normal Safe Synthesizer usage. - Use the Jobs API or SDK for platform jobs. The `nemo safe-synthesizer` CLI exposes `run-local` and `runtime` only. +- Configure released container jobs with `NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform`, `NMP_IMAGE_TAG=`, `NEMO_SAFE_SYNTHESIZER_JOB_MODE=container`, and `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks`. +- Override local task images with `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=`; this bypasses platform registry/tag qualification. +- Use `nemo safe-synthesizer run-local` only for host-local CUDA/GPU development or debugging. +- Use `nemo safe-synthesizer runtime setup` only for host-local runs that need the separate runtime venv. - Treat `data_source` as a fileset URL for platform jobs, usually `/#`. - For local runs, prefer `--data-source ` when the input is already on disk. - If the job uses PII classification, `config.replace_pii.globals.classify.classify_model_provider` must be `/`. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md index ca06ccd7fd..d668e018ef 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md @@ -132,27 +132,25 @@ ## Submit the Spec -=== "CLI (uv run)" - - ```bash - uv run nemo safe-synthesizer runtime setup - uv run nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output - ``` - -=== "CLI" - - ```bash - nemo safe-synthesizer runtime setup - nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output - ``` +Use the Jobs API or SDK for the primary platform-container path. Pass the JSON object above as the `spec` field in the create-job request. Confirm the service is configured for container execution before submission: + +```bash +export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container +export NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform +export NMP_IMAGE_TAG= # match your installed NeMo Platform release +export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks +``` + +Use host-local execution only for local CUDA/GPU development or task debugging: + +```bash +uv run nemo safe-synthesizer runtime setup +uv run nemo safe-synthesizer run-local \ + --workspace default \ + --spec-file nss-job.json \ + --data-source ./input.csv \ + --output-dir ./nss-output +``` ## Next Steps diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md index 81dc6153a6..caccbac6d1 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md @@ -4,14 +4,14 @@ - NeMo CLI access through `nemo` or repo development invocation `uv run nemo`. - Python dependencies synced into the active virtual environment. -- A compatible CUDA-capable GPU and driver for host-local generation. +- A GPU-capable Jobs backend for platform container jobs, or a compatible CUDA-capable GPU and driver for host-local generation. - Files API URL access when the run uses filesets or model fileset setup. - Workspace access to the input fileset, output job, `hf_token_secret`, and any PII classification provider. ## First Checks 1. Resolve the CLI with `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bin/nemo) || echo CLI_NOT_FOUND`. -2. Confirm whether the user is running host-local (`nemo safe-synthesizer run-local`) or a platform job through the Jobs API or SDK. +2. Confirm whether the user is running a platform container job through the Jobs API or SDK, or host-local (`nemo safe-synthesizer run-local`). 3. Inspect the spec file before changing commands. ## Common Failures @@ -22,11 +22,20 @@ Tell the user that the NeMo CLI or the Safe Synthesizer plugin is not installed ### CUDA or GPU initialization fails +- For platform jobs, confirm the job executor profile targets GPU-capable workers. - Confirm the host has a compatible NVIDIA GPU and driver with `nvidia-smi`. - For repo development, verify the plugin runtime with `uv run nemo safe-synthesizer runtime info`. - Recreate the runtime with `uv run nemo safe-synthesizer runtime setup --force` if the engine/CUDA packages are missing or stale. - Host-local Safe Synthesizer training runs directly on the host GPU; a GPU inside another service container is not enough. +### Container image cannot be pulled or is the wrong tag + +- Confirm platform jobs are using container mode: `NEMO_SAFE_SYNTHESIZER_JOB_MODE=container`. +- For released images, verify `NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform`, `NMP_IMAGE_TAG=`, and `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks`. +- For local Docker executor testing, verify `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=safe-synthesizer-tasks:local` and that `docker image inspect safe-synthesizer-tasks:local` succeeds. +- For Kubernetes, push the image to a registry the cluster can pull and set `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF` to that full pushed image reference. +- If the pull fails from `nvcr.io`, confirm NGC credentials or image pull secrets are configured for the Jobs backend. + ### Data source cannot be loaded - For platform jobs, verify `data_source` is a fileset URL: `/#`. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md index 215214d5ea..9b81e6b01f 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md @@ -1,28 +1,42 @@ # Running Safe Synthesizer -## Local Run Prerequisites +## Platform Job Prerequisites -- A Linux host with a CUDA-capable NVIDIA GPU. -- The Safe Synthesizer plugin installed as an editable local plugin, for repo development: `BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python`. -- The separate Safe Synthesizer runtime venv created with `uv run nemo safe-synthesizer runtime setup`. -- A job spec JSON file with `data_source` and `config`. -- A local input file or directory when using `--data-source`. +- The Safe Synthesizer service, Jobs service, and GPU execution backend are available. +- `data_source` points to a platform fileset path such as `default/my-fileset#input.csv`. +- If `hf_token_secret` is set, the named platform secret exists in the target workspace. +- If PII classification is enabled, the model provider exists and is referenced as `/`. +- The Jobs backend can pull the configured Safe Synthesizer task image. -If you submit **platform jobs** (not bare `run-local` with `--data-source`), start services first (`nemo setup --start-services` or `nemo services run`), then optionally register model filesets: +For platform jobs, start services first (`nemo setup --start-services` or `nemo services run`), then optionally register model filesets: ```bash curl -s http://localhost:8080/health/ready uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 ``` -See `docs/safe-synthesizer/about/host-local-development.md` for when the platform is required. +## Container Image Configuration -## Platform Job Prerequisites +Prefer the released task image from NGC: -- The Safe Synthesizer service and Jobs service are available. -- `data_source` points to a platform fileset path such as `default/my-fileset#input.csv`. -- If `hf_token_secret` is set, the named platform secret exists in the target workspace. -- If PII classification is enabled, the model provider exists and is referenced as `/`. +```bash +export NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform +export NMP_IMAGE_TAG= # match your installed NeMo Platform release +export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container +export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks +``` + +This resolves platform job steps to `nvcr.io/nvidia/nemo-platform/safe-synthesizer-tasks:`. + +For a local Docker-built image on a Docker executor, set a full image reference override: + +```bash +docker buildx bake safe-synthesizer-tasks-docker +export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container +export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=safe-synthesizer-tasks:local +``` + +For Kubernetes, push the local build to a registry the cluster can pull, then set `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF` to that pushed image reference. ## Resolve the CLI @@ -33,7 +47,11 @@ Run `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bi ## Choose the Execution Mode -Use host-local execution when the user is iterating on a local machine with CUDA/GPU access: +Use platform jobs for normal Safe Synthesizer usage. The platform compiles the spec into a GPU container step that runs the configured Safe Synthesizer task image. + +Use the Jobs API or SDK to create the job. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands. For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.md`. + +Use host-local execution only when the user is iterating on a local machine with CUDA/GPU access or debugging the task process outside the Jobs backend: ```bash uv run nemo safe-synthesizer runtime setup @@ -44,10 +62,6 @@ uv run nemo safe-synthesizer run-local \ --output-dir ./nss-output ``` -Use the Jobs API or SDK when the user wants the NMP Jobs service to run Safe Synthesizer. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands. - -For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.md`. - ## Minimal Spec Shape ```json diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py index 22311e546a..e1165525a3 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py @@ -119,6 +119,40 @@ async def test_job_config_compiler_with_classify_provider(mock_sdk): assert env["CLASSIFY_LLM_ENDPOINT_PATH"] == "/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1" +@pytest.mark.asyncio +async def test_job_config_compiler_container_mode_uses_safe_synthesizer_tasks_image(mock_sdk, monkeypatch): + monkeypatch.setattr(endpoints.config, "job_mode", "container") + monkeypatch.setattr(endpoints.config, "container_image", "safe-synthesizer-tasks") + monkeypatch.setattr(endpoints.config, "container_image_ref", None) + monkeypatch.setattr(endpoints, "get_qualified_image", lambda name: f"registry.example.com/nemo/{name}:test-tag") + + result = await _compile(_make_spec(), mock_sdk) + + step = next(iter(result["steps"])) + assert step["executor"]["provider"] == "gpu" + assert step["executor"]["container"]["image"] == "registry.example.com/nemo/safe-synthesizer-tasks:test-tag" + assert step["executor"]["container"]["entrypoint"] == [ + "python", + "-m", + "nemo_safe_synthesizer_plugin.tasks.safe_synthesizer", + ] + + +@pytest.mark.asyncio +async def test_job_config_compiler_container_mode_uses_image_ref_override(mock_sdk, monkeypatch): + monkeypatch.setattr(endpoints.config, "job_mode", "container") + monkeypatch.setattr(endpoints.config, "container_image_ref", "safe-synthesizer-tasks:local") + get_qualified_image = MagicMock(side_effect=AssertionError("image ref overrides should not be qualified")) + monkeypatch.setattr(endpoints, "get_qualified_image", get_qualified_image) + + result = await _compile(_make_spec(), mock_sdk) + + step = next(iter(result["steps"])) + assert step["executor"]["provider"] == "gpu" + assert step["executor"]["container"]["image"] == "safe-synthesizer-tasks:local" + get_qualified_image.assert_not_called() + + @pytest.mark.asyncio async def test_job_config_compiler_validates_pretrained_model_job(mock_sdk): mock_sdk.jobs.results.retrieve = AsyncMock(