NVIDIA-NeMo · mckornfield · Jul 9, 2026 · Jul 7, 2026
@@ -43,6 +43,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
     apt-get update && apt-get install -y --no-install-recommends \
         bash \
+        build-essential \
         ca-certificates \
         libgomp1 \
         tini && \
@@ -53,6 +54,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     rm -rf /var/lib/apt/lists/*
 
 ENV UV_PROJECT_ENVIRONMENT=/opt/venv \
+    NEMO_DEPLOYMENT_TYPE=nmp \
     PATH="/opt/venv/bin:${PATH}" \
     UV_LINK_MODE=copy \
     UV_COMPILE_BYTECODE=1 \
@@ -80,6 +82,7 @@ LABEL org.opencontainers.image.title="Safe Synthesizer Tasks" \
 
 # Install the Safe Synthesizer engine/CUDA runtime before copying Platform
 # sources, so plugin-only edits do not invalidate the large dependency layers.
+COPY plugins/nemo-safe-synthesizer/constraints.txt /tmp/safe-synthesizer-constraints.txt
 RUN uv venv /opt/venv
 RUN printf '%s\n' \
         flashinfer-cubin \
@@ -121,6 +124,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --index-strategy unsafe-best-match \
         --torch-backend cu129 \
         --overrides /tmp/safe-synthesizer-overrides.txt \
+        --constraints /tmp/safe-synthesizer-constraints.txt \
         --excludes /tmp/exclude-flashinfer-torch-vllm.txt \
         --requirements /tmp/safe-synthesizer-runtime.txt
 
@@ -132,6 +136,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --index-strategy unsafe-best-match \
         --torch-backend cu129 \
         --overrides /tmp/safe-synthesizer-overrides.txt \
+        --constraints /tmp/safe-synthesizer-constraints.txt \
         --excludes /tmp/exclude-torch-vllm.txt \
         --requirements /tmp/safe-synthesizer-runtime.txt
 
@@ -143,6 +148,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --index-strategy unsafe-best-match \
         --torch-backend cu129 \
         --overrides /tmp/safe-synthesizer-overrides.txt \
+        --constraints /tmp/safe-synthesizer-constraints.txt \
         --excludes /tmp/exclude-vllm.txt \
         --requirements /tmp/safe-synthesizer-runtime.txt
 
@@ -154,6 +160,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --index-strategy unsafe-best-match \
         --torch-backend cu129 \
         --overrides /tmp/safe-synthesizer-overrides.txt \
+        --constraints /tmp/safe-synthesizer-constraints.txt \
         --requirements /tmp/safe-synthesizer-runtime.txt \
         --requirements /tmp/vllm-cu129.txt
 

@@ -36,9 +36,9 @@ For a PyPI install, run the following command in a **terminal (shell)**:
 
 ```shell
 if command -v uv &> /dev/null; then
- uv pip install "nemo-platform[all]" kagglehub matplotlib
+ uv pip install "nemo-platform[all]" 'kagglehub==0.3.13' matplotlib
 else
- pip install "nemo-platform[all]" kagglehub matplotlib
+ pip install "nemo-platform[all]" 'kagglehub==0.3.13' matplotlib
 fi
 ```
 

@@ -890,8 +890,10 @@ safe_synthesizer:
   job_mode: subprocess-local
   # default: 'default'
   job_executor_profile: default
-  # default: 'nmp-gpu-tasks'
-  container_image: nmp-gpu-tasks
+  # default: 'safe-synthesizer-tasks'
+  container_image: safe-synthesizer-tasks
+  # Optional fully qualified task image reference. When set, this bypasses platform NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs.
+  container_image_ref:
   # default: '.nemo/safe-synthesizer-runtime'
   runtime_venv: .nemo/safe-synthesizer-runtime
   # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.2'

@@ -30,3 +30,4 @@ tornado>=6.5.5
 # Runtime compatibility constraints.
 boto3>=1.40.46,<1.40.62
 botocore>=1.40.46,<1.40.62
+gliner<0.2.27
@@ -57,6 +57,10 @@ def _runtime_job_config(job_config: SafeSynthesizerJobConfig) -> dict[str, Any]:
     return config
 
 
+def _container_image() -> str:
+    return config.container_image_ref or get_qualified_image(config.container_image)
+
+
 def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[EnvironmentVariable]) -> PlatformJobStep:
     if config.job_mode == "subprocess-local":
         try:
@@ -94,7 +98,7 @@ def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[Env
             provider="gpu",
             profile=config.job_executor_profile,
             container=ContainerSpec(
-                image=get_qualified_image(config.container_image),
+                image=_container_image(),
                 entrypoint=config.entrypoint,
             ),
             resources=resources,

@@ -22,7 +22,14 @@ class SafeSynthesizerConfig(NemoConfig):
     )
     job_mode: Literal["subprocess-local", "container"] = "subprocess-local"
     job_executor_profile: str = "default"
-    container_image: str = "nmp-gpu-tasks"
+    container_image: str = "safe-synthesizer-tasks"
+    container_image_ref: str | None = Field(
+        default=None,
+        description=(
+            "Optional fully qualified task image reference. When set, this bypasses platform "
+            "NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs."
+        ),
+    )
     runtime_venv: str = ".nemo/safe-synthesizer-runtime"
     runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.2"
     runtime_python_version: str = "3.11"

@@ -12,24 +12,27 @@ Task router for agents helping a person use the NeMo Safe Synthesizer NMP plugin
 
 - The NeMo Safe Synthesizer plugin is installed in the active NeMo Platform environment.
 - The `nemo safe-synthesizer` CLI is available, or repo development can use `uv run nemo safe-synthesizer`.
-- Host-local generation requires a Linux host with a CUDA-capable NVIDIA GPU, compatible drivers, and a runtime created by `nemo safe-synthesizer runtime setup`.
 - Platform jobs require workspace access to the input fileset and any `hf_token_secret` or PII classification provider.
+- Container jobs require a GPU-capable Jobs backend and access to the configured Safe Synthesizer task image.
+- Host-local generation requires a Linux host with a CUDA-capable NVIDIA GPU, compatible drivers, and a runtime created by `nemo safe-synthesizer runtime setup`.
 - Fileset references use `<workspace>/<fileset>#<path>` unless a workflow states otherwise.
 
 ## Route
 
-- Run the plugin locally or submit platform jobs: read `workflows/run.md`.
+- Submit platform container jobs or run host-local development tasks: read `workflows/run.md`.
 - Set or override job parameters: read `workflows/config.md`.
 - Diagnose runtime, install, generation, OOM, validation, or fileset failures: read `workflows/diagnose.md`.
 - Retrieve job result artifacts: read `workflows/results.md`.
 - Interpret outputs, logs, synthetic data, reports, summaries, or adapters: read `workflows/artifacts.md`.
 
 ## Plugin-Specific Rules
 
-- Prefer the plugin CLI surface over upstream-only commands.
-- Use `nemo safe-synthesizer run-local` for host-local CUDA/GPU development.
-- Use `nemo safe-synthesizer runtime setup` to install engine/CUDA dependencies into the separate runtime venv.
+- Prefer platform container jobs for normal Safe Synthesizer usage.
 - Use the Jobs API or SDK for platform jobs. The `nemo safe-synthesizer` CLI exposes `run-local` and `runtime` only.
+- Configure released container jobs with `NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform`, `NMP_IMAGE_TAG=<tag>`, `NEMO_SAFE_SYNTHESIZER_JOB_MODE=container`, and `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks`.
+- Override local task images with `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=<image-ref>`; this bypasses platform registry/tag qualification.
+- Use `nemo safe-synthesizer run-local` only for host-local CUDA/GPU development or debugging.
+- Use `nemo safe-synthesizer runtime setup` only for host-local runs that need the separate runtime venv.
 - Treat `data_source` as a fileset URL for platform jobs, usually `<workspace>/<fileset>#<path>`.
 - For local runs, prefer `--data-source <local-file-or-dir>` when the input is already on disk.
 - If the job uses PII classification, `config.replace_pii.globals.classify.classify_model_provider` must be `<workspace>/<provider_name>`.

@@ -132,27 +132,25 @@
 
 ## Submit the Spec
 
-=== "CLI (uv run)"
-
-    ```bash
-    uv run nemo safe-synthesizer runtime setup
-    uv run nemo safe-synthesizer run-local \
-      --workspace default \
-      --spec-file nss-job.json \
-      --data-source ./input.csv \
-      --output-dir ./nss-output
-    ```
-
-=== "CLI"
-
-    ```bash
-    nemo safe-synthesizer runtime setup
-    nemo safe-synthesizer run-local \
-      --workspace default \
-      --spec-file nss-job.json \
-      --data-source ./input.csv \
-      --output-dir ./nss-output
-    ```
+Use the Jobs API or SDK for the primary platform-container path. Pass the JSON object above as the `spec` field in the create-job request. Confirm the service is configured for container execution before submission:
+
+```bash
+export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container
+export NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform
+export NMP_IMAGE_TAG=<tag>  # match your installed NeMo Platform release
+export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks
+```
+
+Use host-local execution only for local CUDA/GPU development or task debugging:
+
+```bash
+uv run nemo safe-synthesizer runtime setup
+uv run nemo safe-synthesizer run-local \
+  --workspace default \
+  --spec-file nss-job.json \
+  --data-source ./input.csv \
+  --output-dir ./nss-output
+```
 
 ## Next Steps
 

@@ -4,14 +4,14 @@
 
 - NeMo CLI access through `nemo` or repo development invocation `uv run nemo`.
 - Python dependencies synced into the active virtual environment.
-- A compatible CUDA-capable GPU and driver for host-local generation.
+- A GPU-capable Jobs backend for platform container jobs, or a compatible CUDA-capable GPU and driver for host-local generation.
 - Files API URL access when the run uses filesets or model fileset setup.
 - Workspace access to the input fileset, output job, `hf_token_secret`, and any PII classification provider.
 
 ## First Checks
 
 1. Resolve the CLI with `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bin/nemo) || echo CLI_NOT_FOUND`.
-2. Confirm whether the user is running host-local (`nemo safe-synthesizer run-local`) or a platform job through the Jobs API or SDK.
+2. Confirm whether the user is running a platform container job through the Jobs API or SDK, or host-local (`nemo safe-synthesizer run-local`).
 3. Inspect the spec file before changing commands.
 
 ## Common Failures
@@ -22,11 +22,20 @@ Tell the user that the NeMo CLI or the Safe Synthesizer plugin is not installed
 
 ### CUDA or GPU initialization fails
 
+- For platform jobs, confirm the job executor profile targets GPU-capable workers.
 - Confirm the host has a compatible NVIDIA GPU and driver with `nvidia-smi`.
 - For repo development, verify the plugin runtime with `uv run nemo safe-synthesizer runtime info`.
 - Recreate the runtime with `uv run nemo safe-synthesizer runtime setup --force` if the engine/CUDA packages are missing or stale.
 - Host-local Safe Synthesizer training runs directly on the host GPU; a GPU inside another service container is not enough.
 
+### Container image cannot be pulled or is the wrong tag
+
+- Confirm platform jobs are using container mode: `NEMO_SAFE_SYNTHESIZER_JOB_MODE=container`.
+- For released images, verify `NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform`, `NMP_IMAGE_TAG=<tag>`, and `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks`.
+- For local Docker executor testing, verify `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=safe-synthesizer-tasks:local` and that `docker image inspect safe-synthesizer-tasks:local` succeeds.
+- For Kubernetes, push the image to a registry the cluster can pull and set `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF` to that full pushed image reference.
+- If the pull fails from `nvcr.io`, confirm NGC credentials or image pull secrets are configured for the Jobs backend.
+
 ### Data source cannot be loaded
 
 - For platform jobs, verify `data_source` is a fileset URL: `<workspace>/<fileset>#<path>`.

@@ -1,28 +1,42 @@
 # Running Safe Synthesizer
 
-## Local Run Prerequisites
+## Platform Job Prerequisites
 
-- A Linux host with a CUDA-capable NVIDIA GPU.
-- The Safe Synthesizer plugin installed as an editable local plugin, for repo development: `BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python`.
-- The separate Safe Synthesizer runtime venv created with `uv run nemo safe-synthesizer runtime setup`.
-- A job spec JSON file with `data_source` and `config`.
-- A local input file or directory when using `--data-source`.
+- The Safe Synthesizer service, Jobs service, and GPU execution backend are available.
+- `data_source` points to a platform fileset path such as `default/my-fileset#input.csv`.
+- If `hf_token_secret` is set, the named platform secret exists in the target workspace.
+- If PII classification is enabled, the model provider exists and is referenced as `<workspace>/<provider_name>`.
+- The Jobs backend can pull the configured Safe Synthesizer task image.
 
-If you submit **platform jobs** (not bare `run-local` with `--data-source`), start services first (`nemo setup --start-services` or `nemo services run`), then optionally register model filesets:
+For platform jobs, start services first (`nemo setup --start-services` or `nemo services run`), then optionally register model filesets:
 
 ```bash
 curl -s http://localhost:8080/health/ready
 uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080
 ```
 
-See `docs/safe-synthesizer/about/host-local-development.md` for when the platform is required.
+## Container Image Configuration
 
-## Platform Job Prerequisites
+Prefer the released task image from NGC:
 
-- The Safe Synthesizer service and Jobs service are available.
-- `data_source` points to a platform fileset path such as `default/my-fileset#input.csv`.
-- If `hf_token_secret` is set, the named platform secret exists in the target workspace.
-- If PII classification is enabled, the model provider exists and is referenced as `<workspace>/<provider_name>`.
+```bash
+export NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform
+export NMP_IMAGE_TAG=<tag>  # match your installed NeMo Platform release
+export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container
+export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks
+```
+
+This resolves platform job steps to `nvcr.io/nvidia/nemo-platform/safe-synthesizer-tasks:<tag>`.
+
+For a local Docker-built image on a Docker executor, set a full image reference override:
+
+```bash
+docker buildx bake safe-synthesizer-tasks-docker
+export NEMO_SAFE_SYNTHESIZER_JOB_MODE=container
+export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=safe-synthesizer-tasks:local
+```
+
+For Kubernetes, push the local build to a registry the cluster can pull, then set `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF` to that pushed image reference.
 
 ## Resolve the CLI
 
@@ -33,7 +47,11 @@ Run `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bi
 
 ## Choose the Execution Mode
 
-Use host-local execution when the user is iterating on a local machine with CUDA/GPU access:
+Use platform jobs for normal Safe Synthesizer usage. The platform compiles the spec into a GPU container step that runs the configured Safe Synthesizer task image.
+
+Use the Jobs API or SDK to create the job. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands. For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.md`.
+
+Use host-local execution only when the user is iterating on a local machine with CUDA/GPU access or debugging the task process outside the Jobs backend:
 
 ```bash
 uv run nemo safe-synthesizer runtime setup
@@ -44,10 +62,6 @@ uv run nemo safe-synthesizer run-local \
   --output-dir ./nss-output
 ```
 
-Use the Jobs API or SDK when the user wants the NMP Jobs service to run Safe Synthesizer. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands.
-
-For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.md`.
-
 ## Minimal Spec Shape
 
 ```json

@@ -119,6 +119,40 @@ async def test_job_config_compiler_with_classify_provider(mock_sdk):
     assert env["CLASSIFY_LLM_ENDPOINT_PATH"] == "/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1"
 
 
+@pytest.mark.asyncio
+async def test_job_config_compiler_container_mode_uses_safe_synthesizer_tasks_image(mock_sdk, monkeypatch):
+    monkeypatch.setattr(endpoints.config, "job_mode", "container")
+    monkeypatch.setattr(endpoints.config, "container_image", "safe-synthesizer-tasks")
+    monkeypatch.setattr(endpoints.config, "container_image_ref", None)
+    monkeypatch.setattr(endpoints, "get_qualified_image", lambda name: f"registry.example.com/nemo/{name}:test-tag")
+
+    result = await _compile(_make_spec(), mock_sdk)
+
+    step = next(iter(result["steps"]))
+    assert step["executor"]["provider"] == "gpu"
+    assert step["executor"]["container"]["image"] == "registry.example.com/nemo/safe-synthesizer-tasks:test-tag"
+    assert step["executor"]["container"]["entrypoint"] == [
+        "python",
+        "-m",
+        "nemo_safe_synthesizer_plugin.tasks.safe_synthesizer",
+    ]
+
+
+@pytest.mark.asyncio
+async def test_job_config_compiler_container_mode_uses_image_ref_override(mock_sdk, monkeypatch):
+    monkeypatch.setattr(endpoints.config, "job_mode", "container")
+    monkeypatch.setattr(endpoints.config, "container_image_ref", "safe-synthesizer-tasks:local")
+    get_qualified_image = MagicMock(side_effect=AssertionError("image ref overrides should not be qualified"))
+    monkeypatch.setattr(endpoints, "get_qualified_image", get_qualified_image)
+
+    result = await _compile(_make_spec(), mock_sdk)
+
+    step = next(iter(result["steps"]))
+    assert step["executor"]["provider"] == "gpu"
+    assert step["executor"]["container"]["image"] == "safe-synthesizer-tasks:local"
+    get_qualified_image.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_job_config_compiler_validates_pretrained_model_job(mock_sdk):
     mock_sdk.jobs.results.retrieve = AsyncMock(