fix(customizer): grpo sft full serialization fix

docker/rl/Dockerfile.nmp-rl-base

@@ -316,20 +316,21 @@ EOF
 # Excluding here keeps them out of this layer - removing them later would leave them extractable.
 COPY --from=nemo-rl --chown=${BUILD_UID}:${BUILD_GID} --exclude=**/.git . /opt/nemo-rl
 
-# TEMPORARY: NemotronH on DTensor v2 dies in Automodel's parallelizer, which reaches for
-# model.backbone.layers while both the stock-HF and Automodel NemotronH name the trunk
-# `model` (only the HF checkpoint KEYS use a backbone. prefix). Fixed on Automodel main
-# (parallelizer.py: `inner = model.backbone if hasattr(model, "backbone") else model.model`)
-# but NVIDIA-NeMo/RL main still pins 24b47e8, which predates it. Drop this once the
-# submodule is bumped past that fix.
+# TEMPORARY source patches on the pinned NeMo-RL / Automodel trees. Bind-mounted rather than
+# COPY-ed: this stage already runs as ${BUILD_UID}, so a COPY would land root-owned and could
+# not be cleaned up afterwards, and anything left behind ships in the layer.
 #
-# Bind-mounted rather than COPY-ed: this stage already runs as ${BUILD_UID}, so a COPY would
-# land root-owned and could not be cleaned up afterwards, and anything left behind ships in
-# the layer. The mount keeps the patch out of the image entirely.
-RUN --mount=type=bind,source=docker/rl/patches/automodel-nemotronh-trunk.patch,target=/patches/nemotronh.patch \
+# - NemotronH: Automodel's parallelizer reaches for model.backbone.layers while both the
+#   stock-HF and Automodel NemotronH name the trunk `model`. Fixed on Automodel main.
+# - Torch version order: pinned Automodel compares base-version strings, so Torch 2.11 is
+#   treated as older than 2.9 and async checkpointing is disabled.
+# - dtype: NeMo-RL still passes Transformers' deprecated torch_dtype alias.
+# Drop each file once the corresponding submodule pin contains the fix.
+RUN --mount=type=bind,source=docker/rl/patches,target=/patches \
     cd /opt/nemo-rl && \
-    git apply --check -p1 /patches/nemotronh.patch && \
-    git apply -p1 --verbose /patches/nemotronh.patch
+    git apply -p1 --verbose /patches/automodel-nemotronh-trunk.patch && \
+    git apply -p1 --verbose /patches/automodel-torch-version-order.patch && \
+    git apply -p1 --verbose /patches/nemo-rl-transformers-dtype.patch
 
 # Install the workspace ROOT (nemo-rl) editable. --no-install-project above installed every
 # DEPENDENCY and workspace MEMBER (incl. nemo-gym) but skipped the root; this adds it with --no-deps
@@ -466,7 +467,13 @@ for py in /opt/nemo_rl_venv/bin/python /opt/ray_venvs/*/bin/python /opt/gym_venv
     if "${py}" -c 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec("wandb") else 1)'; then
         uv pip install --python "${py}" --upgrade-package wandb "wandb>=0.29.0"
     fi
-    uv pip uninstall --python "${py}" mooncake-transfer-engine-cuda13 || true
+    # Full mlflow needs cryptography<50 and an exact-version skinny; this image uses skinny
+    # plus cryptography>=50. pynvml is a deprecated shim; nvidia-ml-py still provides the
+    # import Torch uses. Mooncake is an optional KV-transfer connector with a CVE in its
+    # vendored etcd wrapper.
+    for pkg in mlflow pynvml mooncake-transfer-engine-cuda13; do
+        uv pip uninstall --python "${py}" "${pkg}" || true
+    done
 done
 
 # Mooncake is an optional vLLM KV-transfer connector. The latest CUDA 13 wheel still embeds a
@@ -642,11 +649,16 @@ for python_path in /opt/ray_venvs/*/bin/python /opt/gym_venvs/*/bin/python; do
 done
 EOF
 
-# Gym falls back to this in-tree cache when a process does not inherit NRL_CONTAINER / UV_CACHE_DIR
-# (the sandboxed Gym host's per-app servers), and fails with EACCES on the build-owned tree. Keeping
-# the fallback writable is the safety net; the real fix is env propagation, which lives in NeMo-RL.
-RUN mkdir -p /opt/nemo-rl/3rdparty/Gym-workspace/Gym/cache && \
-    chown ${RUNTIME_UID}:${RUNTIME_GID} /opt/nemo-rl/3rdparty/Gym-workspace/Gym/cache
+# Gym creates setuptools metadata while building server venvs at runtime. The root package writes
+# cache/nemo_gym.egg-info, while editable server packages write sibling *.egg-info directories under
+# their server-type parent. Keep only those build-output locations writable by the runtime user.
+RUN GYM_ROOT=/opt/nemo-rl/3rdparty/Gym-workspace/Gym && \
+    mkdir -p "${GYM_ROOT}/cache" && \
+    chown -R ${RUNTIME_UID}:${RUNTIME_GID} \
+        "${GYM_ROOT}/cache" \
+        "${GYM_ROOT}/resources_servers" \
+        "${GYM_ROOT}/responses_api_agents" \
+        "${GYM_ROOT}/responses_api_models"
 
 WORKDIR /opt/nemo-rl
 

docker/rl/patches/automodel-torch-version-order.patch

@@ -0,0 +1,13 @@
+diff --git a/3rdparty/Automodel-workspace/Automodel/nemo_automodel/components/checkpoint/checkpointing.py b/3rdparty/Automodel-workspace/Automodel/nemo_automodel/components/checkpoint/checkpointing.py
+index c529cf5..c48d7b4 100644
+--- a/3rdparty/Automodel-workspace/Automodel/nemo_automodel/components/checkpoint/checkpointing.py
++++ b/3rdparty/Automodel-workspace/Automodel/nemo_automodel/components/checkpoint/checkpointing.py
+@@ -67,7 +67,7 @@ def _is_geq_torch_2_9() -> bool:
+     """
+     Check if the current torch version is greater than or equal to 2.9.0.
+     """
+-    return parse(torch.__version__).base_version >= "2.9.0"
++    return parse(torch.__version__) >= parse("2.9.0")
+
+
+ def _is_safetensors_checkpoint(path: str) -> bool:

docker/rl/patches/nemo-rl-transformers-dtype.patch

@@ -0,0 +1,13 @@
+diff --git a/nemo_rl/models/automodel/setup.py b/nemo_rl/models/automodel/setup.py
+index c1799ca..75db1bc 100644
+--- a/nemo_rl/models/automodel/setup.py
++++ b/nemo_rl/models/automodel/setup.py
+@@ -327,7 +327,7 @@ def setup_policy_model(
+     # Load model config
+     model_config = AutoConfig.from_pretrained(
+         model_name,
+-        torch_dtype=torch.float32,  # Always load in float32 for master weights
++        dtype=torch.float32,  # Always load in float32 for master weights
+         trust_remote_code=True,
+         attn_implementation="flash_attention_2" if enable_seq_packing else None,
+         **hf_config_overrides,

docs/cli/troubleshooting.mdx

@@ -35,14 +35,14 @@ source /path/to/venv/bin/activate
 
 **Cause:**
 
-NeMo Platform supports Python 3.12 and 3.13, but uv [picks the interpreter itself](https://docs.astral.sh/uv/concepts/tools/#python-versions), so an install can land on Python 3.14.
+NeMo Platform supports Python 3.12 and 3.13, but uv [picks the interpreter itself](https://docs.astral.sh/uv/concepts/tools/#python-versions), so an install can land on Python 3.14. `nemo setup` exits immediately on an unsupported interpreter instead of starting services.
 
 **Solution:**
 
 Name a supported interpreter. uv downloads one if you do not have it:
 
 ```bash
-uv tool install --python 3.13 "nemo-platform[all]"
+uv tool install --python 3.13 --reinstall "nemo-platform[all]"
 ```
 
 ## Connection Issues

e2e/k8s/scripts/install_opensandbox_minikube.sh

@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Install the shared-kernel OpenSandbox control plane on a running minikube cluster
+# and copy the API-key Secret into the platform job namespace.
+#
+# The platform Helm chart does not install OpenSandbox. Run this after
+# setup_local_minikube_gpu.sh (or setup_local_minikube_cpu.sh), then helm-upgrade
+# the platform with sandboxClusterCapable=true.
+#
+# Charts come from published GitHub Release tarballs by default (no checkout).
+# OpenSandbox does not publish a Helm repo index; tags look like
+# helm/opensandbox-controller/0.2.0. There is no helm/opensandbox-server
+# release, so the server chart is taken from the published opensandbox umbrella
+# tarball. Overlay k8s/helm/examples/opensandbox/*.yaml so images stay on
+# controller v0.2.0 / server v0.2.1 and [secure_runtime] stays unset.
+#
+# Usage:
+#   ./e2e/k8s/scripts/install_opensandbox_minikube.sh
+#
+# Environment:
+#   OPENSANDBOX_DIR              Optional checkout with kubernetes/charts/{opensandbox-controller,opensandbox-server}
+#   OPENSANDBOX_CONTROLLER_CHART Optional chart path or .tgz URL (overrides download / checkout controller)
+#   OPENSANDBOX_SERVER_CHART     Optional chart path or .tgz URL (overrides download / checkout server)
+#   OPENSANDBOX_CONTROLLER_VERSION  GitHub Release chart version (default: 0.2.0)
+#   OPENSANDBOX_UMBRELLA_VERSION    Umbrella tarball that contains the server chart (default: 0.2.2)
+#   KUBE_NAMESPACE               Job / Helm-release namespace (default: default). Alias: NMP_NAMESPACE
+#   MINIKUBE_PROFILE             (default: minikube)
+#   SKIP_VERIFY=1                Skip k8s/helm/examples/opensandbox/verify/shared-kernel.sh
+#   HELM_TIMEOUT                 Helm --wait timeout (default: 10m)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib.sh
+source "${SCRIPT_DIR}/lib.sh"
+
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+EXAMPLES="${REPO_ROOT}/k8s/helm/examples/opensandbox"
+SYSTEM_NS="${SYSTEM_NS:-opensandbox-system}"
+KUBE_NAMESPACE="${KUBE_NAMESPACE:-${NMP_NAMESPACE:-default}}"
+NMP_NAMESPACE="${KUBE_NAMESPACE}"
+MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}"
+HELM_TIMEOUT="${HELM_TIMEOUT:-10m}"
+API_SECRET="opensandbox-server-api-key"
+OPENSANDBOX_CONTROLLER_VERSION="${OPENSANDBOX_CONTROLLER_VERSION:-0.2.0}"
+OPENSANDBOX_UMBRELLA_VERSION="${OPENSANDBOX_UMBRELLA_VERSION:-0.2.2}"
+OPENSANDBOX_RELEASES="${OPENSANDBOX_RELEASES:-https://github.com/opensandbox-group/OpenSandbox/releases/download}"
+
+CHART_CACHE=""
+SERVER_VALUES=""
+cleanup() {
+    rm -f "${SERVER_VALUES:-}"
+    if [ -n "${CHART_CACHE:-}" ]; then
+        rm -rf "${CHART_CACHE}"
+    fi
+}
+trap cleanup EXIT
+
+for tool in kubectl helm openssl python3 curl tar; do
+    if ! command -v "${tool}" >/dev/null 2>&1; then
+        log_error "${tool} is not installed. Please install it first."
+        exit 1
+    fi
+done
+
+if ! kubectl get node >/dev/null 2>&1; then
+    log_error "kubectl cannot reach a cluster. Start minikube first:"
+    log_error "  ./e2e/k8s/scripts/setup_local_minikube_gpu.sh"
+    exit 1
+fi
+
+download_tarball() {
+    local url="$1"
+    local dest="$2"
+    log_info "Downloading ${url}"
+    curl -fsSL -L -o "${dest}" "${url}"
+}
+
+resolve_charts() {
+    if [ -n "${OPENSANDBOX_CONTROLLER_CHART:-}" ] && [ -n "${OPENSANDBOX_SERVER_CHART:-}" ]; then
+        CONTROLLER_CHART="${OPENSANDBOX_CONTROLLER_CHART}"
+        SERVER_CHART="${OPENSANDBOX_SERVER_CHART}"
+        log_info "Using explicit chart paths/URLs"
+        return
+    fi
+    if [ -n "${OPENSANDBOX_CONTROLLER_CHART:-}" ] || [ -n "${OPENSANDBOX_SERVER_CHART:-}" ]; then
+        log_error "Set both OPENSANDBOX_CONTROLLER_CHART and OPENSANDBOX_SERVER_CHART, or neither."
+        exit 1
+    fi
+
+    if [ -n "${OPENSANDBOX_DIR:-}" ]; then
+        CONTROLLER_CHART="${OPENSANDBOX_DIR}/kubernetes/charts/opensandbox-controller"
+        SERVER_CHART="${OPENSANDBOX_DIR}/kubernetes/charts/opensandbox-server"
+        if [ ! -d "${CONTROLLER_CHART}" ] || [ ! -d "${SERVER_CHART}" ]; then
+            log_error "OpenSandbox charts not found under ${OPENSANDBOX_DIR}/kubernetes/charts/"
+            exit 1
+        fi
+        log_info "Using charts from ${OPENSANDBOX_DIR}"
+        return
+    fi
+
+    CHART_CACHE="$(mktemp -d)"
+    local controller_url="${OPENSANDBOX_RELEASES}/helm/opensandbox-controller/${OPENSANDBOX_CONTROLLER_VERSION}/opensandbox-controller-${OPENSANDBOX_CONTROLLER_VERSION}.tgz"
+    local umbrella_url="${OPENSANDBOX_RELEASES}/helm/opensandbox/${OPENSANDBOX_UMBRELLA_VERSION}/opensandbox-${OPENSANDBOX_UMBRELLA_VERSION}.tgz"
+    local controller_tgz="${CHART_CACHE}/opensandbox-controller.tgz"
+    local umbrella_tgz="${CHART_CACHE}/opensandbox.tgz"
+
+    download_tarball "${controller_url}" "${controller_tgz}"
+    download_tarball "${umbrella_url}" "${umbrella_tgz}"
+    tar -xzf "${umbrella_tgz}" -C "${CHART_CACHE}"
+
+    CONTROLLER_CHART="${controller_tgz}"
+    SERVER_CHART="${CHART_CACHE}/opensandbox/charts/opensandbox-server"
+    if [ ! -f "${CONTROLLER_CHART}" ] || [ ! -d "${SERVER_CHART}" ]; then
+        log_error "Published tarballs did not contain the expected charts."
+        log_error "Controller: ${CONTROLLER_CHART}"
+        log_error "Server: ${SERVER_CHART}"
+        log_error "OpenSandbox does not publish a helm/opensandbox-server tarball; the server chart is nested in the opensandbox umbrella chart."
+        exit 1
+    fi
+    log_info "Using published tarballs (controller ${OPENSANDBOX_CONTROLLER_VERSION}, server from umbrella ${OPENSANDBOX_UMBRELLA_VERSION})"
+}
+
+resolve_charts
+
+log_info "Installing shared-kernel OpenSandbox (control plane: ${SYSTEM_NS}, jobs: ${KUBE_NAMESPACE})"
+
+kubectl create namespace "${SYSTEM_NS}" --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace "${KUBE_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
+
+log_info "Applying BatchSandbox template ConfigMap..."
+kubectl apply -f "${REPO_ROOT}/e2e/k8s/values/batchsandbox-minikube-template.yaml"
+
+if ! kubectl get secret nvcrimagepullsecret -n "${KUBE_NAMESPACE}" >/dev/null 2>&1; then
+    log_error "Secret nvcrimagepullsecret is missing in ${KUBE_NAMESPACE}."
+    log_error "The BatchSandbox template hard-codes that name. Run setup_local_minikube_gpu.sh first,"
+    log_error "or create the pull Secret, or edit imagePullSecrets in ${REPO_ROOT}/e2e/k8s/values/batchsandbox-minikube-template.yaml."
+    exit 1
+fi
+
+if kubectl get secret "${API_SECRET}" -n "${SYSTEM_NS}" >/dev/null 2>&1; then
+    log_info "Reusing existing ${API_SECRET} in ${SYSTEM_NS}"
+else
+    log_info "Creating ${API_SECRET} in ${SYSTEM_NS}"
+    kubectl create secret generic "${API_SECRET}" \
+      -n "${SYSTEM_NS}" \
+      --from-literal=api-key="$(openssl rand -hex 32)"
+fi
+
+log_info "Copying ${API_SECRET} into job namespace ${KUBE_NAMESPACE}..."
+kubectl get secret "${API_SECRET}" -n "${SYSTEM_NS}" -o json | python3 -c '
+import json, sys
+secret = json.load(sys.stdin)
+for key in ("uid", "resourceVersion", "creationTimestamp", "namespace", "managedFields"):
+    secret.get("metadata", {}).pop(key, None)
+json.dump(secret, sys.stdout)
+' | kubectl apply -n "${KUBE_NAMESPACE}" -f -
+
+SERVER_VALUES="$(mktemp)"
+sed "s/REPLACE_WITH_RELEASE_NAMESPACE/${KUBE_NAMESPACE}/g" \
+  "${EXAMPLES}/opensandbox-server.yaml" > "${SERVER_VALUES}"
+
+log_info "Helm upgrade opensandbox-controller..."
+helm upgrade --install opensandbox-controller "${CONTROLLER_CHART}" \
+  --namespace "${SYSTEM_NS}" \
+  --wait --timeout "${HELM_TIMEOUT}" \
+  -f "${EXAMPLES}/opensandbox-controller.yaml"
+
+log_info "Helm upgrade opensandbox-server..."
+helm upgrade --install opensandbox-server "${SERVER_CHART}" \
+  --namespace "${SYSTEM_NS}" \
+  --wait --timeout "${HELM_TIMEOUT}" \
+  -f "${SERVER_VALUES}"
+
+log_info "Restarting opensandbox-server to pick up the BatchSandbox template..."
+kubectl rollout restart deployment/opensandbox-server -n "${SYSTEM_NS}"
+kubectl rollout status deployment/opensandbox-server -n "${SYSTEM_NS}" --timeout "${HELM_TIMEOUT}"
+
+if [ "${SKIP_VERIFY:-}" != "1" ]; then
+    log_info "Verifying shared-kernel OpenSandbox..."
+    OPEN_SANDBOX_WORKLOAD_NS="${KUBE_NAMESPACE}" \
+      NMP_NAMESPACE="${KUBE_NAMESPACE}" \
+      "${EXAMPLES}/verify/shared-kernel.sh"
+fi
+
+log_info "=========================================="
+log_info "OpenSandbox shared-kernel install complete"
+log_info "=========================================="
+log_info "Control plane: ${SYSTEM_NS}"
+log_info "Job namespace: ${KUBE_NAMESPACE}"
+log_info "Service DNS:   opensandbox-server.${SYSTEM_NS}.svc.cluster.local"
+log_info ""
+log_info "Point the platform Helm release at the server (then helm upgrade):"
+log_info "  sandboxClusterCapable: true"
+log_info "  opensandbox:"
+log_info "    domain: opensandbox-server.${SYSTEM_NS}.svc.cluster.local"
+log_info "    protocol: http"
+log_info "    apiKeySecret: ${API_SECRET}"
+log_info "    apiKeySecretKey: api-key"
+log_info ""
+log_info "If you previously set sandboxed_gym_default=false to skip OpenSandbox, restore it to true."

e2e/k8s/values/batchsandbox-minikube-template.yaml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# BatchSandbox template for local minikube. Same ConfigMap name as
+# k8s/helm/examples/opensandbox/batchsandbox-template.yaml.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: opensandbox-batchsandbox-template
+  namespace: opensandbox-system
+  labels:
+    app.kubernetes.io/part-of: opensandbox
+    opensandbox.nvidia.com/isolation-class: shared-kernel
+data:
+  batchsandbox-template.yaml: |
+    metadata:
+      labels:
+        opensandbox.nvidia.com/isolation-class: shared-kernel
+        opensandbox.nvidia.com/runtime: shared-kernel
+    spec:
+      replicas: 1
+      template:
+        spec:
+          restartPolicy: Never
+          terminationGracePeriodSeconds: 30
+          # Sandbox pods are built from this template by the OpenSandbox server,
+          # not by the jobs controller, so neither jobs executor_defaults
+          # image_pull_secrets nor the chart's top-level imagePullSecrets apply.
+          imagePullSecrets:
+            - name: nvcrimagepullsecret
+          tolerations:
+            - key: nvidia.com/gpu
+              operator: Exists