fix(e2e): fix local minikube scripts and switch CLI health checks to /status

e2e/k8s/scripts/install_helm_e2e.sh

@@ -18,6 +18,7 @@ HELM_VALUES="${HELM_VALUES:-${HELM_VALUES_FILE:-${REPO_ROOT}/e2e/k8s/values/defa
 HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:-}"
 NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-}"
 NMP_E2E_TAG="${NMP_E2E_TAG:-}"
+NMP_E2E_PULL_POLICY="${NMP_E2E_PULL_POLICY:-}"
 REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-false}"
 POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
 BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox}"
@@ -231,6 +232,13 @@ if [ -n "${NMP_E2E_TAG}" ]; then
     )
 fi
 
+if [ -n "${NMP_E2E_PULL_POLICY}" ]; then
+    HELM_ARGS+=(
+        --set api.image.pullPolicy="${NMP_E2E_PULL_POLICY}"
+        --set core.image.pullPolicy="${NMP_E2E_PULL_POLICY}"
+    )
+fi
+
 log_info "Helm install inputs:"
 printf '  release: %s\n' "${HELM_RELEASE_NAME}"
 printf '  namespace: %s\n' "${NAMESPACE}"

e2e/k8s/scripts/install_nmp_auth_e2e.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 # Install the auth-enabled local E2E harness on minikube.
+#
+# Layers minikube-auth.yaml on top of minikube.yaml so auth-specific
+# config stays minimal and doesn't duplicate base minikube values.
 
 set -euo pipefail
 
@@ -8,7 +11,8 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)"
 
 export NAMESPACE="${NAMESPACE:-${KUBE_NAMESPACE:-default}}"
 export HELM_RELEASE_NAME="${HELM_RELEASE_NAME:-nemo-platform}"
-export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml}"
+export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube.yaml}"
+export HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:-} -f ${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml"
 export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}"
 export NMP_E2E_TAG="${NMP_E2E_TAG:-local}"
 export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"

e2e/k8s/scripts/local_build_and_upgrade.sh

@@ -1,61 +1,65 @@
 #!/usr/bin/env bash
+# Build Docker images locally and deploy to minikube via Helm.
+#
+# This script handles the build step, then delegates the Helm install to
+# install_helm_e2e.sh so install logic lives in one place.
+#
+# Environment variables:
+#   MINIKUBE_PROFILE  - minikube profile name (default: minikube)
+#   NMP_REGISTRY      - image registry (default: docker.io/my-registry)
+#   IMAGE_TAG         - image tag (default: local-<epoch>)
+#   BUILD_ARCH        - target platform (default: auto-detected from host)
+#   HELM_VALUES       - values file (default: e2e/k8s/values/minikube.yaml)
+
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
 
+MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}"
+
 # Check if minikube is running
-if ! minikube status &>/dev/null; then
-  echo "Minikube is not running. Starting minikube..."
-  # Use the setup_local_minikube_gpu.sh script to start minikube,
-  # and ensure the script is in the same directory as this script.
-
-  "$SCRIPT_DIR/setup_local_minikube_gpu.sh"
+if ! minikube status -p "${MINIKUBE_PROFILE}" &>/dev/null; then
+  echo "Minikube profile ${MINIKUBE_PROFILE} is not running. Starting..."
+  MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh"
 fi
 
 # Wait for minikube to be ready
-minikube status
+minikube status -p "${MINIKUBE_PROFILE}"
 
-# Build the images with a local tag and then load them
-# Use epoch seconds (date +%s) so each run gets a unique tag and upgrades pick up new images
+# Use epoch seconds so each run gets a unique tag and upgrades pick up new images
 IMAGE_TAG="${IMAGE_TAG:-local-$(date +%s)}"
 
 # Detect platform for build (match host arch)
 BUILD_ARCH="${BUILD_ARCH:-linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')}"
 GIT_SHA=$(git -C "${REPO_ROOT}" rev-parse HEAD)
 
+NMP_REGISTRY="${NMP_REGISTRY:-docker.io/my-registry}"
+
 echo "Building docker-cpu images with tag $IMAGE_TAG (platform=$BUILD_ARCH)..."
 
-# Allow building directly into minikube's docker daemon
-eval "$(minikube docker-env)"
+# Build directly into minikube's docker daemon
+eval "$(minikube -p "${MINIKUBE_PROFILE}" docker-env)"
 
-# Set the image tag to the git sha
 (
   cd "${REPO_ROOT}"
   CI_COMMIT_SHA="$GIT_SHA" \
     BAKE_TAG="$IMAGE_TAG" \
-    IMAGE_REGISTRY="docker.io/my-registry" \
+    IMAGE_REGISTRY="${NMP_REGISTRY}" \
     BUILD_ARCH="$BUILD_ARCH" \
     docker buildx bake docker-cpu --set "*.platform=$BUILD_ARCH"
 )
 
-
-# Echo the image tags and an example script to run end-to-end tests
-echo "Image tags:"
-echo "  nmp-api: $IMAGE_TAG"
-echo "  nmp-cpu-tasks: $IMAGE_TAG"
-echo "  platform: $IMAGE_TAG"
-echo "----------------------------------------"
-echo "Example script to run end-to-end jobs tests:"
-echo "  NMP_E2E_INTERNAL_HOST=nemo-platform-api:8080 NMP_E2E_REGISTRY=docker.io/my-registry NMP_E2E_TAG=$IMAGE_TAG uv run pytest e2e --kubernetes --cluster-url=http://localhost:80"
 echo "----------------------------------------"
-echo "To rerun the helm install/upgrade, run:"
-echo "  helm upgrade --install nemo-platform k8s/helm/ -f e2e/k8s/values/local.yaml --set \"api.image.tag=$IMAGE_TAG\" --set \"core.image.tag=$IMAGE_TAG\" --set \"platformConfig.platform.image_tag=$IMAGE_TAG\""
+echo "Images built with tag: $IMAGE_TAG"
 echo "----------------------------------------"
 
-# Install/upgrade the helm chart with image tags
-helm upgrade --install nemo-platform k8s/helm/ \
-  -f e2e/k8s/values/local.yaml \
-  --set "api.image.tag=$IMAGE_TAG" \
-  --set "core.image.tag=$IMAGE_TAG" \
-  --set "platformConfig.platform.image_tag=$IMAGE_TAG"
+# Delegate helm install to install_helm_e2e.sh
+export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube.yaml}"
+export NMP_E2E_REGISTRY="${NMP_REGISTRY}"
+export NMP_E2E_TAG="${IMAGE_TAG}"
+export NMP_E2E_PULL_POLICY="Never"
+export MINIKUBE_PROFILE
+export REQUIRE_NMP_E2E_IMAGES=true
+
+exec "$SCRIPT_DIR/install_helm_e2e.sh"

e2e/k8s/values/minikube-auth-portforward.yaml

@@ -1,42 +1,18 @@
-# CPU-only minikube values for local auth E2E verification without ingress.
+# Port-forward overlay — use on top of minikube.yaml + minikube-auth.yaml.
 #
 # Usage:
-#   helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth-portforward.yaml
+#   helm upgrade -i nemo-platform k8s/helm \
+#     -f e2e/k8s/values/minikube.yaml \
+#     -f e2e/k8s/values/minikube-auth.yaml \
+#     -f e2e/k8s/values/minikube-auth-portforward.yaml \
+#     --set api.image.tag=<TAG> ...
 #
-# This keeps the auth-enabled local harness but disables ingress so the stack
-# can be validated through kubectl port-forward on machines where registry.k8s.io
-# is blocked and the ingress addon cannot bootstrap. The chart still keeps split
-# pods on the API service URL while the API pod itself loops back to localhost.
-
-k8s-nim-operator:
-  enabled: false
-
-postgresql:
-  persistence:
-    storageClass: standard
-
-core:
-  storage:
-    storageClass: standard
-    volumePermissionsImage: busybox
+# Disables ingress and envoy proxy for machines where registry.k8s.io is
+# blocked and the ingress addon cannot bootstrap. Use kubectl port-forward
+# instead.
 
 envoyProxy:
   enabled: false
 
 ingress:
   enabled: false
-
-platformConfig:
-  auth:
-    enabled: true
-    policy_decision_point_provider: embedded
-    policy_data_refresh_interval: 2
-    bundle_cache_seconds: 2
-    admin_email: "admin@example.com"
-  inference_gateway:
-    mock_provider_prefix: igw-mock-
-  models:
-    controller:
-      backends:
-        nim_operator:
-          enabled: false

e2e/k8s/values/minikube-auth.yaml

@@ -1,46 +1,24 @@
-# CPU-only minikube values for local auth E2E verification.
+# Auth overlay for minikube — use on top of minikube.yaml.
 #
 # Usage:
-#   helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth.yaml
+#   helm upgrade -i nemo-platform k8s/helm \
+#     -f e2e/k8s/values/minikube.yaml \
+#     -f e2e/k8s/values/minikube-auth.yaml \
+#     --set api.image.tag=<TAG> ...
 #
-# This values file is intentionally local-friendly:
-# - disables the NIM operator dependency
-# - enables platform auth without requiring external OIDC
-# - relies on the chart default that keeps split pods on the API service URL
-#   while the API pod itself loops back to localhost
-# - keeps ingress enabled for browser/curl access through minikube ingress
+# Disables the NIM operator and enables embedded auth with short
+# cache/refresh intervals for fast test feedback.
 
 k8s-nim-operator:
   enabled: false
 
-postgresql:
-  persistence:
-    storageClass: standard
-
-core:
-  storage:
-    storageClass: standard
-    volumePermissionsImage: busybox
-
-ingress:
-  enabled: true
-  className: nginx
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
-
 platformConfig:
   auth:
     enabled: true
     policy_decision_point_provider: embedded
     policy_data_refresh_interval: 2
     bundle_cache_seconds: 2
     admin_email: "admin@example.com"
-  inference_gateway:
-    mock_provider_prefix: igw-mock-
   models:
     controller:
       backends:

e2e/k8s/values/minikube.yaml

@@ -1,7 +1,23 @@
 # Minikube values for local development
 # This is a standalone values file - use it in place of ./default.yaml
 #
-# Usage: helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml
+# Usage (local build):
+#   helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \
+#     --set api.image.repository=docker.io/my-registry/nmp-api \
+#     --set api.image.tag=local --set api.image.pullPolicy=Never \
+#     --set core.image.repository=docker.io/my-registry/nmp-api \
+#     --set core.image.tag=local --set core.image.pullPolicy=Never \
+#     --set platformConfig.platform.image_registry=docker.io/my-registry \
+#     --set platformConfig.platform.image_tag=local
+#
+# Usage (GHCR):
+#   helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \
+#     --set api.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \
+#     --set api.image.tag=latest \
+#     --set core.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \
+#     --set core.image.tag=latest \
+#     --set platformConfig.platform.image_registry=ghcr.io/nvidia-nemo/platform \
+#     --set platformConfig.platform.image_tag=latest
 
 # Enable NIM operator for local GPU testing
 k8s-nim-operator:
@@ -30,6 +46,13 @@ ingress:
     nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
+  hosts:
+    - name: ""
+      paths:
+        - path: /
+          pathType: Prefix
+          service: '{{ include "nemo-platform.ingressBackendService" . }}'
+          port: '{{ include "nemo-platform.ingressBackendPort" . }}'
 
 platformConfig:
   inference_gateway: