Support Authentik workload job e2e

Signed-off-by: Ryan S <267728323+ironcommit@users.noreply.github.com>

Author: ironcommit

Makefile

@@ -123,9 +123,8 @@ docs-check: ## Validate the Fern docs (fern check + validate-mdx + gated-link ch
 	cd docs/fern && npm run check
 
 .PHONY: test-auth-idp
-AUTHENTIK_E2E_LIFECYCLE ?= fresh
 test-auth-idp: ## Run the auth-idp test suite
-	AUTHENTIK_E2E_LIFECYCLE=$(AUTHENTIK_E2E_LIFECYCLE) uv run --frozen pytest tests/auth_idp -v --run-e2e
+	bash contrib/auth/authentik/run.sh test $(ARGS)
 
 .PHONY: docs-check-python-snippets
 docs-check-python-snippets: ## Syntax-check and type-check Python snippets in one doc (DOCS_PATH=...)

contrib/auth/authentik/README.md

@@ -23,7 +23,7 @@ The stack seeds these local-only identities:
 - Human password: `nemo-user-password-dev`
 - Human email: `nemo-user@example.com`
 - CLI OIDC client: `nemo-platform-cli`
-- Workload identity: `svc-nemo-ci`
+- Workload identity: `svc-nemo`
 - Workload group: `nemo-editors`
 
 ## Start The Stack
@@ -35,35 +35,45 @@ contrib/auth/authentik/run.sh stack
 ```
 
 This starts NeMo, Authentik, and the local gateway with the existing default
-NeMo API image, `my-registry/nmp-api:local`. The script does not build images.
+NeMo API image, `my-registry/nmp-api:local`. The `stack` action does not build
+images.
 Leave this process running. Stop it with `Ctrl-C` when you are done; the script
 removes the Compose stack and volumes on exit.
 
 To use a different prebuilt image for the example, pass it explicitly:
 
 ```bash
-contrib/auth/authentik/run.sh stack --image registry.example.com/nemo/nmp-api:<tag>
+export IMAGE_REGISTRY=registry.example.com/nemo
+export BAKE_TAG=<tag>
+
+contrib/auth/authentik/run.sh stack --image "$IMAGE_REGISTRY/nmp-api:$BAKE_TAG"
 ```
 
-The auth-idp test suite has one entrypoint:
+Use the same `IMAGE_REGISTRY` and `BAKE_TAG` values in the shell where you
+submit the workload job so the job container image matches the running NeMo API
+image.
+
+The auth-idp test suite uses the same helper script:
 
 ```bash
-make test-auth-idp
+contrib/auth/authentik/run.sh test
 ```
 
-To run the tests against the same prebuilt image used by the example, pass the
-matching registry and tag to that test target:
+For iteration or prebuilt images, pass options to the script directly. See
+`contrib/auth/authentik/run.sh --help` for the full option list.
 
 ```bash
-IMAGE_REGISTRY=registry.example.com/nemo BAKE_TAG=<tag> make test-auth-idp
+contrib/auth/authentik/run.sh test --lifecycle reuse
+contrib/auth/authentik/run.sh test --image registry.example.com/nemo/nmp-api:<tag>
 ```
 
-Wait until the gateway is ready:
+Wait until the platform is ready through the gateway:
 
 ```bash
-until curl -sf http://127.0.0.1:18080/apis/auth/discovery >/dev/null; do
+until curl -sf http://127.0.0.1:18080/health/ready >/dev/null; do
   sleep 2
 done
+echo "NeMo Platform Ready"
 ```
 
 The local gateway URL is:
@@ -119,91 +129,80 @@ nemo --context authentik-human workspaces members create \
   --workspace "$WORKSPACE" \
   --principal nemo-editors \
   --roles Viewer \
-  --roles JobLogWriter \
+  --roles JobRunner \
   --wait-role-propagation
 ```
 
 Expected result: the human user can manage the workspace, and the workload
-group can read the workspace and upload job logs.
+group can read the workspace from a job.
 
 ## Run A Workload Job
 
 ```bash
 export JOB_NAME=authentik-workload-demo
-export WORKLOAD_TOKEN_SECRET="${JOB_NAME}-token"
 ```
 
-Fetch a local demo token for the seeded workload identity and store it as a
-workspace secret:
+Fetch a local demo token for the seeded workload identity:
 
 ```bash
-export NEMO_WORKLOAD_TOKEN="$(
+export WORKLOAD_ACCESS_TOKEN="$(
   curl -fsS http://127.0.0.1:18080/application/o/token/ \
     -d grant_type=password \
     -d client_id=nemo-platform \
     -d client_secret=nemo-platform-secret-dev \
-    -d username=svc-nemo-ci \
-    -d password=svc-nemo-ci-token-secret-dev \
+    -d username=svc-nemo \
+    -d password=svc-nemo-token-secret-dev \
     -d scope="openid email groups" \
   | python -c 'import json, sys; print(json.load(sys.stdin)["access_token"])'
 )"
-
-printf '%s' "$NEMO_WORKLOAD_TOKEN" \
-  | nemo --context authentik-human secrets create "$WORKLOAD_TOKEN_SECRET" \
-      --workspace "$WORKSPACE" \
-      --from-file -
 ```
 
-Create the job request:
-
-```bash
-python - <<'PY' >/tmp/authentik-workload-job.json
-import json
-import os
-import sys
-
-payload = {
-    "source": "authentik-reference-example",
-    "spec": {"demo": "authentik-workload-auth"},
-    "platform_spec": {
-        "steps": [
-            {
-                "name": "workload-workspace-get",
-                "executor": {
-                    "provider": "cpu",
-                    "profile": "workload",
-                    "container": {
-                        "entrypoint": ["nemo-platform"],
-                        "command": [
-                            "run",
-                            "task",
-                            "--task",
-                            "nmp.hello_world.tasks.workload_workspace_get",
-                        ],
-                    },
-                },
-                "environment": [
-                    {
-                        "name": "NEMO_WORKLOAD_TOKEN",
-                        "from_secret": {"name": os.environ["WORKLOAD_TOKEN_SECRET"]},
-                    }
-                ],
-                "config": {"workspace": os.environ["WORKSPACE"]},
-            }
-        ]
-    },
-}
-
-json.dump(payload, sys.stdout, indent=2)
-PY
-```
+Keep this token in a non-reserved shell variable. Do not export it as
+`NEMO_WORKLOAD_TOKEN` in your shell. The NeMo CLI uses that variable as a
+runtime credential override, which would make later CLI commands run as the
+workload identity instead of `authentik-human`.
 
-Submit the job:
+Submit a job that runs the built-in hello-world workload auth task:
 
 ```bash
-nemo --context authentik-human jobs create "$JOB_NAME" \
+export NMP_API_IMAGE="${NMP_API_IMAGE:-${IMAGE_REGISTRY:-my-registry}/nmp-api:${BAKE_TAG:-local}}"
+
+cat <<EOF | nemo --context authentik-human jobs create "$JOB_NAME" \
   --workspace "$WORKSPACE" \
-  --input-file /tmp/authentik-workload-job.json
+  --input-file -
+{
+  "source": "authentik-reference-example",
+  "spec": {"demo": "authentik-workload-auth"},
+  "platform_spec": {
+    "steps": [
+      {
+        "name": "workload-workspace-get",
+        "executor": {
+          "provider": "cpu",
+          "profile": "workload",
+          "container": {
+            "image": "${NMP_API_IMAGE}",
+            "entrypoint": ["nemo-platform"],
+            "command": [
+              "run",
+              "task",
+              "--task",
+              "nmp.hello_world.tasks.workload_workspace_get"
+            ]
+          }
+        },
+        "environment": [
+          {
+            "name": "NEMO_WORKLOAD_TOKEN",
+            "value": "${WORKLOAD_ACCESS_TOKEN}"
+          }
+        ],
+        "config": {"workspace": "${WORKSPACE}"}
+      }
+    ]
+  }
+}
+EOF
 ```
 
 Watch it complete:
@@ -212,7 +211,11 @@ Watch it complete:
 nemo --context authentik-human jobs get-status "$JOB_NAME" --workspace "$WORKSPACE"
 ```
 
-After the job reaches `completed`, read the logs:
+Expected result: the job reaches `completed`. The workload exits successfully
+only after it uses the Authentik workload token to call NeMo through the gateway
+and retrieve the workspace.
+
+Read the job logs:
 
 ```bash
 nemo --context authentik-human jobs get-logs "$JOB_NAME" \
@@ -226,9 +229,6 @@ Expected result: the logs include:
 Successfully retrieved workspace: authentik-demo
 ```
 
-That confirms the job workload used the Authentik workload token to call NeMo
-through the gateway.
-
 ## Refresh The CLI Session
 
 The example requests `offline_access`, so the CLI stores a refresh token.
@@ -247,7 +247,6 @@ Remove the demo job and workspace if you created them:
 
 ```bash
 nemo --context authentik-human jobs delete "$JOB_NAME" --workspace "$WORKSPACE"
-nemo --context authentik-human secrets delete "$WORKLOAD_TOKEN_SECRET" --workspace "$WORKSPACE"
 nemo --context authentik-human workspaces delete "$WORKSPACE"
 ```
 
@@ -261,12 +260,19 @@ Then stop the stack with `Ctrl-C` in the terminal running
 - `Permission denied`: verify `nemo --context authentik-human auth status`,
   then check
   `nemo --context authentik-human workspaces members list --workspace "$WORKSPACE"`.
+  If `auth status` reports a `NEMO_WORKLOAD_TOKEN` or `NEMO_WORKLOAD_TOKEN_FILE`
+  environment override, run `unset NEMO_WORKLOAD_TOKEN NEMO_WORKLOAD_TOKEN_FILE`
+  and retry.
   If you just created the workspace or member bindings, wait a few seconds and
   retry so the authorization cache can refresh.
 - Job stays `created` or `pending`: confirm the stack was started with
   `contrib/auth/authentik/run.sh stack` and Docker is running.
-- Job completes but logs are missing: confirm `nemo-editors` has `JobLogWriter`
-  on the workspace.
+- Job fails while fetching a secret with `status code 401`: confirm the job
+  request injects `NEMO_WORKLOAD_TOKEN` with `value`, not `from_secret`. The
+  local gateway accepts Authentik bearer tokens, while the launcher secret-fetch
+  path uses internal service headers before the workload starts.
+- Job fails with `Permission denied`: confirm `nemo-editors` still has `Viewer`
+  and `JobRunner` on the workspace before creating the job.
 
 ## Adapting This Example
 

contrib/auth/authentik/blueprints/nemo.yaml

@@ -119,11 +119,11 @@ entries:
       expiring: false
 
   - model: authentik_core.user
-    id: svc-nemo-ci
+    id: svc-nemo
     identifiers:
-      username: svc-nemo-ci
+      username: svc-nemo
     attrs:
-      username: svc-nemo-ci
+      username: svc-nemo
       name: "NeMo demo workload identity"
       type: service_account
       path: service-accounts
@@ -132,12 +132,12 @@ entries:
 
   - model: authentik_core.token
     identifiers:
-      identifier: svc-nemo-ci-token
+      identifier: svc-nemo-token
     attrs:
-      identifier: svc-nemo-ci-token
+      identifier: svc-nemo-token
       intent: app_password
-      user: !KeyOf svc-nemo-ci
-      key: svc-nemo-ci-token-secret-dev
+      user: !KeyOf svc-nemo
+      key: svc-nemo-token-secret-dev
       expiring: false
 
   - model: authentik_brands.brand

contrib/auth/authentik/config/platform-compose-authentik.yaml

@@ -31,16 +31,8 @@ auth:
 entities: {}
 
 jobs:
+  enable_subprocess_executor: false
   executors:
-    - provider: subprocess
-      profile: default
-      backend: subprocess
-      config:
-        working_directory: /data/subprocess-jobs
-        cleanup_completed_jobs_immediately: false
-        ttl_seconds_before_active: 60
-        ttl_seconds_active: 3600
-        ttl_seconds_after_finished: 300
     - provider: cpu
       profile: workload
       backend: docker
@@ -54,12 +46,6 @@ jobs:
       ttl_seconds_before_active: 60
       ttl_seconds_active: 3600
       ttl_seconds_after_finished: 300
-    subprocess:
-      working_directory: /data/subprocess-jobs
-      cleanup_completed_jobs_immediately: false
-      ttl_seconds_before_active: 60
-      ttl_seconds_active: 3600
-      ttl_seconds_after_finished: 300
 
 evaluator:
   recreate_existing_system_entities: true

contrib/auth/authentik/docker-compose.yml

@@ -11,6 +11,9 @@ x-authentik-env: &authentik-env
 services:
   nemo:
     image: ${IMAGE_REGISTRY:-my-registry}/nmp-api:${BAKE_TAG:-local}
+    # The local Docker jobs backend controls sibling workload containers through
+    # the mounted host socket, which Docker Desktop exposes as root-owned.
+    user: "0:0"
     environment:
       MODE: development
       NMP_CONFIG_FILE_PATH: /etc/nmp/config.yaml

contrib/auth/authentik/manifest.yaml

@@ -14,7 +14,7 @@ human_identity:
   username: nemo-user
   expected_email: nemo-user@example.com
 workload_identity:
-  principal_id: svc-nemo-ci
+  principal_id: svc-nemo
   expected_groups:
     - nemo-editors
 workload_contract:
@@ -41,8 +41,8 @@ token_acquisition:
     grant_type: password
     client_id: nemo-platform
     client_secret: nemo-platform-secret-dev
-    username: svc-nemo-ci
-    password: svc-nemo-ci-token-secret-dev
+    username: svc-nemo
+    password: svc-nemo-token-secret-dev
     scope: "openid email groups"
 healthchecks:
   - kind: http