From 154204132eb77b44d784b0b110adc74f1471b84e Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 11:53:32 +0200 Subject: [PATCH 1/7] Introduce executionEngineSpec in CRD; adjust RBAC Signed-off-by: avasilev --- Fournos_Design_Document.md | 6 +- Makefile | 4 +- README.md | 6 +- config/forge/resolve_job.yaml | 4 +- config/forge/samples/job-full.yaml | 4 +- config/forge/samples/job-test-only.yaml | 4 +- config/forge/workflows/tasks.yaml | 4 +- .../samples/test-connectivity-job.yaml | 4 +- dev/job-secret-demo.yaml | 4 +- dev/sample-job.yaml | 4 +- fournos/core/resolve.py | 5 +- fournos/handlers/resolving.py | 9 ++ fournos/settings.py | 4 +- manifests/crd.yaml | 24 ++++- manifests/ocpci-sa/rbac-wip-ns.yaml | 2 +- manifests/ocpci-sa/role.yaml | 2 +- tests/test_exclusive.py | 99 +++++++++++++++---- tests/test_lifecycle.py | 42 ++++++-- tests/test_resolving.py | 89 ++++++++++++----- tests/test_scheduling.py | 49 +++++++-- tests/test_secret_refs.py | 12 ++- tests/test_shutdown.py | 35 ++++++- tests/test_validation.py | 23 ++++- 23 files changed, 348 insertions(+), 91 deletions(-) diff --git a/Fournos_Design_Document.md b/Fournos_Design_Document.md index 860d3f3..8159dd6 100644 --- a/Fournos_Design_Document.md +++ b/Fournos_Design_Document.md @@ -90,7 +90,9 @@ spec: owner: perf-team displayName: nightly-llama3-benchmark cluster: cluster-1 - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: testproj/llmd args: - cks @@ -346,7 +348,7 @@ All settings via environment variables with `FOURNOS_` prefix ([fournos/settings | `FOURNOS_GPU_RESOURCE_PREFIX` | `fournos/gpu-` | Virtual resource name prefix | | `FOURNOS_LOG_LEVEL` | `INFO` | Logging level | | `FOURNOS_GC_INTERVAL_SEC` | `300` | Resource GC interval (seconds) | -| `FOURNOS_RESOLVE_IMAGE` | `image-registry.openshift-image-registry.svc:5000/{namespace}/forge-core:main` | Container image for the resolve Job (`{namespace}` is substituted at runtime) | +| `FOURNOS_RESOLVE_IMAGE_REGISTRY` | `image-registry.openshift-image-registry.svc:5000/{namespace}/` | Registry prefix for the resolve image (`{namespace}` is substituted at runtime). The image name comes from `spec.executionEngineSpec.resolveImage`. | | `FOURNOS_RESOLVE_DEADLINE_SEC` | `300` | Deadline for the resolve Job (seconds) | | `FOURNOS_RESOLVE_JOB_TEMPLATE` | `config/forge/resolve_job.yaml` | Path (relative to project root) to the Job YAML template for the resolve step | diff --git a/Makefile b/Makefile index 78619ae..ea63dd9 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ dev-run: FOURNOS_GC_INTERVAL_SEC=5 \ FOURNOS_NAMESPACE=$(or $(FOURNOS_NAMESPACE),fournos-local-dev) \ FOURNOS_SECRETS_NAMESPACE=$(or $(FOURNOS_SECRETS_NAMESPACE),psap-secrets) \ - FOURNOS_RESOLVE_IMAGE=fournos-mock-resolve:dev \ + FOURNOS_RESOLVE_IMAGE_REGISTRY="" \ FOURNOS_RESOLVE_JOB_TEMPLATE=dev/mock-resolve/resolve_job.yaml \ $(VENV_BIN)python -m fournos @@ -94,7 +94,7 @@ ci-run: FOURNOS_GC_INTERVAL_SEC=5 \ FOURNOS_NAMESPACE=$(or $(FOURNOS_NAMESPACE),psap-automation-ci-test) \ FOURNOS_SECRETS_NAMESPACE=$(or $(FOURNOS_SECRETS_NAMESPACE),psap-secrets) \ - FOURNOS_RESOLVE_IMAGE=fournos-mock-resolve:dev \ + FOURNOS_RESOLVE_IMAGE_REGISTRY="" \ FOURNOS_RESOLVE_JOB_TEMPLATE=dev/mock-resolve/resolve_job.yaml \ $(VENV_BIN)python -m fournos \ --liveness=http://0.0.0.0:8080/healthz > fournos.log 2>&1 & \ diff --git a/README.md b/README.md index 0321f4e..d22cfd0 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,9 @@ spec: gpuType: a100 gpuCount: 2 pipeline: forge-full - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: llmd args: - cks @@ -308,7 +310,7 @@ All settings are read from environment variables with the `FOURNOS_` prefix: | `FOURNOS_GPU_RESOURCE_PREFIX` | `fournos/gpu-` | Resource name prefix for GPU types | | `FOURNOS_LOG_LEVEL` | `INFO` | Logging level | | `FOURNOS_GC_INTERVAL_SEC` | `300` | Resource GC interval (seconds) | -| `FOURNOS_RESOLVE_IMAGE` | `image-registry.openshift-image-registry.svc:5000/{namespace}/forge-core:main` | Container image for the resolve Job (`{namespace}` is substituted at runtime) | +| `FOURNOS_RESOLVE_IMAGE_REGISTRY` | `image-registry.openshift-image-registry.svc:5000/{namespace}/` | Registry prefix for the resolve image (`{namespace}` is substituted at runtime). The image name itself comes from `spec.executionEngineSpec.resolveImage`. | | `FOURNOS_RESOLVE_DEADLINE_SEC` | `300` | Deadline for the resolve Job (seconds) | | `FOURNOS_RESOLVE_JOB_TEMPLATE` | `config/forge/resolve_job.yaml` | Path (relative to project root) to the Job YAML template for the resolve step. Override with `dev/mock-resolve/resolve_job.yaml` for local dev/CI. | | `FOURNOS_ARTIFACT_PVC_SIZE` | `1Gi` | Size of the per-PipelineRun PVC used for shared artifact storage across pipeline tasks | diff --git a/config/forge/resolve_job.yaml b/config/forge/resolve_job.yaml index 5ce3c98..3e7d0f0 100644 --- a/config/forge/resolve_job.yaml +++ b/config/forge/resolve_job.yaml @@ -65,9 +65,9 @@ spec: fi git show --quiet - FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.forge.project) + FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngineSpec.project) if [[ -z "$FORGE_PROJECT" || "$FORGE_PROJECT" == null ]]; then - echo "ERROR: invalid .spec.forge.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" + echo "ERROR: invalid .spec.executionEngineSpec.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" exit 1 fi diff --git a/config/forge/samples/job-full.yaml b/config/forge/samples/job-full.yaml index bb6e12d..aae79ea 100644 --- a/config/forge/samples/job-full.yaml +++ b/config/forge/samples/job-full.yaml @@ -7,7 +7,9 @@ spec: displayName: forge-full-sample cluster: cluster-1 pipeline: forge-full - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: llm_d args: - cks diff --git a/config/forge/samples/job-test-only.yaml b/config/forge/samples/job-test-only.yaml index 9553f1f..eca7eab 100644 --- a/config/forge/samples/job-test-only.yaml +++ b/config/forge/samples/job-test-only.yaml @@ -7,7 +7,9 @@ spec: displayName: forge-sample cluster: cluster-1 pipeline: forge-test-only - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: llm_d args: - cks diff --git a/config/forge/workflows/tasks.yaml b/config/forge/workflows/tasks.yaml index 4bd352d..59caa29 100644 --- a/config/forge/workflows/tasks.yaml +++ b/config/forge/workflows/tasks.yaml @@ -88,9 +88,9 @@ spec: fi git show --quiet - FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.forge.project) + FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngineSpec.project) if [[ -z "$FORGE_PROJECT" || "$FORGE_PROJECT" == null ]]; then - echo "ERROR: invalid .spec.forge.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" + echo "ERROR: invalid .spec.executionEngineSpec.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" exit 1 fi diff --git a/config/fournos-validation/samples/test-connectivity-job.yaml b/config/fournos-validation/samples/test-connectivity-job.yaml index d626331..6dd84e1 100644 --- a/config/fournos-validation/samples/test-connectivity-job.yaml +++ b/config/fournos-validation/samples/test-connectivity-job.yaml @@ -10,6 +10,8 @@ spec: hardware: gpuType: a100 gpuCount: 1 - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: none args: [] diff --git a/dev/job-secret-demo.yaml b/dev/job-secret-demo.yaml index 8b935a0..8d404a9 100644 --- a/dev/job-secret-demo.yaml +++ b/dev/job-secret-demo.yaml @@ -7,7 +7,9 @@ spec: displayName: mock-secret-demo cluster: cluster-1 pipeline: fournos-run-only - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: secret-demo args: - demo diff --git a/dev/sample-job.yaml b/dev/sample-job.yaml index 6511a3b..be58f9a 100644 --- a/dev/sample-job.yaml +++ b/dev/sample-job.yaml @@ -9,7 +9,9 @@ spec: hardware: gpuType: a100 gpuCount: 2 - forge: + executionEngine: forge + executionEngineSpec: + resolveImage: forge-core:main project: llm-d args: - cks diff --git a/fournos/core/resolve.py b/fournos/core/resolve.py index a257c0f..f1a00b8 100644 --- a/fournos/core/resolve.py +++ b/fournos/core/resolve.py @@ -49,6 +49,7 @@ def create_job( *, name: str, owner_ref: dict, + image: str, ) -> dict: job_name = _resolve_job_name(name) labels = {LABEL_MANAGED_BY: "fournos", LABEL_JOB_NAME: name} @@ -64,9 +65,7 @@ def create_job( body["spec"]["template"]["metadata"] = {"labels": labels} container = body["spec"]["template"]["spec"]["containers"][0] - container["image"] = settings.resolve_image.format( - namespace=settings.namespace, - ) + container["image"] = image env_values = { "FJOB_NAME": name, diff --git a/fournos/handlers/resolving.py b/fournos/handlers/resolving.py index e8166bb..8227c7a 100644 --- a/fournos/handlers/resolving.py +++ b/fournos/handlers/resolving.py @@ -13,6 +13,7 @@ from fournos.core.constants import Phase from fournos.core.resolve import ResolveClient +from fournos.settings import settings from fournos.state import ctx from .status import ( @@ -57,10 +58,18 @@ def _ensure_resolve_job(spec, name, conditions, patch, body): if job is not None: return job + engine_spec = spec["executionEngineSpec"] + registry = engine_spec.get( + "resolveImageRegistry", + settings.resolve_image_registry, + ).format(namespace=settings.namespace) + resolve_image = registry + engine_spec["resolveImage"] + try: ctx.resolve.create_job( name=name, owner_ref=owner_ref(body), + image=resolve_image, ) except client.exceptions.ApiException as exc: if exc.status == 409: diff --git a/fournos/settings.py b/fournos/settings.py index 4a0ae33..34323bf 100644 --- a/fournos/settings.py +++ b/fournos/settings.py @@ -17,8 +17,8 @@ class Settings(BaseSettings): gpu_resource_prefix: str = "fournos/gpu-" gc_interval_sec: float = Field(default=300.0, gt=0) log_level: str = "INFO" - resolve_image: str = ( - "image-registry.openshift-image-registry.svc:5000/{namespace}/forge-core:main" + resolve_image_registry: str = ( + "image-registry.openshift-image-registry.svc:5000/{namespace}/" ) resolve_deadline_sec: int = Field(default=300, gt=0) resolve_job_template: str = "config/forge/resolve_job.yaml" diff --git a/manifests/crd.yaml b/manifests/crd.yaml index c749ab3..2248e22 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -51,7 +51,8 @@ spec: spec: type: object required: - - forge + - executionEngine + - executionEngineSpec properties: owner: type: string @@ -82,12 +83,27 @@ spec: gpuCount: type: integer minimum: 1 - forge: + executionEngine: + type: string + description: >- + Execution engine to use for this job (e.g. "forge"). + executionEngineSpec: type: object required: + - resolveImage - project - args properties: + resolveImage: + type: string + description: >- + Short image name for the resolve Job (e.g. "forge-core:main"). + The operator prepends the registry prefix automatically. + resolveImageRegistry: + type: string + description: >- + Registry prefix for the resolve image. Overrides the + operator default when set. Supports {namespace} placeholder. project: type: string args: @@ -105,8 +121,8 @@ spec: into the pipeline. The operator prepends vault- to look up the corresponding K8s Secret (which must carry the fournos.dev/vault-entry=true label) in the secrets - namespace. Populated by Forge during the Resolving - phase when not provided by the user. + namespace. Populated by the execution engine during the + Resolving phase when not provided by the user. items: type: string pattern: "^[a-z0-9]([a-z0-9\\-]{0,61}[a-z0-9])?$" diff --git a/manifests/ocpci-sa/rbac-wip-ns.yaml b/manifests/ocpci-sa/rbac-wip-ns.yaml index a743370..f5eb20a 100644 --- a/manifests/ocpci-sa/rbac-wip-ns.yaml +++ b/manifests/ocpci-sa/rbac-wip-ns.yaml @@ -5,7 +5,7 @@ metadata: namespace: psap-automation-wip rules: - apiGroups: ["fournos.dev"] - resources: ["fournosjobs"] + resources: ["fournosjobs", "fournosjobs/status"] verbs: ["create", "get", "list", "watch", "delete", "patch"] - apiGroups: [""] resources: ["pods"] diff --git a/manifests/ocpci-sa/role.yaml b/manifests/ocpci-sa/role.yaml index e7950f1..7fc0fd9 100644 --- a/manifests/ocpci-sa/role.yaml +++ b/manifests/ocpci-sa/role.yaml @@ -5,7 +5,7 @@ metadata: namespace: psap-automation rules: - apiGroups: ["fournos.dev"] - resources: ["fournosjobs"] + resources: ["fournosjobs", "fournosjobs/status"] verbs: ["create", "get", "list", "watch", "delete", "patch"] - apiGroups: [""] resources: ["pods"] diff --git a/tests/test_exclusive.py b/tests/test_exclusive.py index 868fdf8..e0df9d0 100644 --- a/tests/test_exclusive.py +++ b/tests/test_exclusive.py @@ -84,7 +84,12 @@ def test_exclusive_happy_path(k8s): { "cluster": "cluster-2", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -92,7 +97,7 @@ def test_exclusive_happy_path(k8s): k8s, "test-excl-happy", terminal={Phase.SUCCEEDED, Phase.FAILED}, - timeout=90, + timeout=120, ) assert phase == Phase.SUCCEEDED, job_status_summary(k8s, "test-excl-happy") @@ -108,7 +113,12 @@ def test_exclusive_workload_requests_all_slots(k8s): { "cluster": "cluster-2", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase( @@ -132,7 +142,12 @@ def test_normal_workload_requests_one_slot(k8s): { "exclusive": False, "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase( @@ -154,7 +169,12 @@ def test_exclusive_blocks_cluster_pinned_job(k8s): { "cluster": "cluster-2", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase(k8s, "test-excl-lock", terminal={Phase.RUNNING}, timeout=30) @@ -165,7 +185,12 @@ def test_exclusive_blocks_cluster_pinned_job(k8s): { "exclusive": False, "cluster": "cluster-2", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -211,7 +236,12 @@ def test_exclusive_steers_hardware_only_job(k8s): { "cluster": "cluster-1", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase(k8s, "test-excl-hw", terminal={Phase.RUNNING}, timeout=30) @@ -222,7 +252,12 @@ def test_exclusive_steers_hardware_only_job(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -249,7 +284,12 @@ def test_exclusive_without_cluster_fails(k8s): { "exclusive": True, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -276,7 +316,12 @@ def test_exclusive_waits_for_cluster_to_clear(k8s): { "exclusive": False, "cluster": "cluster-2", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase(k8s, "test-occupant", terminal={Phase.RUNNING}, timeout=30) @@ -287,7 +332,12 @@ def test_exclusive_waits_for_cluster_to_clear(k8s): { "cluster": "cluster-2", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -327,7 +377,12 @@ def test_lock_released_on_completion(k8s): { "cluster": "cluster-1", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) poll_phase(k8s, "test-excl-release", terminal={Phase.RUNNING}, timeout=30) @@ -338,7 +393,12 @@ def test_lock_released_on_completion(k8s): { "exclusive": False, "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -376,9 +436,9 @@ def test_lock_released_on_completion(k8s): def test_exclusive_without_hardware(k8s): """Exclusive + cluster without hardware: locks cluster using only cluster-slot resources. - A noop resolve Job prevents Forge from populating hardware. The - Workload should carry 100 cluster-slots and a nodeSelector but no - GPU resource requests. + A noop resolve Job prevents the execution engine from populating + hardware. The Workload should carry 100 cluster-slots and a + nodeSelector but no GPU resource requests. """ create_noop_resolve_job("test-excl-nohw") @@ -388,7 +448,12 @@ def test_exclusive_without_hardware(k8s): { "cluster": "cluster-2", "exclusive": True, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py index ca6e7df..5a2bb78 100644 --- a/tests/test_lifecycle.py +++ b/tests/test_lifecycle.py @@ -25,7 +25,12 @@ def test_workload_cleaned_after_completion(k8s): "test-wl-cleanup", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -63,7 +68,12 @@ def test_delete_cleans_up_resources(k8s): "test-delete", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -96,7 +106,12 @@ def test_list_multiple_jobs(k8s): "test-list-a", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) create_job( @@ -104,7 +119,12 @@ def test_list_multiple_jobs(k8s): "test-list-b", { "cluster": "cluster-2", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -124,7 +144,12 @@ def test_filter_jobs_by_phase(k8s): "test-filter-ok", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) create_job( @@ -133,7 +158,12 @@ def test_filter_jobs_by_phase(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) diff --git a/tests/test_resolving.py b/tests/test_resolving.py index 7f17299..8d7d1e2 100644 --- a/tests/test_resolving.py +++ b/tests/test_resolving.py @@ -1,7 +1,7 @@ -"""Resolving phase tests — Forge resolution via K8s Jobs. +"""Resolving phase tests — execution engine resolution via K8s Jobs. Tests cover the happy path (with and without user-provided hardware), -hardware validation (invalid GPU type, missing hardware), Forge Job +hardware validation (invalid GPU type, missing hardware), resolve Job failures, shutdown during Resolving, and cleanup via ownerReference cascade. """ @@ -35,14 +35,19 @@ def _set_shutdown(k8s, name: str, mode: str) -> None: def test_happy_path_with_hardware(k8s): - """Job with spec.hardware: user-provided values take precedence over Forge.""" + """Job with spec.hardware: user-provided values take precedence over the execution engine.""" create_job( k8s, "test-resolve-hw", { "cluster": "cluster-3", "hardware": {"gpuType": "h200", "gpuCount": 4}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -88,13 +93,18 @@ def test_happy_path_with_hardware(k8s): def test_happy_path_without_hardware(k8s): - """Job without spec.hardware: Forge populates it during resolution.""" + """Job without spec.hardware: the execution engine populates it during resolution.""" create_job( k8s, "test-resolve-nohw", { "exclusive": False, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -114,7 +124,7 @@ def test_happy_path_without_hardware(k8s): gpu_req = get_workload_gpu_request("test-resolve-nohw", "a100") assert gpu_req == 2, ( - f"Workload should request 2 a100 GPUs from Forge-resolved spec; got {gpu_req}" + f"Workload should request 2 a100 GPUs from resolved spec; got {gpu_req}" ) phase = poll_phase( @@ -130,13 +140,18 @@ def test_happy_path_without_hardware(k8s): def test_cluster_pin_without_hardware(k8s): - """Cluster pin without hardware — Forge provides hardware, Kueue pins to cluster.""" + """Cluster pin without hardware — the execution engine provides hardware, Kueue pins to cluster.""" create_job( k8s, "test-resolve-pin", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -149,7 +164,7 @@ def test_cluster_pin_without_hardware(k8s): gpu_req = get_workload_gpu_request("test-resolve-pin", "a100") assert gpu_req == 2, ( - f"Workload should request 2 a100 GPUs from Forge-resolved spec; got {gpu_req}" + f"Workload should request 2 a100 GPUs from resolved spec; got {gpu_req}" ) phase = poll_phase( @@ -174,7 +189,12 @@ def test_shutdown_during_resolving(k8s): { "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -207,7 +227,12 @@ def test_delete_during_resolving_cleans_up(k8s): { "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -240,7 +265,12 @@ def test_unknown_gpu_type(k8s): { "exclusive": False, "hardware": {"gpuType": "acbd1234", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -263,11 +293,11 @@ def test_unknown_gpu_type(k8s): def test_resolve_job_failure(k8s): - """Forge resolve Job fails -> FournosJob transitions to Failed. + """Resolve Job fails -> FournosJob transitions to Failed. A pre-created resolve Job that immediately exits with code 1 simulates - a Forge failure. The operator should detect the failure and set - phase=Failed with a message mentioning the resolution failure. + an execution engine failure. The operator should detect the failure and + set phase=Failed with a message mentioning the resolution failure. The failed Job is preserved for debugging. """ create_failing_resolve_job("test-resolve-fail") @@ -277,7 +307,12 @@ def test_resolve_job_failure(k8s): "test-resolve-fail", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -311,9 +346,9 @@ def test_resolve_job_failure(k8s): def test_nonexclusive_cluster_without_hardware_fails(k8s): """Non-exclusive + cluster + no hardware → Failed (hardware required). - A noop resolve Job prevents Forge from populating hardware. Since - the job is non-exclusive, the missing hardware is not allowed (only - exclusive+cluster jobs may omit it). + A noop resolve Job prevents the execution engine from populating + hardware. Since the job is non-exclusive, the missing hardware is + not allowed (only exclusive+cluster jobs may omit it). """ create_noop_resolve_job("test-nex-nohw") @@ -323,7 +358,12 @@ def test_nonexclusive_cluster_without_hardware_fails(k8s): { "exclusive": False, "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -358,7 +398,12 @@ def test_resolve_empty_hw(k8s): "test-resolve-noconfig", { "exclusive": False, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) diff --git a/tests/test_scheduling.py b/tests/test_scheduling.py index 3c21259..2c76103 100644 --- a/tests/test_scheduling.py +++ b/tests/test_scheduling.py @@ -25,7 +25,12 @@ def test_cluster_pinned(k8s): "test-cluster", { "cluster": "cluster-2", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -93,7 +98,12 @@ def test_hardware_request(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["llama3", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["llama3", "internal-test"], + }, "priority": "nightly", }, ) @@ -121,7 +131,12 @@ def test_cluster_and_hardware(k8s): { "cluster": "cluster-4", "hardware": {"gpuType": "h200", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["llama3", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["llama3", "internal-test"], + }, }, ) @@ -171,7 +186,12 @@ def test_shared_cluster_with_hardware(k8s): "exclusive": False, "cluster": "cluster-3", "hardware": {"gpuType": "h200", "gpuCount": 4}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -210,7 +230,12 @@ def test_alternative_pipeline_selection(k8s): { "pipeline": "fournos-run-only", "cluster": "cluster-2", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -237,7 +262,12 @@ def test_inadmissible_stays_pending(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -289,7 +319,12 @@ def test_cluster_without_required_gpu_stays_pending(k8s): { "cluster": "cluster-3", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) diff --git a/tests/test_secret_refs.py b/tests/test_secret_refs.py index ad52b6f..5fb5c65 100644 --- a/tests/test_secret_refs.py +++ b/tests/test_secret_refs.py @@ -1,7 +1,7 @@ """End-to-end tests — Vault sync -> secretRef resolution -> PipelineRun. -SecretRefs live on the FournosJob spec and are populated by Forge during -the Resolving phase. The Vault HTTP layer is mocked so no real Vault is +SecretRefs live on the FournosJob spec and are populated by the execution +engine during the Resolving phase. The Vault HTTP layer is mocked so no real Vault is needed, but secrets are created on the live cluster by the sync script, then consumed by a FournosJob whose spec.secretRefs references them. @@ -112,7 +112,9 @@ def test_vault_sync_then_fjob(k8s, core_v1): "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, "secretRefs": [VAULT_ENTRY], - "forge": { + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", "project": "testproj/llmd", "args": ["cks", "internal-test"], }, @@ -179,7 +181,9 @@ def test_missing_secret_ref_fails(k8s): "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, "secretRefs": ["nonexistent-vault-entry"], - "forge": { + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", "project": "testproj/llmd", "args": ["cks", "internal-test"], }, diff --git a/tests/test_shutdown.py b/tests/test_shutdown.py index 7a8884f..6d20ef7 100644 --- a/tests/test_shutdown.py +++ b/tests/test_shutdown.py @@ -43,7 +43,12 @@ def test_stop_pending_job(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -78,7 +83,12 @@ def test_stop_running_job(k8s): "test-stop-running", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -151,7 +161,12 @@ def test_stop_at_creation(k8s): "test-stop-create", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, "shutdown": "Stop", }, ) @@ -178,7 +193,12 @@ def test_stop_completed_job_is_noop(k8s): "test-stop-done", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -218,7 +238,12 @@ def test_terminate_running_job(k8s): "test-term-running", { "cluster": "cluster-1", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) diff --git a/tests/test_validation.py b/tests/test_validation.py index e7c2cb0..4ec7daa 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -2,7 +2,7 @@ Tests cover on_create rejections (unknown cluster, exclusive without cluster) and admission-phase errors (no flavor assigned). Resolving-phase validation -(GPU type, Forge failures) lives in test_resolving.py. +(GPU type, execution engine failures) lives in test_resolving.py. """ import json @@ -27,7 +27,12 @@ def test_unknown_cluster(k8s): "test-unknown", { "cluster": "no-such-cluster", - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -54,7 +59,12 @@ def test_admitted_without_flavor(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 999}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) @@ -128,7 +138,12 @@ def test_implicit_exclusive_without_cluster_fails(k8s): "test-implicit-excl", { "hardware": {"gpuType": "a100", "gpuCount": 2}, - "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]}, + "executionEngine": "forge", + "executionEngineSpec": { + "resolveImage": "fournos-mock-resolve:dev", + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + }, }, ) From b1426475045e914028af8e257d67359f1561c866 Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 12:10:52 +0200 Subject: [PATCH 2/7] Improve registry name handling Signed-off-by: avasilev --- fournos/handlers/resolving.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fournos/handlers/resolving.py b/fournos/handlers/resolving.py index 8227c7a..a2758cf 100644 --- a/fournos/handlers/resolving.py +++ b/fournos/handlers/resolving.py @@ -62,7 +62,9 @@ def _ensure_resolve_job(spec, name, conditions, patch, body): registry = engine_spec.get( "resolveImageRegistry", settings.resolve_image_registry, - ).format(namespace=settings.namespace) + ).replace("{namespace}", settings.namespace) + if registry and not registry.endswith("/"): + registry += "/" resolve_image = registry + engine_spec["resolveImage"] try: From b675c3d4af945fe24f8038f21caf135dd2a756a4 Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 12:17:48 +0200 Subject: [PATCH 3/7] Update documentation Signed-off-by: avasilev --- Fournos_Design_Document.md | 105 +++++++++++++++++----------------- README.md | 54 +++++++++-------- fournos/handlers/resolving.py | 14 ++--- 3 files changed, 90 insertions(+), 83 deletions(-) diff --git a/Fournos_Design_Document.md b/Fournos_Design_Document.md index 8159dd6..c7c24a3 100644 --- a/Fournos_Design_Document.md +++ b/Fournos_Design_Document.md @@ -2,7 +2,7 @@ ## 1. Introduction -*Fournos* (φούρνος) = "oven" in Greek. A Kubernetes operator that accepts benchmark jobs as `FournosJob` custom resources, schedules them via Kueue, and executes them as Tekton PipelineRuns on remote clusters through the FORGE framework. +*Fournos* (φούρνος) = "oven" in Greek. A Kubernetes operator that accepts benchmark jobs as `FournosJob` custom resources, schedules them via Kueue, and executes them as Tekton PipelineRuns on remote clusters through a pluggable execution engine. The operator is built with [kopf](https://kopf.dev/) (Kubernetes Operator Pythonic Framework) and runs as a single-replica Deployment. @@ -14,25 +14,25 @@ flowchart LR subgraph Hub["Hub cluster (psap-automation)"] K8sAPI["Kubernetes API"] Operator["Fournos Operator\n(kopf)"] - ForgeResolve["Forge Resolve\n(K8s Job)"] + ResolveJob["Resolve Job\n(K8s Job)"] Kueue["Kueue"] Tekton["Tekton Pipelines"] - FORGE["FORGE\n(in Tekton Tasks)"] + ExecEngine["Execution Engine\n(in Tekton Tasks)"] K8sAPI --> Operator - Operator --> ForgeResolve - ForgeResolve -->|"patches FournosJob spec"| K8sAPI + Operator --> ResolveJob + ResolveJob -->|"patches FournosJob spec"| K8sAPI Operator --> Kueue Operator --> Tekton - Tekton --> FORGE + Tekton --> ExecEngine end - FORGE -- "remote oc/kubectl\nvia kubeconfig Secrets" --> Target1["Target cluster 1"] - FORGE -- "remote oc/kubectl\nvia kubeconfig Secrets" --> Target2["Target cluster 2"] + ExecEngine -- "remote oc/kubectl\nvia kubeconfig Secrets" --> Target1["Target cluster 1"] + ExecEngine -- "remote oc/kubectl\nvia kubeconfig Secrets" --> Target2["Target cluster 2"] ``` -- **Hub cluster**: hosts the Fournos operator, Kueue, Tekton Pipelines, and FORGE (running inside Tekton Task pods) in the `psap-automation` namespace -- **Target clusters**: nothing installed — FORGE runs on the hub cluster inside Tekton Task pods and communicates with targets via remote `oc`/`kubectl` commands using kubeconfig Secrets +- **Hub cluster**: hosts the Fournos operator, Kueue, Tekton Pipelines, and the execution engine (running inside Tekton Task pods) in the `psap-automation` namespace +- **Target clusters**: nothing installed — the execution engine runs on the hub cluster inside Tekton Task pods and communicates with targets via remote `oc`/`kubectl` commands using kubeconfig Secrets - **Consumers**: interact via `kubectl` (or any Kubernetes client) to create/watch/delete `FournosJob` CRs ## 3. FournosJob CRD @@ -44,10 +44,13 @@ Jobs are submitted as `FournosJob` custom resources ([manifests/crd.yaml](manife | Field | Required | Description | | ---------------------------- | ------------ | ------------------------------------------------------------------------------------------------ | -| `spec.forge.project` | yes | FORGE project path | -| `spec.forge.args` | yes | List of arguments passed to FORGE | -| `spec.forge.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | -| `spec.env` | no | Environment variables available to FORGE (read from the FournosJob spec via K8s API) | +| `spec.executionEngine` | yes | Execution engine to use (e.g. `forge`) | +| `spec.executionEngineSpec.resolveImage` | yes | Short image name for the resolve Job (e.g. `forge-core:main`) | +| `spec.executionEngineSpec.resolveImageRegistry` | no | Registry prefix override (supports `{namespace}` placeholder) | +| `spec.executionEngineSpec.project` | yes | Execution engine project path | +| `spec.executionEngineSpec.args` | yes | List of arguments passed to the execution engine | +| `spec.executionEngineSpec.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | +| `spec.env` | no | Environment variables available to the execution engine (read from the FournosJob spec via K8s API) | | `spec.cluster` | | Pin to a specific cluster (Kueue ResourceFlavor). Since `exclusive` defaults to `true`, this also locks the cluster — set `exclusive: false` for shared access. | | `spec.hardware.gpuType` | | Short GPU model name (e.g. `a100`, `h200`). The operator adds the resource prefix automatically. | | `spec.hardware.gpuCount` | with gpuType | Number of GPUs (minimum 1) | @@ -55,14 +58,14 @@ Jobs are submitted as `FournosJob` custom resources ([manifests/crd.yaml](manife | `spec.displayName` | no | Human-readable job name (defaults to `metadata.name`) | | `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`) | | `spec.priority` | no | Kueue WorkloadPriorityClass name | -| `spec.secretRefs` | no | Vault-synced K8s Secret names (`vault-`) to mount into the pipeline. Populated by Forge during the Resolving phase. Each must be a K8s Secret with `fournos.dev/vault-entry=true` in `FOURNOS_SECRETS_NAMESPACE`. During the Admitted phase the operator copies them into the operator namespace and mounts them as a projected volume at `/var/run/secrets/fournos//`. | +| `spec.secretRefs` | no | Vault-synced K8s Secret names (`vault-`) to mount into the pipeline. Populated by the execution engine during the Resolving phase. Each must be a K8s Secret with `fournos.dev/vault-entry=true` in `FOURNOS_SECRETS_NAMESPACE`. During the Admitted phase the operator copies them into the operator namespace and mounts them as a projected volume at `/var/run/secrets/fournos//`. | | `spec.exclusive` | no (default `true`) | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. Hardware is optional — when omitted the Workload only requests cluster-slot resources for locking. | | `spec.shutdown` | no | Shutdown action: `Stop` (graceful, runs finally tasks) or `Terminate` (immediate, skips finally tasks). Both wait for the PipelineRun to finish before releasing Kueue quota. | -`spec.hardware` is required unless the job uses exclusive cluster locking (`exclusive: true` + `cluster`), in which case it may be omitted — the Workload only needs cluster-slot resources. Every job passes through the mandatory Resolving phase where Forge populates `spec.hardware` (if not already set) and `spec.secretRefs` directly on the FournosJob. Since `exclusive` defaults to `true`, any job with `spec.cluster` locks the cluster exclusively — including jobs that also specify `spec.hardware`. Set `exclusive: false` for shared access. Jobs without `spec.cluster` must set `exclusive: false`. +`spec.hardware` is required unless the job uses exclusive cluster locking (`exclusive: true` + `cluster`), in which case it may be omitted — the Workload only needs cluster-slot resources. Every job passes through the mandatory Resolving phase where the execution engine populates `spec.hardware` (if not already set) and `spec.secretRefs` directly on the FournosJob. Since `exclusive` defaults to `true`, any job with `spec.cluster` locks the cluster exclusively — including jobs that also specify `spec.hardware`. Set `exclusive: false` for shared access. Jobs without `spec.cluster` must set `exclusive: false`. -`metadata.name` is the unique identifier for the job. Use `metadata.generateName` for auto-generated unique names (e.g. `generateName: nightly-benchmark-` produces `nightly-benchmark-x7k2m`). `spec.displayName` is a human-readable label for external correlation — it does not need to be unique. FORGE reads it directly from the FournosJob spec via the K8s API. +`metadata.name` is the unique identifier for the job. Use `metadata.generateName` for auto-generated unique names (e.g. `generateName: nightly-benchmark-` produces `nightly-benchmark-x7k2m`). `spec.displayName` is a human-readable label for external correlation — it does not need to be unique. The execution engine reads it directly from the FournosJob spec via the K8s API. ### Status @@ -150,7 +153,7 @@ sequenceDiagram participant Client participant K8sAPI as Kubernetes API participant Operator as Fournos Operator - participant Forge as Forge Resolve Job + participant Resolve as Resolve Job participant Kueue participant Tekton @@ -159,10 +162,10 @@ sequenceDiagram Operator->>Operator: validate spec Operator->>K8sAPI: set phase=Resolving - Note over Operator,Forge: timer (5s): create resolve Job - Operator->>Forge: create resolve K8s Job (ownerRef → FournosJob) - Forge->>K8sAPI: patch FournosJob spec with hardware, secretRefs - Forge-->>Operator: Job completed + Note over Operator,Resolve: timer (5s): create resolve Job + Operator->>Resolve: create resolve K8s Job (ownerRef → FournosJob) + Resolve->>K8sAPI: patch FournosJob spec with hardware, secretRefs + Resolve-->>Operator: Job completed Operator->>Operator: read FournosJob spec, validate hardware + secretRefs Operator->>Kueue: create Workload (cluster-slot=1 or 100) @@ -197,9 +200,9 @@ sequenceDiagram 1. **on_create**: Operator validates the spec (cluster exists if specified, `exclusive` requires `cluster` — and `exclusive` defaults to `true`). If `spec.shutdown` is set (`Stop` or `Terminate`), immediately sets `phase=Stopped`. Otherwise sets `phase=Resolving`. -2. **timer (Resolving)**: Launches a Forge resolve K8s Job that patches the FournosJob spec with `hardware` (if not user-provided) and `secretRefs`. Polls the Job for completion. On success, reads the FournosJob spec, validates hardware (GPU type checked against Kueue; hardware is optional for exclusive+cluster jobs), validates `secretRefs` against Vault secrets, creates the Kueue Workload (exclusive jobs request all 100 `fournos/cluster-slot` units; non-exclusive jobs request 1), and sets `phase=Pending`. Failed resolve Jobs are preserved for debugging. +2. **timer (Resolving)**: Launches a resolve K8s Job (using the configured execution engine image) that patches the FournosJob spec with `hardware` (if not user-provided) and `secretRefs`. Polls the Job for completion. On success, reads the FournosJob spec, validates hardware (GPU type checked against Kueue; hardware is optional for exclusive+cluster jobs), validates `secretRefs` against Vault secrets, creates the Kueue Workload (exclusive jobs request all 100 `fournos/cluster-slot` units; non-exclusive jobs request 1), and sets `phase=Pending`. Failed resolve Jobs are preserved for debugging. 3. **timer (Pending)**: Polls the Workload for Kueue admission. On admission, extracts the assigned cluster and sets `phase=Admitted`. -4. **timer (Admitted)**: Reads `secretRefs` from the FournosJob spec, copies each referenced secret from `secrets_namespace` into the operator namespace (per-job name `-`, with `ownerReferences` for automatic cleanup), resolves the kubeconfig Secret, creates the Tekton PipelineRun with `FJOB_NAME` + `FOURNOS_NAMESPACE` params (so FORGE can look up the full spec), a projected `vault-secrets` volume mounting all copied secrets at `/var/run/secrets/fournos//`, and `ownerReferences` pointing at the FournosJob, sets `phase=Running`. +4. **timer (Admitted)**: Reads `secretRefs` from the FournosJob spec, copies each referenced secret from `secrets_namespace` into the operator namespace (per-job name `-`, with `ownerReferences` for automatic cleanup), resolves the kubeconfig Secret, creates the Tekton PipelineRun with `FJOB_NAME` + `FOURNOS_NAMESPACE` params (so the execution engine can look up the full spec), a projected `vault-secrets` volume mounting all copied secrets at `/var/run/secrets/fournos//`, and `ownerReferences` pointing at the FournosJob, sets `phase=Running`. 5. **timer (Running)**: Polls the PipelineRun for completion. On success/failure, deletes the Workload and sets `phase=Succeeded` or `phase=Failed`. 6. **timer (any non-terminal phase, shutdown)**: If `spec.shutdown` is set (`Stop` or `Terminate`) and the job has a PipelineRun (Admitted/Running), the timer cancels the PipelineRun — `Stop` uses Tekton's `CancelledRunFinally` (runs `finally` tasks), `Terminate` uses `Cancelled` (skips `finally` tasks) — and sets `phase=Stopping`. The Workload is **not** deleted yet — it stays alive to hold the cluster slot while the PipelineRun winds down. If no PipelineRun exists (Pending), the Workload is deleted immediately and the job goes straight to `phase=Stopped`. 7. **timer (Stopping)**: Polls the PipelineRun until it reaches a terminal state (`succeeded` or `failed`). Once complete, deletes the Workload to release Kueue quota and sets `phase=Stopped`. @@ -221,9 +224,9 @@ The operator is split across several modules: - **`fournos/handlers/`** — phase handler package: - `status.py` — condition helpers, `owner_ref`, `create_workload_for_job`, shared constants - `lifecycle.py` — `on_create`, `reconcile_pending` (early phases) - - `resolving.py` — `reconcile_resolving` (Forge resolve Job management, spec validation, Workload creation) + - `resolving.py` — `reconcile_resolving` (resolve Job management, spec validation, Workload creation) - `execution.py` — `reconcile_admitted`, `reconcile_running` (PipelineRun management), `handle_shutdown` / `reconcile_stopping` (shutdown flow) -- **`fournos/core/resolve.py`** — `ResolveClient` for managing Forge resolve K8s Jobs (create, status) +- **`fournos/core/resolve.py`** — `ResolveClient` for managing resolve K8s Jobs (create, status) - **`fournos/state.py`** — shared client instances (`_OperatorState` dataclass with `kueue`, `tekton`, `registry`, `resolve`) Kopf handlers registered in `operator.py`: @@ -233,12 +236,12 @@ Kopf handlers registered in `operator.py`: | ------------------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------------- | | `@kopf.on.startup` | Process start | Load kubeconfig, initialise clients into `state.ctx`, start resource GC thread | | `@kopf.on.create` / `@kopf.on.resume` | New or existing CR | Validate spec, set `phase=Resolving` | -| `@kopf.timer(interval=5.0)` | Every 5s while phase ∈ {Resolving, Pending, Admitted, Running, Stopping} | Drive the state machine: Resolving creates Forge Job (which patches FournosJob spec), validates results, creates Workload; Pending polls admission; Admitted creates PipelineRun; Running polls completion; Stopping polls PipelineRun for completion. Shutdown is checked in every non-terminal phase. | +| `@kopf.timer(interval=5.0)` | Every 5s while phase ∈ {Resolving, Pending, Admitted, Running, Stopping} | Drive the state machine: Resolving creates resolve Job (which patches FournosJob spec), validates results, creates Workload; Pending polls admission; Admitted creates PipelineRun; Running polls completion; Stopping polls PipelineRun for completion. Shutdown is checked in every non-terminal phase. | The timer's `when` guard ensures it stops firing once the job reaches a terminal phase (`Succeeded`, `Failed`, or `Stopped`), so completed jobs have zero ongoing overhead. -Validation failures (unknown cluster, `exclusive` without `cluster`) result in immediate `phase=Failed` with a descriptive `message` during `on_create`. Hardware and secretRef validation failures occur during the Resolving phase after the Forge Job completes. +Validation failures (unknown cluster, `exclusive` without `cluster`) result in immediate `phase=Failed` with a descriptive `message` during `on_create`. Hardware and secretRef validation failures occur during the Resolving phase after the resolve Job completes. ### Resource GC @@ -248,41 +251,41 @@ A background daemon thread runs a garbage collection loop at a configurable inte Job state is stored entirely in Kubernetes resources — no in-memory store: -- **FournosJob CRs**: the primary user-facing resource; `.spec` includes hardware and secretRefs (populated by Forge during Resolving if not user-provided); `.status` tracks phase, assigned cluster, PipelineRun name, dashboard URL +- **FournosJob CRs**: the primary user-facing resource; `.spec` includes hardware and secretRefs (populated by the execution engine during Resolving if not user-provided); `.status` tracks phase, assigned cluster, PipelineRun name, dashboard URL - **Kueue Workloads**: carry job name as labels; admission state from conditions and `status.admission.podSetAssignments` - **Tekton PipelineRuns**: carry job name as labels; execution status from conditions The operator is stateless and crash-safe. On restart, `@kopf.on.resume` re-evaluates existing CRs and the timer picks up where it left off. -## 7. FORGE integration +## 7. Execution engine integration -FORGE is an existing benchmark execution framework that runs on the hub cluster inside Tekton Task pods and owns all operations on target clusters — setup, benchmark execution, and cleanup — by issuing remote `oc`/`kubectl` commands via kubeconfig Secrets. Fournos has a strict separation of concerns: it handles cluster selection, scheduling, and bookkeeping, but never interacts with target clusters directly. +The execution engine is the benchmark framework that runs on the hub cluster inside Tekton Task pods and owns all operations on target clusters — setup, benchmark execution, and cleanup — by issuing remote `oc`/`kubectl` commands via kubeconfig Secrets. Fournos has a strict separation of concerns: it handles cluster selection, scheduling, and bookkeeping, but never interacts with target clusters directly. Instead of extracting individual fields from the FournosJob spec and passing them as separate pipeline params, the operator passes two identifiers to both the resolve Job and the Tekton Pipeline: - **`FJOB_NAME`** — the FournosJob `metadata.name` - **`FOURNOS_NAMESPACE`** — the operator namespace -FORGE uses these to look up the full FournosJob spec via the Kubernetes API, giving it access to all configuration in one go (`spec.forge`, `spec.env`, etc.) without the operator needing to serialize and forward individual fields. +The execution engine uses these to look up the full FournosJob spec via the Kubernetes API, giving it access to all configuration in one go (`spec.executionEngineSpec`, `spec.env`, etc.) without the operator needing to serialize and forward individual fields. -FORGE reads `spec.displayName` (or `metadata.name`) directly from the FournosJob spec for its own resource naming and correlation. +The execution engine reads `spec.displayName` (or `metadata.name`) directly from the FournosJob spec for its own resource naming and correlation. -**Hub configuration vs mocks:** `config/forge/` is the authoritative layout for deploying FORGE on the hub (workflows, images, samples). Tasks in `config/forge/workflows/tasks.yaml` and `config/fournos-validation/workflows/tasks.yaml` implement the parameter interface for real clusters. `dev/mock-pipelines/` holds echo/sleep Tekton stand-ins used only by kind-based dev setup and tests—not a substitute for `config/forge/`. +**Hub configuration vs mocks:** `config/forge/` is the authoritative layout for deploying the execution engine on the hub (workflows, images, samples). Tasks in `config/forge/workflows/tasks.yaml` and `config/fournos-validation/workflows/tasks.yaml` implement the parameter interface for real clusters. `dev/mock-pipelines/` holds echo/sleep Tekton stand-ins used only by kind-based dev setup and tests — not a substitute for `config/forge/`. ## 8. Tekton Pipelines and Tasks -The Task and Pipeline YAML checked in under `config/forge/workflows/` is what you apply on OpenShift for real workloads. Pipelines under `dev/mock-pipelines/` exist for local kind clusters and automated tests; they reuse the same `spec.pipeline` names but are not the production FORGE definitions. +The Task and Pipeline YAML checked in under `config/forge/workflows/` is what you apply on OpenShift for real workloads. Pipelines under `dev/mock-pipelines/` exist for local kind clusters and automated tests; they reuse the same `spec.pipeline` names but are not the production definitions. ### Tasks ([config/forge/workflows/tasks.yaml](config/forge/workflows/tasks.yaml)) -FORGE-owned tasks (stubs in this repo, replaced by real FORGE implementation): +Execution-engine-owned tasks (stubs in this repo, replaced by the real execution engine implementation): -| Task | Description | -| ----------------- | --------------------------------------------------- | -| `fournos-prepare` | FORGE: set up the target cluster | -| `fournos-run` | FORGE: run the benchmark against the target cluster | -| `fournos-cleanup` | FORGE: clean up resources on the target cluster | +| Task | Description | +| ----------------- | -------------------------------------------------------------- | +| `fournos-prepare` | Execution engine: set up the target cluster | +| `fournos-run` | Execution engine: run the benchmark against the target cluster | +| `fournos-cleanup` | Execution engine: clean up resources on the target cluster | ### Pipelines @@ -364,12 +367,12 @@ fournos/ __init__.py # Re-exports for operator.py status.py # Condition helpers, owner_ref, create_workload_for_job lifecycle.py # on_create, reconcile_pending - resolving.py # reconcile_resolving (Forge resolve Job, spec validation, Workload creation) + resolving.py # reconcile_resolving (resolve Job, spec validation, Workload creation) execution.py # reconcile_admitted, reconcile_running core/ constants.py # Shared label keys, Phase enum, cluster-slot constants clusters.py # ClusterRegistry (kubeconfig lookup, secretRef resolution, cross-namespace secret copying) - resolve.py # ResolveClient (Forge resolve Job management) + resolve.py # ResolveClient (resolve Job management) tekton.py # TektonClient (PipelineRun CRUD) kueue.py # KueueClient (Workload CRUD, admission checks, cluster-slot requests) manifests/ @@ -379,18 +382,18 @@ manifests/ config/ kueue-cluster-config.yaml # ResourceFlavors, ClusterQueue, WorkloadPriorityClasses kueue-config.yaml # LocalQueue (namespace-scoped) - forge/ # Hub cluster: real FORGE ImageStreams, Builds, Tekton workflows, samples (not mocks) + forge/ # Hub cluster: real execution engine ImageStreams, Builds, Tekton workflows, samples (not mocks) dev/ setup.sh # kind cluster setup (Tekton + Kueue + CRDs + mock resources + mock resolve image) mock-kueue-config.yaml # Dev Kueue config (mock clusters, quotas) mock-pipelines/ # Echo/sleep Tekton Tasks and Pipelines for kind only - mock-resolve/ # Mock Forge resolve image (Dockerfile + resolve.sh + resolve_job.yaml template) for local dev/CI + mock-resolve/ # Mock resolve image (Dockerfile + resolve.sh + resolve_job.yaml template) for local dev/CI sample-job.yaml # Example FournosJob CR for testing tests/ conftest.py # Fixtures (kubernetes client, helpers, cleanup) test_scheduling.py # Cluster pin, hardware, both, alt pipeline, inadmissible, wrong GPU, optional spec fields, default-exclusive slot count test_validation.py # Unknown cluster, admitted without flavor, implicit exclusive without cluster - test_resolving.py # Resolving phase: happy paths, hardware precedence, Forge failures, GPU validation, non-exclusive cluster without hardware + test_resolving.py # Resolving phase: happy paths, hardware precedence, resolve failures, GPU validation, non-exclusive cluster without hardware test_lifecycle.py # Workload cleanup, delete cleanup, list, filter by phase test_resource_gc.py # Stale Workload/PipelineRun garbage collection test_exclusive.py # Exclusive cluster locking (happy path, blocking, occupancy, lock release) @@ -409,8 +412,8 @@ README.md - **CRD-based operator** (kopf) — consumers interact via `kubectl` / Kubernetes API, getting RBAC, audit logging, and `kubectl wait` for free - **Unified Kueue scheduling** — all jobs flow through Kueue for consistent quota tracking and priority ordering. Cluster-pinned jobs use `nodeSelector` to constrain admission to a single ResourceFlavor; hardware-request jobs leave all flavors eligible. -- **Separation of concerns** — Fournos owns scheduling, bookkeeping, and parameter passing; FORGE owns all target-cluster operations (setup, execution, cleanup). Fournos never touches target clusters directly. -- **FORGE is opaque** — Fournos never validates FORGE config; it passes `FJOB_NAME` and `FOURNOS_NAMESPACE` so FORGE can look up the full FournosJob spec via the K8s API +- **Separation of concerns** — Fournos owns scheduling, bookkeeping, and parameter passing; the execution engine (e.g. FORGE) owns all target-cluster operations (setup, execution, cleanup). Fournos never touches target clusters directly. +- **Execution engine is opaque** — Fournos never validates execution engine config; it passes `FJOB_NAME` and `FOURNOS_NAMESPACE` so the execution engine can look up the full FournosJob spec via the K8s API - **Tekton for execution, Kueue for scheduling** — virtual Workload pattern with `fournos/gpu-`* resources - **Stateless operator** — all job state lives in Kubernetes resources (FournosJob CRs, PipelineRuns, Workloads), not in memory. Crash-safe via `on_resume`. - **Timer-based reconciliation** — the operator polls Workload admission and PipelineRun completion via a kopf timer (5s interval), eliminating the need for callback tasks or watch streams on third-party resources @@ -418,8 +421,8 @@ README.md - **ownerReferences for cascade deletion** — Workloads, PipelineRuns, resolve Jobs, and copied secrets all carry `ownerReferences` pointing at their FournosJob, so Kubernetes automatically cascade-deletes them when the job is removed - **Exclusive locking via Kueue semaphore** — each cluster flavor has 100 `fournos/cluster-slot` units. Non-exclusive jobs request 1 slot; exclusive jobs (the default) request all 100. Kueue enforces mutual exclusion atomically — no operator-level blocking, labels, or in-memory state needed. Hardware-only jobs are automatically steered to clusters with available slots. Exclusive jobs with a cluster may omit hardware — the Workload only needs cluster-slot resources for locking. - **Shutdown via spec field** — the `spec.shutdown` enum supports two modes: `Stop` (Tekton `CancelledRunFinally` — runs `finally` cleanup tasks) and `Terminate` (Tekton `Cancelled` — skips `finally` tasks). Both transition to an intermediate `Stopping` phase while the PipelineRun winds down. The Workload (and its quota) is kept alive until the PipelineRun completes, ensuring the cluster slot is not released prematurely. Only then does the operator delete the Workload and set `phase=Stopped`. The enum is extensible for future shutdown strategies. The FournosJob stays around in `Stopped` phase for inspection, unlike deletion which cascades and removes the record. -- **Mandatory Resolving phase** — every job passes through a `Resolving` phase before entering `Pending`. During this phase, a Forge K8s Job runs to determine hardware requirements (`gpuType`, `gpuCount`) and secret references (`secretRefs`). Forge patches these values directly into the FournosJob spec (hardware only when not already user-provided). The operator validates them after the Job completes. Failed resolve Jobs are preserved for debugging. -- **Vault-based secret management with cross-namespace injection** — pipeline secrets originate in a HashiCorp Vault and are synced to K8s Secrets on demand via `hacks/sync_vault_secrets.py` into a dedicated secrets namespace (`FOURNOS_SECRETS_NAMESPACE`, default `psap-secrets`). The K8s Secret name uses a `vault-` prefix followed by the Vault entry name (e.g. `vault-my-creds`); entries that are not valid DNS-1123 names are rejected during sync. Each synced Secret carries a `fournos.dev/vault-entry=true` label. Secret references are populated by Forge during the Resolving phase on `spec.secretRefs`. The operator validates them in the secrets namespace before creating the Workload (during Resolving). During the Admitted phase, the operator copies each referenced secret from the secrets namespace into the operator namespace with a per-job name (`-`) and `ownerReferences` to the FournosJob for automatic cleanup. The copies are combined into a single projected volume (`vault-secrets`) mounted at `/var/run/secrets/fournos//`, with each secret's keys placed under a subdirectory matching its original name. This avoids key collisions across secrets and works with a static `volumeMount` in the Task YAML regardless of how many secrets a job uses. An empty projected volume (no secrets) is always emitted so the static mount never fails. Missing or non-vault refs fail the job during Resolving rather than creating a broken PipelineRun. +- **Mandatory Resolving phase** — every job passes through a `Resolving` phase before entering `Pending`. During this phase, a resolve K8s Job (using the configured execution engine image) runs to determine hardware requirements (`gpuType`, `gpuCount`) and secret references (`secretRefs`). The resolve Job patches these values directly into the FournosJob spec (hardware only when not already user-provided). The operator validates them after the Job completes. Failed resolve Jobs are preserved for debugging. +- **Vault-based secret management with cross-namespace injection** — pipeline secrets originate in a HashiCorp Vault and are synced to K8s Secrets on demand via `hacks/sync_vault_secrets.py` into a dedicated secrets namespace (`FOURNOS_SECRETS_NAMESPACE`, default `psap-secrets`). The K8s Secret name uses a `vault-` prefix followed by the Vault entry name (e.g. `vault-my-creds`); entries that are not valid DNS-1123 names are rejected during sync. Each synced Secret carries a `fournos.dev/vault-entry=true` label. Secret references are populated by the execution engine during the Resolving phase on `spec.secretRefs`. The operator validates them in the secrets namespace before creating the Workload (during Resolving). During the Admitted phase, the operator copies each referenced secret from the secrets namespace into the operator namespace with a per-job name (`-`) and `ownerReferences` to the FournosJob for automatic cleanup. The copies are combined into a single projected volume (`vault-secrets`) mounted at `/var/run/secrets/fournos//`, with each secret's keys placed under a subdirectory matching its original name. This avoids key collisions across secrets and works with a static `volumeMount` in the Task YAML regardless of how many secrets a job uses. An empty projected volume (no secrets) is always emitted so the static mount never fails. Missing or non-vault refs fail the job during Resolving rather than creating a broken PipelineRun. - **Multiple pipelines** — `fournos-full` (prepare → run → cleanup) and `fournos-run-only` (run only), selectable per job -- **Target clusters need nothing installed** — FORGE runs on the hub cluster inside Tekton Task pods and communicates with targets via remote `oc`/`kubectl` commands through kubeconfig Secrets (stored in the dedicated secrets namespace) +- **Target clusters need nothing installed** — the execution engine runs on the hub cluster inside Tekton Task pods and communicates with targets via remote `oc`/`kubectl` commands through kubeconfig Secrets (stored in the dedicated secrets namespace) diff --git a/README.md b/README.md index d22cfd0..443b124 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,15 @@ Fournos is a Kubernetes operator that schedules benchmark jobs via [Kueue](https://kueue.sigs.k8s.io/) and executes them as -[Tekton](https://tekton.dev/) PipelineRuns on remote clusters through the -FORGE framework. +[Tekton](https://tekton.dev/) PipelineRuns on remote clusters through a +pluggable execution engine. Jobs are submitted as `FournosJob` custom resources. Every job first -passes through a mandatory **Resolving** phase where a Forge Job populates -GPU requirements and secret references directly on the FournosJob spec. The -operator then creates a Kueue Workload for quota management, waits for -admission, and launches the corresponding Tekton PipelineRun. +passes through a mandatory **Resolving** phase where a resolve Job (driven +by the configured execution engine) populates GPU requirements and secret +references directly on the FournosJob spec. The operator then creates a +Kueue Workload for quota management, waits for admission, and launches the +corresponding Tekton PipelineRun. ## Cluster dependencies @@ -76,10 +77,13 @@ oc delete FournosJob -n $FOURNOS_NAMESPACE # cleanup | Field | Required | Description | |---|---|---| -| `spec.forge.project` | yes | FORGE project path | -| `spec.forge.args` | yes | List of arguments passed to FORGE | -| `spec.forge.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | -| `spec.env` | no | Environment variables available to FORGE (read from the FournosJob spec via K8s API) | +| `spec.executionEngine` | yes | Execution engine to use (e.g. `forge`) | +| `spec.executionEngineSpec.resolveImage` | yes | Short image name for the resolve Job (e.g. `forge-core:main`) | +| `spec.executionEngineSpec.resolveImageRegistry` | no | Registry prefix override (supports `{namespace}` placeholder) | +| `spec.executionEngineSpec.project` | yes | Execution engine project path | +| `spec.executionEngineSpec.args` | yes | List of arguments passed to the execution engine | +| `spec.executionEngineSpec.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | +| `spec.env` | no | Environment variables available to the execution engine (read from the FournosJob spec via K8s API) | | `spec.cluster` | \* | Pin to a specific cluster (Kueue ResourceFlavor). Since `exclusive` defaults to `true`, this also locks the cluster — set `exclusive: false` for shared access. | | `spec.hardware.gpuType` | \* | Short GPU model name — e.g. `a100`, `h200`. The operator prepends the `FOURNOS_GPU_RESOURCE_PREFIX` (default `fournos/gpu-`) automatically, so do **not** include the full resource path. | | `spec.hardware.gpuCount` | with gpuType | Number of GPUs (minimum 1) | @@ -87,15 +91,15 @@ oc delete FournosJob -n $FOURNOS_NAMESPACE # cleanup | `spec.displayName` | no | Human-readable job name (defaults to `metadata.name`) | | `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`) | | `spec.priority` | no | Kueue WorkloadPriorityClass name | -| `spec.secretRefs` | no | Vault-synced K8s Secret names (prefixed with `vault-`) to mount into the pipeline. Populated by Forge during the Resolving phase. The operator validates each name in `FOURNOS_SECRETS_NAMESPACE`, copies the secrets into the operator namespace, and mounts them as a projected volume at `/var/run/secrets/fournos//`. | +| `spec.secretRefs` | no | Vault-synced K8s Secret names (prefixed with `vault-`) to mount into the pipeline. Populated by the execution engine during the Resolving phase. The operator validates each name in `FOURNOS_SECRETS_NAMESPACE`, copies the secrets into the operator namespace, and mounts them as a projected volume at `/var/run/secrets/fournos//`. | | `spec.exclusive` | no (default `true`) | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. Hardware is optional — when omitted the Workload only requests cluster-slot resources for locking. | | `spec.shutdown` | no | Shutdown action: `Stop` cancels gracefully (Tekton `CancelledRunFinally` — runs `finally` tasks); `Terminate` cancels immediately (Tekton `Cancelled` — skips `finally` tasks). Both wait for the PipelineRun to finish before releasing Kueue quota. | \* `spec.hardware` is required unless the job uses exclusive cluster locking (`exclusive: true` + `cluster`), in which case it may be omitted — the Workload only needs cluster-slot resources. Every job passes through the -Resolving phase where Forge populates `spec.hardware` (if not already set) -and `spec.secretRefs` directly on the FournosJob. Since `exclusive` defaults +Resolving phase where the execution engine populates `spec.hardware` (if +not already set) and `spec.secretRefs` directly on the FournosJob. Since `exclusive` defaults to `true`, any job with `spec.cluster` locks the cluster exclusively — including jobs that also specify `spec.hardware`. Set `exclusive: false` for shared access (hardware is then required). Jobs without `spec.cluster` must @@ -138,7 +142,7 @@ make dev-teardown # deletes the kind cluster `dev-setup` installs real Tekton Pipelines and Kueue controllers into the kind cluster, but substitutes lightweight mock Tasks (echo + sleep) in place of the -real FORGE runner. The dev environment uses its own Kueue config +real execution engine runner. The dev environment uses its own Kueue config (`dev/mock-kueue-config.yaml`) with four mock clusters and synthetic GPU quotas, plus matching kubeconfig Secrets (`kubeconfig-cluster-{1..4}`) in the dedicated secrets namespace (`psap-secrets`). @@ -152,7 +156,7 @@ make test # integration tests (operator must be running) ## Deployment -**FORGE on the hub:** [`config/forge/`](config/forge/) is the real OpenShift configuration for this repo—ImageStreams, Builds, Tekton Tasks and Pipelines, and sample jobs you apply to a cluster. It is **not** the same as the lightweight stand-ins under [`dev/mock-pipelines/`](dev/mock-pipelines/), which [`make dev-setup`](#local-development) installs on kind for local testing only. +**Execution engine on the hub:** [`config/forge/`](config/forge/) is the real OpenShift configuration for this repo — ImageStreams, Builds, Tekton Tasks and Pipelines, and sample jobs you apply to a cluster. It is **not** the same as the lightweight stand-ins under [`dev/mock-pipelines/`](dev/mock-pipelines/), which [`make dev-setup`](#local-development) installs on kind for local testing only. Prepare the namespaces ```bash @@ -215,14 +219,14 @@ oc get fournosjobs -n $FOURNOS_NAMESPACE -w # should reach Succeeded ``` This runs the `fournos-validate-only` pipeline, which only checks `oc -cluster-info` against the target — no FORGE workload is launched. If the job +cluster-info` against the target — no benchmark workload is launched. If the job reaches `Succeeded`, the kubeconfig secret and Kueue quota are correctly configured. If it fails, check the operator logs and the PipelineRun status for details. -### Deploying the FORGE workflow configuration +### Deploying the execution engine workflow configuration -Apply the production FORGE assets from `config/forge/` (not the kind mocks in `dev/mock-pipelines/`). Deploy the cluster configuration (Builds + Tekton): +Apply the production execution engine assets from `config/forge/` (not the kind mocks in `dev/mock-pipelines/`). Deploy the cluster configuration (Builds + Tekton): ```bash oc apply -n $FOURNOS_NAMESPACE -f config/forge/images/is_forge.yaml @@ -274,7 +278,7 @@ make sync-vault-secrets-dry-run # preview only The synced secrets are labelled `fournos.dev/vault-entry=true` and `app.kubernetes.io/managed-by=fournos-vault-sync` for easy identification. -Secret references are populated by Forge during the Resolving phase directly +Secret references are populated by the execution engine during the Resolving phase directly on the FournosJob `spec.secretRefs` field. The operator validates each referenced Secret exists in the secrets namespace and carries the vault label during the Resolving phase, then copies them into the operator @@ -318,16 +322,16 @@ All settings are read from environment variables with the `FOURNOS_` prefix: ## Architecture ``` -FournosJob CR ──→ Operator ──→ Forge Resolve Job (patches FournosJob spec) ──→ Kueue Workload ──→ (admission) ──→ Tekton PipelineRun ──→ FORGE ──→ target cluster +FournosJob CR ──→ Operator ──→ Resolve Job (e.g. FORGE, patches FournosJob spec) ──→ Kueue Workload ──→ (admission) ──→ Tekton PipelineRun ──→ Execution Engine (e.g. FORGE) ──→ target cluster ``` The operator runs as a single-replica Deployment using [kopf](https://kopf.dev/). On each `FournosJob`, it: -1. **Resolves** job requirements by launching a Forge K8s Job that populates the FournosJob spec with GPU type/count and secret references +1. **Resolves** job requirements by launching a resolve K8s Job (using the configured execution engine image) that populates the FournosJob spec with GPU type/count and secret references 2. **Creates** a Kueue Workload with the resolved GPU resources (owned by the FournosJob via `ownerReferences`) 3. **Polls** (5 s timer) for Kueue admission and assigned cluster -4. **Copies** referenced Vault secrets from the secrets namespace into the operator namespace (per-job copies with `ownerReferences` for automatic cleanup) and **launches** a Tekton PipelineRun with `FJOB_NAME` + `FOURNOS_NAMESPACE` (so FORGE can look up the full FournosJob spec), the secrets mounted as a projected volume at `/var/run/secrets/fournos/` (owned by the FournosJob via `ownerReferences`), and a shared `artifacts` workspace backed by a `volumeClaimTemplate` PVC for cross-task artifact storage (managed by Tekton) +4. **Copies** referenced Vault secrets from the secrets namespace into the operator namespace (per-job copies with `ownerReferences` for automatic cleanup) and **launches** a Tekton PipelineRun with `FJOB_NAME` + `FOURNOS_NAMESPACE` (so the execution engine can look up the full FournosJob spec), the secrets mounted as a projected volume at `/var/run/secrets/fournos/` (owned by the FournosJob via `ownerReferences`), and a shared `artifacts` workspace backed by a `volumeClaimTemplate` PVC for cross-task artifact storage (managed by Tekton) 5. **Watches** the PipelineRun until completion 6. **Deletes** the Workload to release Kueue quota @@ -342,9 +346,9 @@ deleted and the job moves to `phase=Stopped`. Deleting a FournosJob automatically cascade-deletes its Workload and PipelineRun through Kubernetes owner references. -Target clusters need nothing installed — FORGE runs on the hub cluster inside -Tekton Task pods and communicates with targets via `oc`/`kubectl` through -kubeconfig Secrets. +Target clusters need nothing installed — the execution engine runs on the hub +cluster inside Tekton Task pods and communicates with targets via +`oc`/`kubectl` through kubeconfig Secrets. For a detailed breakdown of the CRD, scheduling, operator internals, and key design decisions, see the [Design Document](Fournos_Design_Document.md). diff --git a/fournos/handlers/resolving.py b/fournos/handlers/resolving.py index a2758cf..2c0eba5 100644 --- a/fournos/handlers/resolving.py +++ b/fournos/handlers/resolving.py @@ -1,6 +1,6 @@ """Resolving handler — reconcile_resolving. -Covers the Resolving phase: launching a Forge resolve Job that patches +Covers the Resolving phase: launching a resolve Job that patches the FournosJob spec with hardware and secretRefs, validating the results, and creating the Kueue Workload to transition into Pending. """ @@ -94,7 +94,7 @@ def _ensure_resolve_job(spec, name, conditions, patch, body): "Resolving", "Resolve Job created, waiting for completion", ) - patch.status["message"] = "Resolving job requirements via Forge" + patch.status["message"] = "Resolving job requirements" logger.info("Job %s: created resolve Job", name) return None @@ -118,7 +118,7 @@ def _check_job_finished(job, name, conditions, patch): patch, conditions, name, - f"Forge resolution failed: {message}", + f"Resolution failed: {message}", reason="Failed", cond_message=message, ) @@ -135,7 +135,7 @@ def _resolve_hardware( ) -> tuple[str | None, int | None]: """Determine and validate GPU requirements from the FournosJob spec. - Forge populates ``spec.hardware`` when absent. The GPU type is + The resolve Job populates ``spec.hardware`` when absent. The GPU type is validated against Kueue unless this is an exclusive cluster-lock job with no hardware requirements. @@ -161,7 +161,7 @@ def _resolve_hardware( conditions, name, "No hardware requirements: spec.hardware not populated " - "after Forge resolution", + "after resolution", reason="NoHardware", cond_message="No hardware requirements found", ) @@ -169,7 +169,7 @@ def _resolve_hardware( logger.warning( "Job %s: exclusive cluster lock without hardware — " "Workload will only request cluster-slot resources " - "(Forge may not have populated spec.hardware)", + "(resolve Job may not have populated spec.hardware)", name, ) return None, 0 @@ -277,7 +277,7 @@ def _create_workload_and_transition( COND_RESOLVED, "True", "Resolved", - "Forge resolution complete", + "Resolution complete", ) set_condition( patch, From a12a70acebad59a55c8d3ff30051ccec1f64c578 Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 16:01:15 +0200 Subject: [PATCH 4/7] Redesign executionEngine CRD field; use Pipeline Annotation for resolve image Signed-off-by: avasilev --- Fournos_Design_Document.md | 33 ++--- Makefile | 2 - README.md | 25 ++-- config/forge/resolve_job.yaml | 4 +- config/forge/samples/job-full.yaml | 11 +- config/forge/samples/job-test-only.yaml | 11 +- config/forge/workflows/pipeline-full.yaml | 2 + .../forge/workflows/pipeline-test-only.yaml | 2 + config/forge/workflows/tasks.yaml | 4 +- .../samples/test-connectivity-job.yaml | 9 +- .../workflows/pipeline-validate-only.yaml | 2 + dev/job-secret-demo.yaml | 11 +- dev/mock-pipelines/pipeline-full.yaml | 2 + dev/mock-pipelines/pipeline-run-only.yaml | 2 + dev/mock-pipelines/tasks.yaml | 2 + dev/sample-job.yaml | 25 ++-- fournos/core/tekton.py | 12 ++ fournos/handlers/resolving.py | 38 +++-- fournos/settings.py | 3 - manifests/crd.yaml | 35 +---- manifests/rbac/role_fournos.yaml | 3 + tests/test_exclusive.py | 130 +++++++++--------- tests/test_lifecycle.py | 60 ++++---- tests/test_resolving.py | 90 ++++++------ tests/test_scheduling.py | 70 +++++----- tests/test_secret_refs.py | 20 +-- tests/test_shutdown.py | 50 +++---- tests/test_validation.py | 30 ++-- 28 files changed, 344 insertions(+), 344 deletions(-) diff --git a/Fournos_Design_Document.md b/Fournos_Design_Document.md index c7c24a3..2e076bf 100644 --- a/Fournos_Design_Document.md +++ b/Fournos_Design_Document.md @@ -44,19 +44,14 @@ Jobs are submitted as `FournosJob` custom resources ([manifests/crd.yaml](manife | Field | Required | Description | | ---------------------------- | ------------ | ------------------------------------------------------------------------------------------------ | -| `spec.executionEngine` | yes | Execution engine to use (e.g. `forge`) | -| `spec.executionEngineSpec.resolveImage` | yes | Short image name for the resolve Job (e.g. `forge-core:main`) | -| `spec.executionEngineSpec.resolveImageRegistry` | no | Registry prefix override (supports `{namespace}` placeholder) | -| `spec.executionEngineSpec.project` | yes | Execution engine project path | -| `spec.executionEngineSpec.args` | yes | List of arguments passed to the execution engine | -| `spec.executionEngineSpec.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | +| `spec.executionEngine` | yes | Execution engine configuration. The single top-level key is the engine name (e.g. `forge`); its value is opaque engine-specific config passed through as-is. | | `spec.env` | no | Environment variables available to the execution engine (read from the FournosJob spec via K8s API) | | `spec.cluster` | | Pin to a specific cluster (Kueue ResourceFlavor). Since `exclusive` defaults to `true`, this also locks the cluster — set `exclusive: false` for shared access. | | `spec.hardware.gpuType` | | Short GPU model name (e.g. `a100`, `h200`). The operator adds the resource prefix automatically. | | `spec.hardware.gpuCount` | with gpuType | Number of GPUs (minimum 1) | | `spec.owner` | no | Team or individual that owns this job | | `spec.displayName` | no | Human-readable job name (defaults to `metadata.name`) | -| `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`) | +| `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`). The Pipeline must carry a `fournos.dev/resolve-image` annotation with the full image reference for the resolve Job. | | `spec.priority` | no | Kueue WorkloadPriorityClass name | | `spec.secretRefs` | no | Vault-synced K8s Secret names (`vault-`) to mount into the pipeline. Populated by the execution engine during the Resolving phase. Each must be a K8s Secret with `fournos.dev/vault-entry=true` in `FOURNOS_SECRETS_NAMESPACE`. During the Admitted phase the operator copies them into the operator namespace and mounts them as a projected volume at `/var/run/secrets/fournos//`. | | `spec.exclusive` | no (default `true`) | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. Hardware is optional — when omitted the Workload only requests cluster-slot resources for locking. | @@ -93,14 +88,13 @@ spec: owner: perf-team displayName: nightly-llama3-benchmark cluster: cluster-1 - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: testproj/llmd - args: - - cks - configOverrides: - batch_size: 64 + executionEngine: + forge: + project: testproj/llmd + args: + - cks + configOverrides: + batch_size: 64 env: OCPCI_SUITE: regression OCPCI_VARIANT: nightly @@ -200,7 +194,7 @@ sequenceDiagram 1. **on_create**: Operator validates the spec (cluster exists if specified, `exclusive` requires `cluster` — and `exclusive` defaults to `true`). If `spec.shutdown` is set (`Stop` or `Terminate`), immediately sets `phase=Stopped`. Otherwise sets `phase=Resolving`. -2. **timer (Resolving)**: Launches a resolve K8s Job (using the configured execution engine image) that patches the FournosJob spec with `hardware` (if not user-provided) and `secretRefs`. Polls the Job for completion. On success, reads the FournosJob spec, validates hardware (GPU type checked against Kueue; hardware is optional for exclusive+cluster jobs), validates `secretRefs` against Vault secrets, creates the Kueue Workload (exclusive jobs request all 100 `fournos/cluster-slot` units; non-exclusive jobs request 1), and sets `phase=Pending`. Failed resolve Jobs are preserved for debugging. +2. **timer (Resolving)**: Reads the `fournos.dev/resolve-image` annotation from the Tekton Pipeline referenced by `spec.pipeline` and launches a resolve K8s Job using that image. The resolve Job patches the FournosJob spec with `hardware` (if not user-provided) and `secretRefs`. Polls the Job for completion. On success, reads the FournosJob spec, validates hardware (GPU type checked against Kueue; hardware is optional for exclusive+cluster jobs), validates `secretRefs` against Vault secrets, creates the Kueue Workload (exclusive jobs request all 100 `fournos/cluster-slot` units; non-exclusive jobs request 1), and sets `phase=Pending`. Failed resolve Jobs are preserved for debugging. 3. **timer (Pending)**: Polls the Workload for Kueue admission. On admission, extracts the assigned cluster and sets `phase=Admitted`. 4. **timer (Admitted)**: Reads `secretRefs` from the FournosJob spec, copies each referenced secret from `secrets_namespace` into the operator namespace (per-job name `-`, with `ownerReferences` for automatic cleanup), resolves the kubeconfig Secret, creates the Tekton PipelineRun with `FJOB_NAME` + `FOURNOS_NAMESPACE` params (so the execution engine can look up the full spec), a projected `vault-secrets` volume mounting all copied secrets at `/var/run/secrets/fournos//`, and `ownerReferences` pointing at the FournosJob, sets `phase=Running`. 5. **timer (Running)**: Polls the PipelineRun for completion. On success/failure, deletes the Workload and sets `phase=Succeeded` or `phase=Failed`. @@ -266,7 +260,7 @@ Instead of extracting individual fields from the FournosJob spec and passing the - **`FJOB_NAME`** — the FournosJob `metadata.name` - **`FOURNOS_NAMESPACE`** — the operator namespace -The execution engine uses these to look up the full FournosJob spec via the Kubernetes API, giving it access to all configuration in one go (`spec.executionEngineSpec`, `spec.env`, etc.) without the operator needing to serialize and forward individual fields. +The execution engine uses these to look up the full FournosJob spec via the Kubernetes API, giving it access to all configuration in one go (`spec.executionEngine`, `spec.env`, etc.) without the operator needing to serialize and forward individual fields. The execution engine reads `spec.displayName` (or `metadata.name`) directly from the FournosJob spec for its own resource naming and correlation. @@ -302,6 +296,8 @@ All pipelines declare an `artifacts` workspace backed by a `volumeClaimTemplate` The `spec.pipeline` field in `FournosJob` selects which pipeline to use (default: `fournos-full`). +Every Pipeline must carry a `fournos.dev/resolve-image` annotation with the full image reference for the resolve Job (e.g. `image-registry.openshift-image-registry.svc:5000/psap-automation/forge-core:main`). The operator reads this annotation during the Resolving phase and uses it directly as the container image for the resolve K8s Job. + Completion detection is handled by the operator's timer polling PipelineRun conditions — no callback task is needed. ## 9. Kueue configuration @@ -351,7 +347,6 @@ All settings via environment variables with `FOURNOS_` prefix ([fournos/settings | `FOURNOS_GPU_RESOURCE_PREFIX` | `fournos/gpu-` | Virtual resource name prefix | | `FOURNOS_LOG_LEVEL` | `INFO` | Logging level | | `FOURNOS_GC_INTERVAL_SEC` | `300` | Resource GC interval (seconds) | -| `FOURNOS_RESOLVE_IMAGE_REGISTRY` | `image-registry.openshift-image-registry.svc:5000/{namespace}/` | Registry prefix for the resolve image (`{namespace}` is substituted at runtime). The image name comes from `spec.executionEngineSpec.resolveImage`. | | `FOURNOS_RESOLVE_DEADLINE_SEC` | `300` | Deadline for the resolve Job (seconds) | | `FOURNOS_RESOLVE_JOB_TEMPLATE` | `config/forge/resolve_job.yaml` | Path (relative to project root) to the Job YAML template for the resolve step | @@ -421,7 +416,7 @@ README.md - **ownerReferences for cascade deletion** — Workloads, PipelineRuns, resolve Jobs, and copied secrets all carry `ownerReferences` pointing at their FournosJob, so Kubernetes automatically cascade-deletes them when the job is removed - **Exclusive locking via Kueue semaphore** — each cluster flavor has 100 `fournos/cluster-slot` units. Non-exclusive jobs request 1 slot; exclusive jobs (the default) request all 100. Kueue enforces mutual exclusion atomically — no operator-level blocking, labels, or in-memory state needed. Hardware-only jobs are automatically steered to clusters with available slots. Exclusive jobs with a cluster may omit hardware — the Workload only needs cluster-slot resources for locking. - **Shutdown via spec field** — the `spec.shutdown` enum supports two modes: `Stop` (Tekton `CancelledRunFinally` — runs `finally` cleanup tasks) and `Terminate` (Tekton `Cancelled` — skips `finally` tasks). Both transition to an intermediate `Stopping` phase while the PipelineRun winds down. The Workload (and its quota) is kept alive until the PipelineRun completes, ensuring the cluster slot is not released prematurely. Only then does the operator delete the Workload and set `phase=Stopped`. The enum is extensible for future shutdown strategies. The FournosJob stays around in `Stopped` phase for inspection, unlike deletion which cascades and removes the record. -- **Mandatory Resolving phase** — every job passes through a `Resolving` phase before entering `Pending`. During this phase, a resolve K8s Job (using the configured execution engine image) runs to determine hardware requirements (`gpuType`, `gpuCount`) and secret references (`secretRefs`). The resolve Job patches these values directly into the FournosJob spec (hardware only when not already user-provided). The operator validates them after the Job completes. Failed resolve Jobs are preserved for debugging. +- **Mandatory Resolving phase** — every job passes through a `Resolving` phase before entering `Pending`. During this phase, a resolve K8s Job runs to determine hardware requirements (`gpuType`, `gpuCount`) and secret references (`secretRefs`). The resolve image is specified by the `fournos.dev/resolve-image` annotation on the Tekton Pipeline (selected via `spec.pipeline`). The resolve Job patches these values directly into the FournosJob spec (hardware only when not already user-provided). The operator validates them after the Job completes. Failed resolve Jobs are preserved for debugging. - **Vault-based secret management with cross-namespace injection** — pipeline secrets originate in a HashiCorp Vault and are synced to K8s Secrets on demand via `hacks/sync_vault_secrets.py` into a dedicated secrets namespace (`FOURNOS_SECRETS_NAMESPACE`, default `psap-secrets`). The K8s Secret name uses a `vault-` prefix followed by the Vault entry name (e.g. `vault-my-creds`); entries that are not valid DNS-1123 names are rejected during sync. Each synced Secret carries a `fournos.dev/vault-entry=true` label. Secret references are populated by the execution engine during the Resolving phase on `spec.secretRefs`. The operator validates them in the secrets namespace before creating the Workload (during Resolving). During the Admitted phase, the operator copies each referenced secret from the secrets namespace into the operator namespace with a per-job name (`-`) and `ownerReferences` to the FournosJob for automatic cleanup. The copies are combined into a single projected volume (`vault-secrets`) mounted at `/var/run/secrets/fournos//`, with each secret's keys placed under a subdirectory matching its original name. This avoids key collisions across secrets and works with a static `volumeMount` in the Task YAML regardless of how many secrets a job uses. An empty projected volume (no secrets) is always emitted so the static mount never fails. Missing or non-vault refs fail the job during Resolving rather than creating a broken PipelineRun. - **Multiple pipelines** — `fournos-full` (prepare → run → cleanup) and `fournos-run-only` (run only), selectable per job - **Target clusters need nothing installed** — the execution engine runs on the hub cluster inside Tekton Task pods and communicates with targets via remote `oc`/`kubectl` commands through kubeconfig Secrets (stored in the dedicated secrets namespace) diff --git a/Makefile b/Makefile index ea63dd9..81dc316 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,6 @@ dev-run: FOURNOS_GC_INTERVAL_SEC=5 \ FOURNOS_NAMESPACE=$(or $(FOURNOS_NAMESPACE),fournos-local-dev) \ FOURNOS_SECRETS_NAMESPACE=$(or $(FOURNOS_SECRETS_NAMESPACE),psap-secrets) \ - FOURNOS_RESOLVE_IMAGE_REGISTRY="" \ FOURNOS_RESOLVE_JOB_TEMPLATE=dev/mock-resolve/resolve_job.yaml \ $(VENV_BIN)python -m fournos @@ -94,7 +93,6 @@ ci-run: FOURNOS_GC_INTERVAL_SEC=5 \ FOURNOS_NAMESPACE=$(or $(FOURNOS_NAMESPACE),psap-automation-ci-test) \ FOURNOS_SECRETS_NAMESPACE=$(or $(FOURNOS_SECRETS_NAMESPACE),psap-secrets) \ - FOURNOS_RESOLVE_IMAGE_REGISTRY="" \ FOURNOS_RESOLVE_JOB_TEMPLATE=dev/mock-resolve/resolve_job.yaml \ $(VENV_BIN)python -m fournos \ --liveness=http://0.0.0.0:8080/healthz > fournos.log 2>&1 & \ diff --git a/README.md b/README.md index 443b124..a69eee8 100644 --- a/README.md +++ b/README.md @@ -51,14 +51,13 @@ spec: gpuType: a100 gpuCount: 2 pipeline: forge-full - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: llmd - args: - - cks - configOverrides: - batch_size: 64 + executionEngine: + forge: + project: llmd + args: + - cks + configOverrides: + batch_size: 64 env: OCPCI_SUITE: regression OCPCI_VARIANT: nightly @@ -77,19 +76,14 @@ oc delete FournosJob -n $FOURNOS_NAMESPACE # cleanup | Field | Required | Description | |---|---|---| -| `spec.executionEngine` | yes | Execution engine to use (e.g. `forge`) | -| `spec.executionEngineSpec.resolveImage` | yes | Short image name for the resolve Job (e.g. `forge-core:main`) | -| `spec.executionEngineSpec.resolveImageRegistry` | no | Registry prefix override (supports `{namespace}` placeholder) | -| `spec.executionEngineSpec.project` | yes | Execution engine project path | -| `spec.executionEngineSpec.args` | yes | List of arguments passed to the execution engine | -| `spec.executionEngineSpec.configOverrides` | no | Arbitrary YAML overrides passed to the test framework | +| `spec.executionEngine` | yes | Execution engine configuration. The single top-level key is the engine name (e.g. `forge`); its value is opaque engine-specific config passed through as-is. | | `spec.env` | no | Environment variables available to the execution engine (read from the FournosJob spec via K8s API) | | `spec.cluster` | \* | Pin to a specific cluster (Kueue ResourceFlavor). Since `exclusive` defaults to `true`, this also locks the cluster — set `exclusive: false` for shared access. | | `spec.hardware.gpuType` | \* | Short GPU model name — e.g. `a100`, `h200`. The operator prepends the `FOURNOS_GPU_RESOURCE_PREFIX` (default `fournos/gpu-`) automatically, so do **not** include the full resource path. | | `spec.hardware.gpuCount` | with gpuType | Number of GPUs (minimum 1) | | `spec.owner` | no | Team or individual that owns this job | | `spec.displayName` | no | Human-readable job name (defaults to `metadata.name`) | -| `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`) | +| `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`). The Pipeline must carry a `fournos.dev/resolve-image` annotation with the full image reference for the resolve Job. | | `spec.priority` | no | Kueue WorkloadPriorityClass name | | `spec.secretRefs` | no | Vault-synced K8s Secret names (prefixed with `vault-`) to mount into the pipeline. Populated by the execution engine during the Resolving phase. The operator validates each name in `FOURNOS_SECRETS_NAMESPACE`, copies the secrets into the operator namespace, and mounts them as a projected volume at `/var/run/secrets/fournos//`. | | `spec.exclusive` | no (default `true`) | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. Hardware is optional — when omitted the Workload only requests cluster-slot resources for locking. | @@ -314,7 +308,6 @@ All settings are read from environment variables with the `FOURNOS_` prefix: | `FOURNOS_GPU_RESOURCE_PREFIX` | `fournos/gpu-` | Resource name prefix for GPU types | | `FOURNOS_LOG_LEVEL` | `INFO` | Logging level | | `FOURNOS_GC_INTERVAL_SEC` | `300` | Resource GC interval (seconds) | -| `FOURNOS_RESOLVE_IMAGE_REGISTRY` | `image-registry.openshift-image-registry.svc:5000/{namespace}/` | Registry prefix for the resolve image (`{namespace}` is substituted at runtime). The image name itself comes from `spec.executionEngineSpec.resolveImage`. | | `FOURNOS_RESOLVE_DEADLINE_SEC` | `300` | Deadline for the resolve Job (seconds) | | `FOURNOS_RESOLVE_JOB_TEMPLATE` | `config/forge/resolve_job.yaml` | Path (relative to project root) to the Job YAML template for the resolve step. Override with `dev/mock-resolve/resolve_job.yaml` for local dev/CI. | | `FOURNOS_ARTIFACT_PVC_SIZE` | `1Gi` | Size of the per-PipelineRun PVC used for shared artifact storage across pipeline tasks | diff --git a/config/forge/resolve_job.yaml b/config/forge/resolve_job.yaml index 3e7d0f0..bae0635 100644 --- a/config/forge/resolve_job.yaml +++ b/config/forge/resolve_job.yaml @@ -65,9 +65,9 @@ spec: fi git show --quiet - FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngineSpec.project) + FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngine.forge.project) if [[ -z "$FORGE_PROJECT" || "$FORGE_PROJECT" == null ]]; then - echo "ERROR: invalid .spec.executionEngineSpec.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" + echo "ERROR: invalid .spec.executionEngine.forge.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" exit 1 fi diff --git a/config/forge/samples/job-full.yaml b/config/forge/samples/job-full.yaml index aae79ea..b202502 100644 --- a/config/forge/samples/job-full.yaml +++ b/config/forge/samples/job-full.yaml @@ -7,12 +7,11 @@ spec: displayName: forge-full-sample cluster: cluster-1 pipeline: forge-full - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: llm_d - args: - - cks + executionEngine: + forge: + project: llm_d + args: + - cks env: OCPCI_SUITE: regression OCPCI_VARIANT: nightly diff --git a/config/forge/samples/job-test-only.yaml b/config/forge/samples/job-test-only.yaml index eca7eab..4a1b267 100644 --- a/config/forge/samples/job-test-only.yaml +++ b/config/forge/samples/job-test-only.yaml @@ -7,12 +7,11 @@ spec: displayName: forge-sample cluster: cluster-1 pipeline: forge-test-only - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: llm_d - args: - - cks + executionEngine: + forge: + project: llm_d + args: + - cks env: OCPCI_SUITE: regression OCPCI_VARIANT: nightly diff --git a/config/forge/workflows/pipeline-full.yaml b/config/forge/workflows/pipeline-full.yaml index 77f8f2c..326f31d 100644 --- a/config/forge/workflows/pipeline-full.yaml +++ b/config/forge/workflows/pipeline-full.yaml @@ -3,6 +3,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: forge-full + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main spec: workspaces: - name: artifacts diff --git a/config/forge/workflows/pipeline-test-only.yaml b/config/forge/workflows/pipeline-test-only.yaml index 5c1411b..231f274 100644 --- a/config/forge/workflows/pipeline-test-only.yaml +++ b/config/forge/workflows/pipeline-test-only.yaml @@ -4,6 +4,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: forge-test-only + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main spec: workspaces: - name: artifacts diff --git a/config/forge/workflows/tasks.yaml b/config/forge/workflows/tasks.yaml index 59caa29..5953574 100644 --- a/config/forge/workflows/tasks.yaml +++ b/config/forge/workflows/tasks.yaml @@ -88,9 +88,9 @@ spec: fi git show --quiet - FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngineSpec.project) + FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngine.forge.project) if [[ -z "$FORGE_PROJECT" || "$FORGE_PROJECT" == null ]]; then - echo "ERROR: invalid .spec.executionEngineSpec.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" + echo "ERROR: invalid .spec.executionEngine.forge.project='$FORGE_PROJECT' in $FOURNOS_NAMESPACE/$FJOB_NAME" exit 1 fi diff --git a/config/fournos-validation/samples/test-connectivity-job.yaml b/config/fournos-validation/samples/test-connectivity-job.yaml index 6dd84e1..9bee0ed 100644 --- a/config/fournos-validation/samples/test-connectivity-job.yaml +++ b/config/fournos-validation/samples/test-connectivity-job.yaml @@ -10,8 +10,7 @@ spec: hardware: gpuType: a100 gpuCount: 1 - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: none - args: [] + executionEngine: + forge: + project: none + args: [] diff --git a/config/fournos-validation/workflows/pipeline-validate-only.yaml b/config/fournos-validation/workflows/pipeline-validate-only.yaml index 9e188b2..d1d3ed0 100644 --- a/config/fournos-validation/workflows/pipeline-validate-only.yaml +++ b/config/fournos-validation/workflows/pipeline-validate-only.yaml @@ -3,6 +3,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: fournos-validate-only + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main spec: params: - name: fjob-name diff --git a/dev/job-secret-demo.yaml b/dev/job-secret-demo.yaml index 8d404a9..116556c 100644 --- a/dev/job-secret-demo.yaml +++ b/dev/job-secret-demo.yaml @@ -7,9 +7,8 @@ spec: displayName: mock-secret-demo cluster: cluster-1 pipeline: fournos-run-only - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: secret-demo - args: - - demo + executionEngine: + forge: + project: secret-demo + args: + - demo diff --git a/dev/mock-pipelines/pipeline-full.yaml b/dev/mock-pipelines/pipeline-full.yaml index fd3d3cc..8c90d7a 100644 --- a/dev/mock-pipelines/pipeline-full.yaml +++ b/dev/mock-pipelines/pipeline-full.yaml @@ -3,6 +3,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: fournos-full + annotations: + fournos.dev/resolve-image: fournos-mock-resolve:dev spec: workspaces: - name: artifacts diff --git a/dev/mock-pipelines/pipeline-run-only.yaml b/dev/mock-pipelines/pipeline-run-only.yaml index b87433e..9174795 100644 --- a/dev/mock-pipelines/pipeline-run-only.yaml +++ b/dev/mock-pipelines/pipeline-run-only.yaml @@ -4,6 +4,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: fournos-run-only + annotations: + fournos.dev/resolve-image: fournos-mock-resolve:dev spec: workspaces: - name: artifacts diff --git a/dev/mock-pipelines/tasks.yaml b/dev/mock-pipelines/tasks.yaml index 3ab0570..805f989 100644 --- a/dev/mock-pipelines/tasks.yaml +++ b/dev/mock-pipelines/tasks.yaml @@ -87,6 +87,8 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: fournos-run-only + annotations: + fournos.dev/resolve-image: fournos-mock-resolve:dev spec: workspaces: - name: artifacts diff --git a/dev/sample-job.yaml b/dev/sample-job.yaml index be58f9a..f729b44 100644 --- a/dev/sample-job.yaml +++ b/dev/sample-job.yaml @@ -9,19 +9,18 @@ spec: hardware: gpuType: a100 gpuCount: 2 - executionEngine: forge - executionEngineSpec: - resolveImage: forge-core:main - project: llm-d - args: - - cks - - something-else - configOverrides: - vllm: - version: 0.15.1 - logging: - enabled: true - level: debug + executionEngine: + forge: + project: llm-d + args: + - cks + - something-else + configOverrides: + vllm: + version: 0.15.1 + logging: + enabled: true + level: debug env: OCPCI_SUITE: regression OCPCI_VARIANT: nightly diff --git a/fournos/core/tekton.py b/fournos/core/tekton.py index e96052e..67f3ec7 100644 --- a/fournos/core/tekton.py +++ b/fournos/core/tekton.py @@ -12,8 +12,11 @@ TEKTON_GROUP = "tekton.dev" TEKTON_VERSION = "v1" +TEKTON_PIPELINE_PLURAL = "pipelines" TEKTON_PIPELINE_RUN_PLURAL = "pipelineruns" +ANNOTATION_RESOLVE_IMAGE = "fournos.dev/resolve-image" + def _build_secrets_volume(resolved: list[ResolvedSecret]) -> dict: """Build a single projected volume combining all per-job secret copies. @@ -118,6 +121,15 @@ def create_pipeline_run( logger.info("Created PipelineRun %s", name) return result + def get_pipeline(self, name: str) -> dict: + return self._k8s.get_namespaced_custom_object( + group=TEKTON_GROUP, + version=TEKTON_VERSION, + namespace=settings.namespace, + plural=TEKTON_PIPELINE_PLURAL, + name=name, + ) + def get_pipeline_run(self, name: str) -> dict: return self._k8s.get_namespaced_custom_object( group=TEKTON_GROUP, diff --git a/fournos/handlers/resolving.py b/fournos/handlers/resolving.py index 2c0eba5..a24b854 100644 --- a/fournos/handlers/resolving.py +++ b/fournos/handlers/resolving.py @@ -13,7 +13,7 @@ from fournos.core.constants import Phase from fournos.core.resolve import ResolveClient -from fournos.settings import settings +from fournos.core.tekton import ANNOTATION_RESOLVE_IMAGE from fournos.state import ctx from .status import ( @@ -49,6 +49,9 @@ def _resolve_failed(patch, conditions, name, message, *, reason, cond_message=No def _ensure_resolve_job(spec, name, conditions, patch, body): """Create the resolve Job if it doesn't exist yet. + The resolve image is read from the ``fournos.dev/resolve-image`` + annotation on the Tekton Pipeline referenced by ``spec.pipeline``. + Returns the existing Job dict, or None if the Job was just created (or a 409 race was hit) — the caller should return and wait for the next reconcile tick. Returns ``False`` on fatal creation failure @@ -58,14 +61,31 @@ def _ensure_resolve_job(spec, name, conditions, patch, body): if job is not None: return job - engine_spec = spec["executionEngineSpec"] - registry = engine_spec.get( - "resolveImageRegistry", - settings.resolve_image_registry, - ).replace("{namespace}", settings.namespace) - if registry and not registry.endswith("/"): - registry += "/" - resolve_image = registry + engine_spec["resolveImage"] + pipeline_name = spec.get("pipeline", "fournos-full") + try: + pipeline = ctx.tekton.get_pipeline(pipeline_name) + except client.exceptions.ApiException as exc: + _resolve_failed( + patch, + conditions, + name, + f"Failed to fetch Pipeline '{pipeline_name}': {exc.reason}", + reason="PipelineNotFound", + ) + return False + + annotations = pipeline.get("metadata", {}).get("annotations") or {} + resolve_image = annotations.get(ANNOTATION_RESOLVE_IMAGE) + if not resolve_image: + _resolve_failed( + patch, + conditions, + name, + f"Pipeline '{pipeline_name}' is missing the " + f"'{ANNOTATION_RESOLVE_IMAGE}' annotation", + reason="MissingResolveImage", + ) + return False try: ctx.resolve.create_job( diff --git a/fournos/settings.py b/fournos/settings.py index 34323bf..a258d27 100644 --- a/fournos/settings.py +++ b/fournos/settings.py @@ -17,9 +17,6 @@ class Settings(BaseSettings): gpu_resource_prefix: str = "fournos/gpu-" gc_interval_sec: float = Field(default=300.0, gt=0) log_level: str = "INFO" - resolve_image_registry: str = ( - "image-registry.openshift-image-registry.svc:5000/{namespace}/" - ) resolve_deadline_sec: int = Field(default=300, gt=0) resolve_job_template: str = "config/forge/resolve_job.yaml" artifact_pvc_size: str = "10Gi" diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 2248e22..8923046 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -52,7 +52,6 @@ spec: type: object required: - executionEngine - - executionEngineSpec properties: owner: type: string @@ -84,36 +83,12 @@ spec: type: integer minimum: 1 executionEngine: - type: string - description: >- - Execution engine to use for this job (e.g. "forge"). - executionEngineSpec: type: object - required: - - resolveImage - - project - - args - properties: - resolveImage: - type: string - description: >- - Short image name for the resolve Job (e.g. "forge-core:main"). - The operator prepends the registry prefix automatically. - resolveImageRegistry: - type: string - description: >- - Registry prefix for the resolve image. Overrides the - operator default when set. Supports {namespace} placeholder. - project: - type: string - args: - type: array - items: - type: string - configOverrides: - type: object - description: Overrides passed to the test framework as YAML. - x-kubernetes-preserve-unknown-fields: true + description: >- + Execution engine configuration. The top-level key is the + engine name (e.g. "forge") and its value is the + engine-specific config. + x-kubernetes-preserve-unknown-fields: true secretRefs: type: array description: >- diff --git a/manifests/rbac/role_fournos.yaml b/manifests/rbac/role_fournos.yaml index fda7952..13784f5 100644 --- a/manifests/rbac/role_fournos.yaml +++ b/manifests/rbac/role_fournos.yaml @@ -10,6 +10,9 @@ rules: - apiGroups: ["kueue.x-k8s.io"] resources: ["workloads"] verbs: ["create", "get", "list", "watch", "patch", "delete"] + - apiGroups: ["tekton.dev"] + resources: ["pipelines"] + verbs: ["get"] - apiGroups: ["tekton.dev"] resources: ["pipelineruns"] verbs: ["create", "get", "list", "watch", "patch", "delete"] diff --git a/tests/test_exclusive.py b/tests/test_exclusive.py index e0df9d0..13bd3ae 100644 --- a/tests/test_exclusive.py +++ b/tests/test_exclusive.py @@ -84,11 +84,11 @@ def test_exclusive_happy_path(k8s): { "cluster": "cluster-2", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -113,11 +113,11 @@ def test_exclusive_workload_requests_all_slots(k8s): { "cluster": "cluster-2", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -142,11 +142,11 @@ def test_normal_workload_requests_one_slot(k8s): { "exclusive": False, "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -169,11 +169,11 @@ def test_exclusive_blocks_cluster_pinned_job(k8s): { "cluster": "cluster-2", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -185,11 +185,11 @@ def test_exclusive_blocks_cluster_pinned_job(k8s): { "exclusive": False, "cluster": "cluster-2", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -236,11 +236,11 @@ def test_exclusive_steers_hardware_only_job(k8s): { "cluster": "cluster-1", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -252,11 +252,11 @@ def test_exclusive_steers_hardware_only_job(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -284,11 +284,11 @@ def test_exclusive_without_cluster_fails(k8s): { "exclusive": True, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -316,11 +316,11 @@ def test_exclusive_waits_for_cluster_to_clear(k8s): { "exclusive": False, "cluster": "cluster-2", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -332,11 +332,11 @@ def test_exclusive_waits_for_cluster_to_clear(k8s): { "cluster": "cluster-2", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -377,11 +377,11 @@ def test_lock_released_on_completion(k8s): { "cluster": "cluster-1", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -393,11 +393,11 @@ def test_lock_released_on_completion(k8s): { "exclusive": False, "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -448,11 +448,11 @@ def test_exclusive_without_hardware(k8s): { "cluster": "cluster-2", "exclusive": True, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py index 5a2bb78..84b9f45 100644 --- a/tests/test_lifecycle.py +++ b/tests/test_lifecycle.py @@ -25,11 +25,11 @@ def test_workload_cleaned_after_completion(k8s): "test-wl-cleanup", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -68,11 +68,11 @@ def test_delete_cleans_up_resources(k8s): "test-delete", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -106,11 +106,11 @@ def test_list_multiple_jobs(k8s): "test-list-a", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -119,11 +119,11 @@ def test_list_multiple_jobs(k8s): "test-list-b", { "cluster": "cluster-2", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -144,11 +144,11 @@ def test_filter_jobs_by_phase(k8s): "test-filter-ok", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -158,11 +158,11 @@ def test_filter_jobs_by_phase(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_resolving.py b/tests/test_resolving.py index 8d7d1e2..d842008 100644 --- a/tests/test_resolving.py +++ b/tests/test_resolving.py @@ -42,11 +42,11 @@ def test_happy_path_with_hardware(k8s): { "cluster": "cluster-3", "hardware": {"gpuType": "h200", "gpuCount": 4}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -99,11 +99,11 @@ def test_happy_path_without_hardware(k8s): "test-resolve-nohw", { "exclusive": False, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -146,11 +146,11 @@ def test_cluster_pin_without_hardware(k8s): "test-resolve-pin", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -189,11 +189,11 @@ def test_shutdown_during_resolving(k8s): { "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -227,11 +227,11 @@ def test_delete_during_resolving_cleans_up(k8s): { "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -265,11 +265,11 @@ def test_unknown_gpu_type(k8s): { "exclusive": False, "hardware": {"gpuType": "acbd1234", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -307,11 +307,11 @@ def test_resolve_job_failure(k8s): "test-resolve-fail", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -358,11 +358,11 @@ def test_nonexclusive_cluster_without_hardware_fails(k8s): { "exclusive": False, "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -398,11 +398,11 @@ def test_resolve_empty_hw(k8s): "test-resolve-noconfig", { "exclusive": False, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_scheduling.py b/tests/test_scheduling.py index 2c76103..b862bd1 100644 --- a/tests/test_scheduling.py +++ b/tests/test_scheduling.py @@ -25,11 +25,11 @@ def test_cluster_pinned(k8s): "test-cluster", { "cluster": "cluster-2", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -98,11 +98,11 @@ def test_hardware_request(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["llama3", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["llama3", "internal-test"], + } }, "priority": "nightly", }, @@ -131,11 +131,11 @@ def test_cluster_and_hardware(k8s): { "cluster": "cluster-4", "hardware": {"gpuType": "h200", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["llama3", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["llama3", "internal-test"], + } }, }, ) @@ -186,11 +186,11 @@ def test_shared_cluster_with_hardware(k8s): "exclusive": False, "cluster": "cluster-3", "hardware": {"gpuType": "h200", "gpuCount": 4}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -230,11 +230,11 @@ def test_alternative_pipeline_selection(k8s): { "pipeline": "fournos-run-only", "cluster": "cluster-2", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -262,11 +262,11 @@ def test_inadmissible_stays_pending(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -319,11 +319,11 @@ def test_cluster_without_required_gpu_stays_pending(k8s): { "cluster": "cluster-3", "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_secret_refs.py b/tests/test_secret_refs.py index 5fb5c65..a52fa89 100644 --- a/tests/test_secret_refs.py +++ b/tests/test_secret_refs.py @@ -112,11 +112,11 @@ def test_vault_sync_then_fjob(k8s, core_v1): "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, "secretRefs": [VAULT_ENTRY], - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -181,11 +181,11 @@ def test_missing_secret_ref_fails(k8s): "cluster": "cluster-1", "hardware": {"gpuType": "a100", "gpuCount": 2}, "secretRefs": ["nonexistent-vault-entry"], - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_shutdown.py b/tests/test_shutdown.py index 6d20ef7..9c4bfeb 100644 --- a/tests/test_shutdown.py +++ b/tests/test_shutdown.py @@ -43,11 +43,11 @@ def test_stop_pending_job(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 100}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -83,11 +83,11 @@ def test_stop_running_job(k8s): "test-stop-running", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -161,11 +161,11 @@ def test_stop_at_creation(k8s): "test-stop-create", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, "shutdown": "Stop", }, @@ -193,11 +193,11 @@ def test_stop_completed_job_is_noop(k8s): "test-stop-done", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -238,11 +238,11 @@ def test_terminate_running_job(k8s): "test-term-running", { "cluster": "cluster-1", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 4ec7daa..cdf6d00 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -27,11 +27,11 @@ def test_unknown_cluster(k8s): "test-unknown", { "cluster": "no-such-cluster", - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -59,11 +59,11 @@ def test_admitted_without_flavor(k8s): { "exclusive": False, "hardware": {"gpuType": "a100", "gpuCount": 999}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) @@ -138,11 +138,11 @@ def test_implicit_exclusive_without_cluster_fails(k8s): "test-implicit-excl", { "hardware": {"gpuType": "a100", "gpuCount": 2}, - "executionEngine": "forge", - "executionEngineSpec": { - "resolveImage": "fournos-mock-resolve:dev", - "project": "testproj/llmd", - "args": ["cks", "internal-test"], + "executionEngine": { + "forge": { + "project": "testproj/llmd", + "args": ["cks", "internal-test"], + } }, }, ) From 667c9774f8c464b748ecf0cd2e762a6cc6897707 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 4 May 2026 17:08:05 +0200 Subject: [PATCH 5/7] tests: forge: deploy/orchestration/deploy: correctly enable non-strict vault validation --- tests/forge/deploy/orchestration/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/forge/deploy/orchestration/deploy.py b/tests/forge/deploy/orchestration/deploy.py index fea5398..c8775d4 100644 --- a/tests/forge/deploy/orchestration/deploy.py +++ b/tests/forge/deploy/orchestration/deploy.py @@ -220,7 +220,8 @@ def _setup_kubeconfig_from_vault(): # Check if vault configuration exists vaults = config.project.get_config("vaults", print=False) - vault.init(vaults, strict=False) + vault.disable_strict_validation() + vault.init(vaults) # Try to get kubeconfig vault configuration kubeconfig_vault_name = config.project.get_config( From 8a0b4a3f3457d1777d2f39d0b2226652aeb5ca46 Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 17:57:22 +0200 Subject: [PATCH 6/7] Fix log message Signed-off-by: avasilev --- fournos/handlers/lifecycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fournos/handlers/lifecycle.py b/fournos/handlers/lifecycle.py index 19eebbd..ee19b7b 100644 --- a/fournos/handlers/lifecycle.py +++ b/fournos/handlers/lifecycle.py @@ -72,7 +72,7 @@ def on_create(spec, name, namespace, status, patch, body): patch.meta.setdefault("labels", {})[LABEL_EXCLUSIVE_CLUSTER] = cluster patch.status["phase"] = Phase.RESOLVING - patch.status["message"] = "Resolving job requirements via Forge" + patch.status["message"] = "Resolving job requirements" logger.info("Job %s: phase=Resolving", name) From 4b7abd0ac0e9a8accedd8db55cd5a370d75d7cae Mon Sep 17 00:00:00 2001 From: avasilev Date: Mon, 4 May 2026 18:07:03 +0200 Subject: [PATCH 7/7] Fix namespace substitution Signed-off-by: avasilev --- config/forge/workflows/pipeline-full.yaml | 2 +- config/forge/workflows/pipeline-test-only.yaml | 2 +- config/fournos-validation/workflows/pipeline-validate-only.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/forge/workflows/pipeline-full.yaml b/config/forge/workflows/pipeline-full.yaml index 326f31d..09dd3d0 100644 --- a/config/forge/workflows/pipeline-full.yaml +++ b/config/forge/workflows/pipeline-full.yaml @@ -4,7 +4,7 @@ kind: Pipeline metadata: name: forge-full annotations: - fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main spec: workspaces: - name: artifacts diff --git a/config/forge/workflows/pipeline-test-only.yaml b/config/forge/workflows/pipeline-test-only.yaml index 231f274..3593031 100644 --- a/config/forge/workflows/pipeline-test-only.yaml +++ b/config/forge/workflows/pipeline-test-only.yaml @@ -5,7 +5,7 @@ kind: Pipeline metadata: name: forge-test-only annotations: - fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main spec: workspaces: - name: artifacts diff --git a/config/fournos-validation/workflows/pipeline-validate-only.yaml b/config/fournos-validation/workflows/pipeline-validate-only.yaml index d1d3ed0..303f235 100644 --- a/config/fournos-validation/workflows/pipeline-validate-only.yaml +++ b/config/fournos-validation/workflows/pipeline-validate-only.yaml @@ -4,7 +4,7 @@ kind: Pipeline metadata: name: fournos-validate-only annotations: - fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/$NAMESPACE/forge-core:main + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main spec: params: - name: fjob-name