Refine scheduling for exclusive, cluster and hw spec configs

README.md

@@ -78,22 +78,26 @@ oc delete FournosJob -n $FOURNOS_NAMESPACE <name>      # cleanup
 | `spec.forge.args` | yes | List of arguments passed to FORGE |
 | `spec.forge.configOverrides` | no | Arbitrary YAML overrides passed to the test framework |
 | `spec.env` | no | Environment variables passed to the pipeline as a `KEY=VALUE` env file |
-| `spec.cluster` | \* | Pin to a specific cluster (Kueue ResourceFlavor) |
+| `spec.cluster` | \* | Pin to a specific cluster (Kueue ResourceFlavor). Since `exclusive` defaults to `true`, this also locks the cluster — set `exclusive: false` for shared access. |
 | `spec.hardware.gpuType` | \* | Short GPU model name — e.g. `a100`, `h200`. The operator prepends the `FOURNOS_GPU_RESOURCE_PREFIX` (default `fournos/gpu-`) automatically, so do **not** include the full resource path. |
 | `spec.hardware.gpuCount` | with gpuType | Number of GPUs (minimum 1) |
 | `spec.owner` | no | Team or individual that owns this job |
 | `spec.displayName` | no | Human-readable job name (defaults to `metadata.name`) |
 | `spec.pipeline` | no | Tekton Pipeline name (default: `fournos-full`) |
 | `spec.priority` | no | Kueue WorkloadPriorityClass name |
 | `spec.secretRefs` | no | Vault-synced K8s Secret names (prefixed with `vault-`) to mount into the pipeline. Populated by Forge during the Resolving phase. The operator validates each name in `FOURNOS_SECRETS_NAMESPACE`, copies the secrets into the operator namespace, and mounts them as a projected volume at `/var/run/secrets/fournos/<entry-name>/`. |
-| `spec.exclusive` | no | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. |
+| `spec.exclusive` | no (default `true`) | If `true`, locks the target cluster so no other FournosJob can run there. Requires `spec.cluster`. Hardware is optional — when omitted the Workload only requests cluster-slot resources for locking. |
 | `spec.shutdown` | no | Shutdown action: `Stop` cancels gracefully (Tekton `CancelledRunFinally` — runs `finally` tasks); `Terminate` cancels immediately (Tekton `Cancelled` — skips `finally` tasks). Both wait for the PipelineRun to finish before releasing Kueue quota. |
 
-\* `spec.cluster` and `spec.hardware` are both optional. Every job passes
-through the Resolving phase where Forge populates `spec.hardware` (if not
-already set) and `spec.secretRefs` directly on the FournosJob.
-`spec.cluster` can be set alongside `spec.hardware` to pin a hardware request
-to a specific cluster.
+\* `spec.hardware` is required unless the job uses exclusive cluster locking
+(`exclusive: true` + `cluster`), in which case it may be omitted — the
+Workload only needs cluster-slot resources. Every job passes through the
+Resolving phase where Forge populates `spec.hardware` (if not already set)
+and `spec.secretRefs` directly on the FournosJob. Since `exclusive` defaults
+to `true`, any job with `spec.cluster` locks the cluster exclusively —
+including jobs that also specify `spec.hardware`. Set `exclusive: false` for
+shared access (hardware is then required). Jobs without `spec.cluster` must
+set `exclusive: false`.
 
 ### Status
 

fournos/core/kueue.py

@@ -36,7 +36,7 @@ def create_workload(
         gpu_type: str | None = None,
         gpu_count: int = 0,
         cluster: str | None = None,
-        exclusive: bool = False,
+        exclusive: bool = True,
         priority: str | None = None,
         owner_ref: dict | None = None,
     ) -> dict:

fournos/handlers/lifecycle.py

@@ -48,7 +48,7 @@ def on_create(spec, name, namespace, status, patch, body):
         return
 
     cluster = spec.get("cluster")
-    exclusive = spec.get("exclusive", False)
+    exclusive = spec["exclusive"]
 
     if exclusive and not cluster:
         patch.status["phase"] = Phase.FAILED
@@ -173,7 +173,7 @@ def reconcile_pending(spec, name, status, patch, body):
         new_msg, log_msg = _pending_status(
             wl_message,
             cluster,
-            spec.get("exclusive", False),
+            spec["exclusive"],
             locker,
         )
         if status.get("message") != new_msg:

fournos/handlers/resolving.py

@@ -119,30 +119,52 @@ def _check_job_finished(job, name, conditions, patch):
     return True
 
 
-def _resolve_hardware(spec, name, conditions, patch):
+def _resolve_hardware(
+    spec,
+    name,
+    conditions,
+    patch,
+) -> tuple[str | None, int | None]:
     """Determine and validate GPU requirements from the FournosJob spec.
 
     Forge populates ``spec.hardware`` when absent.  The GPU type is
-    always validated against Kueue.
+    validated against Kueue unless this is an exclusive cluster-lock
+    job with no hardware requirements.
 
-    Returns ``(gpu_type, gpu_count)`` on success, or ``None`` if
-    validation failed (patch already set to Failed).
+    Exclusive jobs pinned to a cluster may omit hardware — the Workload
+    only needs cluster-slot resources for locking.
+
+    Returns:
+        (gpu_type, gpu_count): On success.  ``gpu_type`` is ``None``
+            and ``gpu_count`` is ``0`` for exclusive-only jobs.
+        (None, None): Validation failed; the patch has already been set
+            to ``Failed`` with a descriptive message.
     """
     hardware = spec.get("hardware") or {}
     gpu_type = hardware.get("gpuType")
     gpu_count = hardware.get("gpuCount", 0)
 
+    exclusive_lock = spec["exclusive"] and spec.get("cluster")
+
     if not gpu_type or not gpu_count:
-        _resolve_failed(
-            patch,
-            conditions,
+        if not exclusive_lock:
+            _resolve_failed(
+                patch,
+                conditions,
+                name,
+                "No hardware requirements: spec.hardware not populated "
+                "after Forge resolution",
+                reason="NoHardware",
+                cond_message="No hardware requirements found",
+            )
+            return None, None
+        logger.warning(
+            "Job %s: exclusive cluster lock without hardware — "
+            "Workload will only request cluster-slot resources "
+            "(Forge may not have populated spec.hardware)",
             name,
-            "No hardware requirements: spec.hardware not populated "
-            "after Forge resolution",
-            reason="NoHardware",
-            cond_message="No hardware requirements found",
         )
-        return None
+        return None, 0
 
     try:
         known_gpu_types = ctx.kueue.list_gpu_types()
@@ -155,7 +177,7 @@ def _resolve_hardware(spec, name, conditions, patch):
             reason="InvalidGPUType",
             cond_message=f"Kueue API error: {exc.reason}",
         )
-        return None
+        return None, None
     if not known_gpu_types:
         _resolve_failed(
             patch,
@@ -166,7 +188,7 @@ def _resolve_hardware(spec, name, conditions, patch):
             cond_message="No GPU types found in any ClusterQueue",
         )
         logger.error("Job %s: no GPU types found in any ClusterQueue", name)
-        return None
+        return None, None
     if gpu_type not in known_gpu_types:
         _resolve_failed(
             patch,
@@ -177,7 +199,7 @@ def _resolve_hardware(spec, name, conditions, patch):
             reason="InvalidGPUType",
             cond_message=f"GPU type '{gpu_type}' not available",
         )
-        return None
+        return None, None
 
     return gpu_type, gpu_count
 
@@ -218,7 +240,7 @@ def _create_workload_and_transition(
             gpu_type=gpu_type,
             gpu_count=gpu_count,
             cluster=spec.get("cluster"),
-            exclusive=spec.get("exclusive", False),
+            exclusive=spec["exclusive"],
             priority=spec.get("priority"),
             owner_ref=owner_ref(body),
         )
@@ -277,14 +299,12 @@ def reconcile_resolving(spec, name, status, patch, body):
     if not _check_job_finished(job, name, conditions, patch):
         return
 
-    hw = _resolve_hardware(spec, name, conditions, patch)
-    if hw is None:
+    gpu_type, gpu_count = _resolve_hardware(spec, name, conditions, patch)
+    if gpu_count is None:
         return
 
     if not _validate_secret_refs(spec, name, conditions, patch):
         return
-
-    gpu_type, gpu_count = hw
     _create_workload_and_transition(
         spec, name, conditions, patch, body, gpu_type, gpu_count
     )

fournos/handlers/status.py

@@ -70,7 +70,7 @@ def create_workload_for_job(spec, name, body):
         gpu_type=hardware.get("gpuType") if hardware else None,
         gpu_count=hardware.get("gpuCount", 0) if hardware else 0,
         cluster=spec.get("cluster"),
-        exclusive=spec.get("exclusive", False),
+        exclusive=spec["exclusive"],
         priority=spec.get("priority"),
         owner_ref=owner_ref(body),
     )

manifests/crd.yaml

@@ -117,12 +117,14 @@ spec:
                     type: string
                 exclusive:
                   type: boolean
-                  default: false
+                  default: true
                   description: >-
                     Lock the target cluster for exclusive use by this job.
                     When true, the job waits until the cluster has no other active
                     jobs and then prevents any new job from being scheduled there.
-                    Requires 'cluster' to be set.
+                    Requires 'cluster' to be set.  Hardware is optional — when
+                    omitted the Workload only requests cluster-slot resources
+                    for locking.
                 priority:
                   type: string
                 shutdown:

tests/test_exclusive.py

@@ -13,8 +13,11 @@
 from tests.conftest import (
     NAMESPACE,
     create_job,
+    create_noop_resolve_job,
     get_job,
     get_workload_cluster_slots,
+    get_workload_gpu_request,
+    get_workload_node_selector,
     job_status_summary,
     poll_phase,
     workload_exists,
@@ -127,6 +130,7 @@ def test_normal_workload_requests_one_slot(k8s):
         k8s,
         "test-normal-slots",
         {
+            "exclusive": False,
             "cluster": "cluster-1",
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -159,6 +163,7 @@ def test_exclusive_blocks_cluster_pinned_job(k8s):
         k8s,
         "test-blocked-pin",
         {
+            "exclusive": False,
             "cluster": "cluster-2",
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -215,6 +220,7 @@ def test_exclusive_steers_hardware_only_job(k8s):
         k8s,
         "test-hw-avoid",
         {
+            "exclusive": False,
             "hardware": {"gpuType": "a100", "gpuCount": 2},
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -268,6 +274,7 @@ def test_exclusive_waits_for_cluster_to_clear(k8s):
         k8s,
         "test-occupant",
         {
+            "exclusive": False,
             "cluster": "cluster-2",
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -329,6 +336,7 @@ def test_lock_released_on_completion(k8s):
         k8s,
         "test-waiting",
         {
+            "exclusive": False,
             "cluster": "cluster-1",
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -363,3 +371,57 @@ def test_lock_released_on_completion(k8s):
         timeout=60,
     )
     assert phase == Phase.SUCCEEDED, job_status_summary(k8s, "test-waiting")
+
+
+def test_exclusive_without_hardware(k8s):
+    """Exclusive + cluster without hardware: locks cluster using only cluster-slot resources.
+
+    A noop resolve Job prevents Forge from populating hardware.  The
+    Workload should carry 100 cluster-slots and a nodeSelector but no
+    GPU resource requests.
+    """
+    create_noop_resolve_job("test-excl-nohw")
+
+    create_job(
+        k8s,
+        "test-excl-nohw",
+        {
+            "cluster": "cluster-2",
+            "exclusive": True,
+            "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
+        },
+    )
+
+    poll_phase(
+        k8s,
+        "test-excl-nohw",
+        terminal={Phase.PENDING, Phase.ADMITTED, Phase.RUNNING},
+        timeout=45,
+    )
+
+    slots = get_workload_cluster_slots("test-excl-nohw")
+    assert slots == MAX_CLUSTER_SLOTS, (
+        f"Exclusive Workload should request {MAX_CLUSTER_SLOTS} slots, got {slots}"
+    )
+
+    ns = get_workload_node_selector("test-excl-nohw")
+    assert ns == {"fournos.dev/cluster": "cluster-2"}, (
+        f"Workload nodeSelector should pin to cluster-2, got {ns}"
+    )
+
+    a100 = get_workload_gpu_request("test-excl-nohw", "a100")
+    h200 = get_workload_gpu_request("test-excl-nohw", "h200")
+    assert a100 == 0 and h200 == 0, (
+        f"Workload should have no GPU requests, got a100={a100}, h200={h200}"
+    )
+
+    phase = poll_phase(
+        k8s,
+        "test-excl-nohw",
+        terminal={Phase.SUCCEEDED, Phase.FAILED},
+        timeout=90,
+    )
+    assert phase == Phase.SUCCEEDED, job_status_summary(k8s, "test-excl-nohw")
+
+    job = get_job(k8s, "test-excl-nohw")
+    assert job["status"]["cluster"] == "cluster-2"

tests/test_lifecycle.py

@@ -131,6 +131,7 @@ def test_filter_jobs_by_phase(k8s):
         k8s,
         "test-filter-stuck",
         {
+            "exclusive": False,
             "hardware": {"gpuType": "a100", "gpuCount": 100},
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },

tests/test_resolving.py

@@ -93,6 +93,7 @@ def test_happy_path_without_hardware(k8s):
         k8s,
         "test-resolve-nohw",
         {
+            "exclusive": False,
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
     )
@@ -237,6 +238,7 @@ def test_unknown_gpu_type(k8s):
         k8s,
         "test-bad-gpu",
         {
+            "exclusive": False,
             "hardware": {"gpuType": "acbd1234", "gpuCount": 2},
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
@@ -306,6 +308,42 @@ def test_resolve_job_failure(k8s):
     )
 
 
+def test_nonexclusive_cluster_without_hardware_fails(k8s):
+    """Non-exclusive + cluster + no hardware → Failed (hardware required).
+
+    A noop resolve Job prevents Forge from populating hardware.  Since
+    the job is non-exclusive, the missing hardware is not allowed (only
+    exclusive+cluster jobs may omit it).
+    """
+    create_noop_resolve_job("test-nex-nohw")
+
+    create_job(
+        k8s,
+        "test-nex-nohw",
+        {
+            "exclusive": False,
+            "cluster": "cluster-1",
+            "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
+        },
+    )
+
+    phase = poll_phase(
+        k8s,
+        "test-nex-nohw",
+        terminal={Phase.FAILED},
+        message_substring="No hardware requirements",
+        timeout=45,
+    )
+    assert phase == Phase.FAILED, job_status_summary(k8s, "test-nex-nohw")
+
+    conditions = {
+        c["type"]: c
+        for c in get_job(k8s, "test-nex-nohw")["status"].get("conditions", [])
+    }
+    assert conditions["Resolved"]["status"] == "False"
+    assert conditions["Resolved"]["reason"] == "NoHardware"
+
+
 def test_resolve_empty_hw(k8s):
     """Resolve Job succeeds but doesn't populate spec.hardware -> Failed.
 
@@ -319,6 +357,7 @@ def test_resolve_empty_hw(k8s):
         k8s,
         "test-resolve-noconfig",
         {
+            "exclusive": False,
             "forge": {"project": "testproj/llmd", "args": ["cks", "internal-test"]},
         },
     )