diff --git a/simpler_setup/tools/swimlane_converter.py b/simpler_setup/tools/swimlane_converter.py index 0dd802204..527486b7d 100644 --- a/simpler_setup/tools/swimlane_converter.py +++ b/simpler_setup/tools/swimlane_converter.py @@ -1401,7 +1401,7 @@ def generate_chrome_trace_json( # noqa: PLR0912, PLR0913, PLR0915 "complete": "good", # green "dispatch": "terrible", # red "release": "olive", # deferred-release drain (on_task_release work) - "wire": "thread_state_running", # drain_wiring_queue pass + "wire": "thread_state_running", # legacy scheduler wire phase "dummy": "grey", # dummy_drain pass (Resolve nests inside) "early_dispatch": "rail_animation", # speculative early-dispatch staging # Inner phase — nests inside Complete or Dummy via time containment diff --git a/src/a2a3/platform/include/common/l2_swimlane_profiling.h b/src/a2a3/platform/include/common/l2_swimlane_profiling.h index b7a4fc354..8014790c6 100644 --- a/src/a2a3/platform/include/common/l2_swimlane_profiling.h +++ b/src/a2a3/platform/include/common/l2_swimlane_profiling.h @@ -504,8 +504,7 @@ enum class L2SwimlaneSchedPhaseKind : uint32_t { // tasks_processed = subtasks published this iter. Release = 2, // Deferred-release drain (on_task_release work). // tasks_processed = slots released this iter. - Wire = 3, // drain_wiring_queue: pop wired tasks into ready queues. - // tasks_processed = wired count. + Wire = 3, // Legacy scheduler wiring phase, kept for old traces. Dummy = 4, // dummy_drain outer bar: covers handling of all dummies // popped this iter. tasks_processed = dummy_got count. EarlyDispatch = 5, // try_speculative_early_dispatch: speculative pre-staging diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index dbbfb5cd0..1f97121a8 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -135,12 +135,11 @@ struct RingSchedState { int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS - // Cache Line 1+: Thread 0 only (wiring dep_pool, cache-isolated) + // Cache Line 1+: Orch-side wiring dep_pool, cache-isolated alignas(64) PTO2DepListPool dep_pool; }; RingSchedState ring_sched_states[PTO2_MAX_RING_DEPTH]; -PTO2SpscQueue wiring_queue; // global SPSC queue: orchestrator pushes, scheduler thread 0 drains ``` `slot_states`, `task_window_size`, and `task_window_mask` are no longer duplicated — callers access them via `ring->get_slot_state_by_*()` and other ring header accessors. The ring pointer shares cache line 0 with `last_task_alive` and `advance_lock`. @@ -198,10 +197,10 @@ Dependency edges use `PTO2TaskSlotState*` pointers, which naturally span rings: ### 5.3 DepPool Reclamation -DepPool is exclusively managed by scheduler thread 0 (allocation during wiring, reclamation during watermark advancement): +DepPool entries are allocated by the orchestrator during Orch-side wiring and reclaimed during watermark advancement: ```text -// Called by scheduler thread 0 during wiring_queue drain: +// Called during ring watermark advancement: dep_pool_reclaim(ring_id): la = ring->fc.last_task_alive newest_consumed = la - 1 diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index bcc50f25d..2442f4429 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -418,7 +418,7 @@ Key members: - `rings[PTO2_MAX_RING_DEPTH]`: per-ring `PTO2RingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). - `tensor_map`, `tensor_pool`: dependency tracking - `scope_tasks[]`, `scope_begins[]`, `scope_stack_top`: scope nesting stack (flat buffer partitioned by level) -- `scheduler`: pointer to scheduler state (for wiring queue and ready queue access) +- `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) - `gm_heap_base`, `gm_heap_size`: GM heap for output buffers ### 7.2 Task Submission Flow (`PTO2OrchestratorState::submit_task`) @@ -430,17 +430,15 @@ Key members: | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | -| 5 | **Record fanin metadata**: store producer pointers in `payload->fanin_inline_slot_states[]` (+ spill pool if >64); increment each producer's `fanout_count` (no lock needed — single writer). This step runs **before** `payload.init()`. | -| 6 | **Push to wiring queue**: push to global `PTO2SpscQueue`; scheduler thread 0 asynchronously wires fanout edges (lock + dep_pool + early_finished check + ready push) | +| 5 | **Record fanin metadata**: store producer pointers in `payload->fanin_inline_slot_states[]` (+ spill pool if >64); claim each live producer by incrementing `fanout_count` under that producer's `fanout_lock`. This step runs **before** `payload.init()`. | +| 6 | **Orch-side wiring / ready publish**: the orchestrator wires live fanout edges into the per-ring dep_pool; zero-fanin and already-completed fanin tasks publish directly to ready queues | -> **Note**: Fanout wiring (Steps 4–7 in earlier versions) has been moved from the -> orchestrator submit hot path to the scheduler's global `wiring_queue` (SPSC). This reduces the -> orchestrator's shared L2 cache / memory bus pressure, as the orchestrator no longer -> acquires `fanout_lock` or allocates from `dep_pool` during submission. +> **Note**: Fanout wiring is now completed before publish in the orchestrator submit path. +> Scheduler threads consume ready queues directly. -### 7.3 Deferred Fanout Wiring (Scheduler Wiring Queue) +### 7.3 Orch-Side Fanout Wiring -The orchestrator pushes each submitted task to the global `scheduler->wiring_queue` (a wait-free SPSC queue). Scheduler thread 0 drains this queue in batches, deferring if the queue holds fewer than a full batch of items to reduce contention (unless a final flush is needed at end of execution). For each task: +The orchestrator completes fanout wiring before publishing a task to the ready queues. For each task with live producers: 1. Sets `fanin_count = N + 1` (+1 redundance to prevent premature readiness) 2. For each producer in `payload->fanin_slot_states[]`: @@ -449,7 +447,9 @@ The orchestrator pushes each submitted task to the global `scheduler->wiring_que - If not completed: prepends consumer to producer's `fanout_head` via `dep_pool.prepend` - **Releases** `fanout_lock` 3. Atomically releases the +1 redundance + early_finished count via `fanin_refcount.fetch_add` -4. If all deps satisfied: pushes task to ready queue +4. If all deps satisfied: pushes task to the routed ready queue + +Zero-fanin tasks and tasks whose claimed producers are already completed skip dep_pool entry allocation and publish directly to the routed ready queue. The scheduler's completion handler mirrors this: @@ -700,11 +700,9 @@ With `PTO2_SERIAL_ORCH_SCHED=1`, scheduler threads still wait for `runtime_init_ready_` first, then additionally wait for `orchestrator_done_` before entering `resolve_and_dispatch()`. The default is off, preserving the current overlapped orch/sched pipeline. Serial mode is intended for measurement -and debugging. During the serial wait, scheduler thread 0 may drain deferred -wiring records to prevent bounded wiring-queue backpressure, but it does not -dispatch AICore work until `orchestrator_done_` is set. Large graphs may still -require larger task-ring, heap, or dependency-pool capacity because no task -execution/reclaim happens during graph build. +and debugging. It does not dispatch AICore work until `orchestrator_done_` is +set. Large graphs may still require larger task-ring, heap, or dependency-pool +capacity because no task execution/reclaim happens during graph build. --- diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 50f734fee..1d8ee744b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -163,7 +163,7 @@ This project-defined flattened numbering is kept unchanged. 1. Validate submit arguments. 2. Allocate mixed-task ID and initialize descriptor/payload/slot_state once. 3. Lookup producers via TensorMap; collect fanin metadata and increment producers' `fanout_count`. -4. Push task to scheduler's wiring queue (scheduler thread 0 asynchronously wires fanout edges and determines readiness). +4. Wire fanout edges on the orchestrator side and publish ready tasks to the scheduler queues. 5. Dispatch all active lanes atomically when resources allow. 6. Aggregate completion and release downstream once. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp index b7cc58794..fbf43cf9b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp @@ -279,9 +279,9 @@ static bool append_fanin_or_fail( // producer; ++'ing it would corrupt an unrelated task. // (2) Already CONSUMED in place — finished, output ready, no real edge. // In either case, adding it to the fanin and bumping fanout_count would leave - // a stale ++/release pair (wire_task drops the fanout edge but keeps the fanin - // slot, so on_task_release still release_producer()'s it) that desyncs the - // slot's refcount (rc != fc) and wedges in-order reclaim. Claiming a live + // a stale ++/release pair (Orch-side wiring drops the fanout edge but keeps + // the fanin slot, so on_task_release still release_producer()'s it) that + // desyncs the slot's refcount (rc != fc) and wedges in-order reclaim. Claiming a live // producer under the lock pins it: fanout_count now counts us, so it cannot // reach CONSUMED (rc == fc) until we release it in on_task_release, keeping the // slot's generation stable until then. check_and_handle_consumed flips @@ -297,8 +297,9 @@ static bool append_fanin_or_fail( // keyed to the live producer, and doing it before the ++ still suppresses a // double-count for a producer named twice in one submission. prod_state->lock_fanout(); + PTO2TaskState pstate = prod_state->task_state.load(std::memory_order_acquire); bool gone = prod_state->task == nullptr || prod_state->task->task_id.local() != producer_task_id.local() || - prod_state->task_state.load(std::memory_order_acquire) == PTO2_TASK_CONSUMED; + pstate == PTO2_TASK_CONSUMED; bool claim = !gone && !fanin_builder->mark_seen(prod_ring, prod_slot); if (claim) { // Low bits hold the consumer count; bit31 is the scope ref. The consumer @@ -345,6 +346,83 @@ static bool append_fanin_or_fail( return true; } +static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { + if (fanin_builder.count == 0) return true; + return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool { + return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED; + }); +} + +static void mark_orch_wired_no_edges(PTO2SchedulerState *sched, PTO2TaskSlotState &slot_state) { + auto &rss = sched->ring_sched_states[slot_state.ring_id]; + sched->mark_dep_pool_position(rss, slot_state); +} + +static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, PTO2TaskSlotState &slot_state, int32_t wfanin) { + PTO2SchedulerState *sched = orch->scheduler; + auto &rss = sched->ring_sched_states[slot_state.ring_id]; + + // dep_pool is orchestrator-exclusive (no lock). Block until space frees. + // A stall here is usually a *symptom* of the scheduler having stalled + // (e.g. a scheduler timeout latches sched_error_code); latching our own + // dep_pool overflow would mask that root cause. So bail without latching + // when any fatal is already set elsewhere, and only declare a genuine + // dep_pool deadlock after a backstop that runs STRICTLY LONGER than the + // scheduler timeout — so a scheduler-side fault (e.g. -100) surfaces first + // — with no forward progress and no other fault. + if (rss.dep_pool.available() >= wfanin) { + sched->wire_fanin_task(rss, slot_state, wfanin); + return true; + } + + // Backstop = scheduler timeout + alloc-deadlock slack. Must exceed the + // scheduler timeout so the scheduler's own fault (-100) is latched and + // observed here before this dep_pool overflow (-4). + const uint64_t backstop_cycles = + static_cast(PLATFORM_SCHEDULER_TIMEOUT_MS) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) + + PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES; + + int spin_count = 0; + int32_t prev_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire); + uint64_t block_cycle0 = 0; + bool block_timing = false; + while (rss.dep_pool.available() < wfanin) { + rss.dep_pool.reclaim(*rss.ring, prev_last_alive); + if (rss.dep_pool.available() >= wfanin) break; + + spin_count++; + int32_t cur_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire); + if (cur_last_alive > prev_last_alive) { + spin_count = 0; + prev_last_alive = cur_last_alive; + block_timing = false; + } else if ((spin_count & 1023) == 0) { + // A fatal latched elsewhere (scheduler timeout -> sched_error_code, + // or an earlier orch_error_code) is the root cause of the stall — + // unwind without latching dep_pool overflow so that code surfaces. + if (orch->sm_header->sched_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE || + orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { + orch->fatal = true; + return false; + } + // Absolute-time backstop, stable across contention. get_sys_cnt_aicpu + // is an MMIO read, so sample it only once per 1024 spins. + uint64_t now = get_sys_cnt_aicpu(); + if (!block_timing) { + block_cycle0 = now; + block_timing = true; + } else if (now - block_cycle0 >= backstop_cycles) { + orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW); + return false; + } + } + SPIN_WAIT_HINT(); + } + + sched->wire_fanin_task(rss, slot_state, wfanin); + return true; +} + static void scope_tasks_push(PTO2OrchestratorState *orch, PTO2TaskSlotState *task_slot_state); struct PTO2PreparedTask { @@ -428,8 +506,8 @@ static bool prepare_task( // returns a slot whose previous occupant is CONSUMED and quiescent (alloc // spins until last_task_alive passes it; in-order reclaim + acquire load), // and the slot is not published to any scheduler thread until the - // wiring.queue.push at the end of submit_task_common — so this reset is - // race-free. Doing it here (not relying on the scheduler's eager + // Orch-side wiring publish at the end of submit_task_common — so this reset + // is race-free. Doing it here (not relying on the scheduler's eager // reset-after-CONSUMED, which only covers the contiguously-reclaimed tail) // makes every reused slot self-clean, which lets the per-boot SM init skip // its O(window) per-slot loop. bind_ring is slot-invariant but cheap to @@ -437,6 +515,7 @@ static bool prepare_task( out->slot_state->bind_ring(ring_id); out->slot_state->reset_for_reuse(); out->slot_state->fanin_count = 0; + out->slot_state->dep_pool_mark = 0; out->payload->prefetch(args.tensor_count(), args.scalar_count()); @@ -445,14 +524,14 @@ static bool prepare_task( // here lets RingSchedState::init() skip the O(window_size) bind loop. // Both writes hit the same 64B slot_state cache line we're about to // dirty below, so the extra cost is two stores on an already-hot line. - // Must precede the scheduler wiring.queue.push at the end of - // submit_task_common — that push is the first read of slot_state->task / - // slot_state->payload by another thread. + // Must precede the Orch-side wiring publish at the end of + // submit_task_common — that publish is the first read of slot_state->task / + // slot_state->payload by scheduler threads. out->slot_state->bind_buffers(out->payload, out->task); // prepare_task does NO payload writes: all payload content (tensors/scalars + // early-dispatch spec fields) is initialized in PTO2TaskPayload::init, the - // single payload-init point, which runs before the scheduler wiring push. + // single payload-init point, which runs before Orch-side wiring publish. // Fields already reset by advance_ring_pointers (eager reset after CONSUMED): // fanout_lock=0, fanout_count=PTO2_FANOUT_SCOPE_BIT, fanout_head=nullptr, @@ -467,7 +546,7 @@ static bool prepare_task( static_cast(block_num * __builtin_popcount(active_mask.core_mask())); out->slot_state->logical_block_num = block_num; out->slot_state->active_mask = active_mask; - // fanin_count is set by scheduler during wiring + // fanin_count is set during Orch-side wiring scope_tasks_push(orch, out->slot_state); return true; @@ -685,8 +764,8 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // Shared body for submit_task / submit_dummy_task. Caller has already validated // args.has_error, decided active_mask (empty for dummy), and resolved the per-slot // kernel_ids (all INVALID_KERNEL_ID for dummy). Performs tensormap sync, fanin -// computation (explicit_deps + auto), output registration, slot init, and pushes -// to the scheduler wiring queue. +// computation (explicit_deps + auto), output registration, slot init, and +// Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id @@ -864,28 +943,26 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_LAP(g_orch_args_cycle); - // === STEP 6: push to wiring queue === - // Deferred wiring: orchestrator only stores dependency metadata and increments - // fanout_count. The actual fanout_head wiring (lock + dep_pool + early_finished) - // is handled asynchronously by scheduler thread 0 via the wiring queue. - // Push to global wiring queue — scheduler sets fanin_count, wires fanout, checks readiness - if (!sched->wiring.queue.push(&cur_slot_state)) { - // producer_blocked is the wiring deadlock detector's "orchestrator is - // stuck in push" observable: set ONLY while we actually spin (queue - // full), cleared on exit, so the just-filled-then-scope_end case (push - // succeeded, no spin) never trips a false deadlock. Also poll the shared - // orch_error_code so a fatal latched by any party (e.g. that detector) - // breaks this otherwise-unbounded spin and unwinds orchestration. - sched->wiring.producer_blocked.store(1, std::memory_order_release); - while (!sched->wiring.queue.push(&cur_slot_state)) { - if (orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { - orch->fatal = true; - sched->wiring.producer_blocked.store(0, std::memory_order_release); - return result; - } - SPIN_WAIT_HINT(); + // === STEP 6: wire on the orchestrator side and publish readiness === + // Zero-fanin tasks and tasks whose claimed producers are already completed + // do not need fanout links or dep_pool entries. Tasks with live producers + // allocate fanout links here before any scheduler thread can dispatch them. + if (fanin_builder.count == 0) { + cur_slot_state.fanin_count = 1; + cur_slot_state.fanin_refcount.store(1, std::memory_order_release); + mark_orch_wired_no_edges(sched, cur_slot_state); + sched->push_ready_routed(&cur_slot_state); + } else if (all_claimed_fanin_completed(fanin_builder)) { + int32_t ready_seed = fanin_builder.count + 1; + cur_slot_state.fanin_count = ready_seed; + payload.dispatch_fanin.store(fanin_builder.count, std::memory_order_release); + cur_slot_state.fanin_refcount.store(ready_seed, std::memory_order_release); + mark_orch_wired_no_edges(sched, cur_slot_state); + sched->push_ready_routed(&cur_slot_state); + } else { + if (!orch_wire_live_fanin_task(orch, cur_slot_state, fanin_builder.count)) { + return result; } - sched->wiring.producer_blocked.store(0, std::memory_order_release); } CYCLE_COUNT_LAP(g_orch_fanin_cycle); @@ -1071,7 +1148,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) { // required so scope_end can release the producer-side reference and // drive the slot to CONSUMED, but worker dispatch fields are never // observed for hidden alloc tasks. - prepared.slot_state->task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + prepared.slot_state->mark_completed(); } orch->inline_completed_tasks++; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h index 97f318d40..05fab8762 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h @@ -172,8 +172,7 @@ class PTO2TaskAllocator { prev_last_alive = last_alive; block_timing = false; } else if ((spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this otherwise-unbounded spin; the + // A fatal latched elsewhere breaks this otherwise-unbounded spin; the // caller maps the failed alloc to orch_mark_fatal. Polled on the // cold path only -- error_code_ptr_ is orch_error_code. if (error_code_ptr_ != nullptr && error_code_ptr_->load(std::memory_order_acquire) != PTO2_ERROR_NONE) { diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp index 83a44c957..459abafcc 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp @@ -108,7 +108,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa constexpr int kSegmentCap = 64; const PTO2TaskSlotState *seg[kSegmentCap]; int seg_count = 0; - bool signaled = false; bool failed = false; auto wait_one_producer = [&](const PTO2TaskSlotState &slot) { @@ -119,8 +118,7 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this wait; cold path only. + // A fatal latched elsewhere breaks this wait; cold path only. if (orch.sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { failed = true; return; @@ -147,8 +145,7 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa (slot.fanout_count & ~PTO2_FANOUT_SCOPE_BIT)) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this wait; cold path only. + // A fatal latched elsewhere breaks this wait; cold path only. if (orch.sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { failed = true; return; @@ -186,10 +183,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa if (failed) return; } seg[seg_count++] = &s; - if (!signaled) { - orch.scheduler->wiring.orch_needs_drain.store(true, std::memory_order_release); - signaled = true; - } }; auto do_wait = [&]() { @@ -212,9 +205,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa }; do_wait(); - if (signaled) { - orch.scheduler->wiring.orch_needs_drain.store(false, std::memory_order_release); - } return !failed; } MAYBE_UNINITIALIZED_END diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h index 64f4c6319..6426ee386 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h @@ -223,8 +223,8 @@ PTO2Runtime *runtime_init_data_from_layout( /** * Phase 3 — wire every arena-internal pointer field (rt->sm_handle, * rt->aicore_mailbox, orchestrator.{scope_tasks, scope_begins, scheduler, - * tensor_map.*, rings[].fanin_pool.base}, scheduler.{ready_queues, dep_pool, - * wiring.queue}) so each holds arena.base() + offset. Idempotent — runs on + * tensor_map.*, rings[].fanin_pool.base}, scheduler.{ready_queues, dep_pool}) + * so each holds arena.base() + offset. Idempotent — runs on * both host (writing host-mirror addresses) and AICPU (writing device * addresses) sides. */ diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h index 01194134a..e62dd574f 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h @@ -90,9 +90,6 @@ // Cross-thread early-dispatch work queue (power of two) #define PTO2_EARLY_DISPATCH_QUEUE_SIZE 64 -// Wiring queue -#define PTO2_WRIRING_QUEUE_SIZE 1024 // Per-shape queue size - // Fanin storage #define PTO2_FANIN_INLINE_CAP 64 @@ -401,6 +398,15 @@ static_assert( // never reach -> provable deadlock. static constexpr uint32_t PTO2_FANOUT_SCOPE_BIT = 0x80000000u; +enum PTO2ReadyState : uint8_t { + PTO2_READY_UNCLAIMED = 0, + PTO2_READY_CLAIMED = 1, +}; + +enum PTO2CompletionFlag : uint8_t { + PTO2_COMPLETION_DONE = 2, +}; + struct alignas(64) PTO2TaskSlotState { // Fanout lock + list (accessed together under lock in on_task_complete) std::atomic fanout_lock; // Per-task spinlock (0=unlocked, 1=locked) @@ -437,8 +443,8 @@ struct alignas(64) PTO2TaskSlotState { // sequenced before on_subtask_complete's acq_rel fetch_add and the read // after, so all earlier subtasks' writes are visible to the last subtask. std::atomic any_subtask_deferred{false}; - uint8_t _async_pad{0}; - int32_t dep_pool_mark{0}; // Dep pool top after wiring (thread-0-only) + std::atomic ready_state{PTO2_READY_UNCLAIMED}; + int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring std::atomic completed_subtasks{0}; // Each core completion increments by 1 int16_t total_required_subtasks{0}; // = logical_block_num * popcount(active_mask) @@ -467,6 +473,15 @@ struct alignas(64) PTO2TaskSlotState { task = t; } + void mark_completed() { + task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + ready_state.fetch_or(PTO2_COMPLETION_DONE, std::memory_order_release); + } + + bool is_completion_flag_set() const { + return (ready_state.load(std::memory_order_acquire) & PTO2_COMPLETION_DONE) != 0; + } + /** * Reset dynamic scheduling fields for slot reuse. * Called by advance_ring_pointers() after a slot transitions to CONSUMED @@ -487,6 +502,7 @@ struct alignas(64) PTO2TaskSlotState { completed_subtasks.store(0, std::memory_order_relaxed); next_block_idx.store(0, std::memory_order_relaxed); any_subtask_deferred.store(false, std::memory_order_relaxed); + ready_state.store(PTO2_READY_UNCLAIMED, std::memory_order_relaxed); // Note: payload spec fields (spec_state / staged_core_mask / dispatch_fanin / // spec_chain_*) are NOT reset here — this method skips the payload by // contract. They are (re)initialized in PTO2TaskPayload::init on every diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h index 3ed6058ce..9265c04ac 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h @@ -386,135 +386,6 @@ bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); void ready_queue_destroy(PTO2ReadyQueue *queue); -// ============================================================================= -// SPSC Queue (Single-Producer Single-Consumer, wait-free) -// ============================================================================= -// -// Bounded ring buffer optimized for the wiring queue use case: -// - Producer: orchestrator thread (push) -// - Consumer: scheduler thread 0 (pop_batch) -// -// Design based on Rigtorp's cached-index technique: each side caches -// the other's index locally, avoiding cross-core cache line bouncing -// on the hot path. Only when the local cache says "full" or "empty" -// does the thread issue an acquire load on the remote index. -// -// Memory layout: 5 cache-line-aligned fields ensure zero false sharing. - -struct alignas(64) PTO2SpscQueue { - // --- Producer cache lines (orchestrator thread) --- - alignas(64) std::atomic head_{0}; - alignas(64) uint64_t tail_cached_{0}; - - // --- Consumer cache lines (scheduler thread 0) --- - alignas(64) std::atomic tail_{0}; - alignas(64) uint64_t head_cached_{0}; - - // --- Shared Cacheline (read only) with mask and data ptr (immutable after init) --- - alignas(64) PTO2TaskSlotState **buffer_{nullptr}; - uint64_t mask_{0}; - - // Padding to exactly 5 cache lines - char padding[64 - sizeof(PTO2TaskSlotState **) - sizeof(uint64_t)]; - - // Reserve the backing buffer region on the supplied arena. Returns the - // region offset, to be passed to init_from_layout() after the arena is - // committed. Cache-line aligned: the buffer is shared between the - // orchestrator (push) and scheduler thread 0 (pop_batch), so its base - // must not false-share with neighboring regions. - static size_t reserve_layout(DeviceArena &arena, uint64_t capacity) { - return arena.reserve(capacity * sizeof(uintptr_t), PTO2_ALIGN_SIZE); - } - - // Writes everything except the arena-internal `buffer_` pointer field - // (zeros the slot pointer array, mask/head/tail). The host pre-builds the - // image without storing a host address in buffer_; the AICPU wires - // buffer_ at boot via wire_arena_pointers(). - bool init_data_from_layout(DeviceArena &arena, size_t buffer_off, uint64_t capacity) { - if (capacity == 0 || (capacity & (capacity - 1)) != 0) return false; - auto *buf = static_cast(arena.region_ptr(buffer_off)); - // calloc'd-equivalent: zero the slot pointers so spurious early pops - // observe nullptr. - for (uint64_t i = 0; i < capacity; i++) - buf[i] = nullptr; - mask_ = capacity - 1; - head_.store(0, std::memory_order_relaxed); - tail_.store(0, std::memory_order_relaxed); - tail_cached_ = 0; - head_cached_ = 0; - return true; - } - - // Wire the arena-internal pointer. Called by both host (with host arena) - // and AICPU (with device arena attached to the prebuilt image). - void wire_arena_pointers(DeviceArena &arena, size_t buffer_off) { - buffer_ = static_cast(arena.region_ptr(buffer_off)); - } - - void reset_for_reuse() { - uint64_t h = head_.load(std::memory_order_relaxed); - tail_.store(h, std::memory_order_relaxed); - tail_cached_ = h; - head_cached_ = h; - } - - // Arena owns the buffer; here we only forget our pointer. - void destroy() { buffer_ = nullptr; } - - // Push one item (producer only). Returns false if queue is full. - // Full condition: next_h - tail > mask_ (i.e. > capacity-1), so the - // effective usable capacity is capacity-1 (one slot is wasted as a - // sentinel to distinguish full from empty). uint64_t wrapping is safe - // since head and tail are monotonically increasing and subtraction - // wraps correctly. - bool push(PTO2TaskSlotState *item) { - uint64_t h = head_.load(std::memory_order_relaxed); - uint64_t next_h = h + 1; - if (next_h - tail_cached_ > mask_) { - tail_cached_ = tail_.load(std::memory_order_acquire); - if (next_h - tail_cached_ > mask_) { - return false; - } - } - buffer_[h & mask_] = item; - head_.store(next_h, std::memory_order_release); - return true; - } - - // Pop up to max_count items (consumer only). Returns actual count. - int pop_batch(PTO2TaskSlotState **out, int max_count) { - uint64_t t = tail_.load(std::memory_order_relaxed); - uint64_t avail = head_cached_ - t; - if (avail < static_cast(max_count)) { - head_cached_ = head_.load(std::memory_order_acquire); - avail = head_cached_ - t; - if (avail == 0) return 0; - } - int count = (avail < static_cast(max_count)) ? static_cast(avail) : max_count; - for (int i = 0; i < count; i++) { - out[i] = buffer_[(t + i) & mask_]; - } - tail_.store(t + count, std::memory_order_release); - return count; - } - - // Approximate size (used for backoff decisions, not exact). - uint64_t size() const { - uint64_t h = head_.load(std::memory_order_acquire); - uint64_t t = tail_.load(std::memory_order_acquire); - return h - t; - } - - // Full ⟺ the producer's next push() would fail: size has reached the - // usable capacity (mask_ = capacity - 1, one slot reserved as sentinel). - // Used by the wiring-queue deadlock detector to prove the orchestrator is - // blocked in push(). - bool full() const { return size() >= mask_; } -}; - -static_assert(sizeof(PTO2SpscQueue) == 5 * 64, "PTO2SpscQueue must be exactly 5 cache lines (320B)"); -// ============================================================================= - /** * Statistics returned by mixed-task completion processing */ @@ -535,9 +406,7 @@ struct PTO2SchedulerLayout { size_t off_dummy_ready_queue_slots; size_t off_early_dispatch_queue_slots; size_t off_dep_pool_entries[PTO2_MAX_RING_DEPTH]; - size_t off_wiring_spsc_buffer; uint64_t ready_queue_capacity; - uint64_t spsc_capacity; int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]; }; @@ -559,12 +428,8 @@ struct PTO2SchedulerState { int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS - // --- Cache Line 1+: Thread 0 only (wiring dep_pool) --- + // --- Cache Line 1+: Orch-side wiring dep_pool --- alignas(64) PTO2DepListPool dep_pool; - // One-shot latch for the wiring-queue deadlock report (thread 0 only): - // the drain breaks on dep_pool exhaustion every call while wedged, so - // the tier-1 structural diagnostic is emitted once, not per call. - bool dep_deadlock_reported = false; #if PTO2_PROFILING // Published only for scope_stats; orchestrator must not read dep_pool's non-atomic counters directly. alignas(64) std::atomic dep_pool_snapshot_tail; @@ -627,41 +492,6 @@ struct PTO2SchedulerState { // the dispatch loop and completed inline -- never goes to AICore. PTO2ReadyQueue dummy_ready_queue; - // Wiring subsystem — groups all wiring-related state for cache-line isolation. - // - // Three cache-line regions by writer: - // 1. batch_* / backoff — thread 0 exclusive (local batch buffer) - // 2. queue — SPSC: orchestrator push, thread 0 pop - // 3. orch_needs_drain — orchestrator write, thread 0 read - struct alignas(64) WiringState { - static constexpr uint64_t BATCH_SIZE = 30; - static constexpr int BACKOFF_LIMIT = 32; - - // --- Thread 0 exclusive: local batch buffer + backoff --- - int batch_count = 0; - int batch_index = 0; - int backoff_counter = 0; - PTO2TaskSlotState *batch[BATCH_SIZE]; - - // --- SPSC queue: orchestrator (push) ↔ thread 0 (pop) --- - PTO2SpscQueue queue; - - // --- Orchestrator write, thread 0 read --- - alignas(64) std::atomic orch_needs_drain{false}; - // Set to 1 only while the orchestrator is actually spinning in - // queue.push() (queue full), cleared on a successful push. The wiring - // deadlock detector reads this as the producer-blocked observable: it - // proves the orchestrator is stuck BEFORE its scope_end, as opposed to - // having just filled the queue with its last in-scope push and being - // about to call scope_end (which would release the head -> no deadlock). - std::atomic producer_blocked{0}; - } wiring; - - static_assert( - offsetof(WiringState, queue) == 256, "WiringState: batch region must be exactly 4 cache lines before queue" - ); - static_assert(sizeof(WiringState) == 640, "WiringState must be exactly 10 cache lines (640B)"); - alignas(64) AsyncWaitList async_wait_list; // Statistics (cold path, isolated from hot-path fields) @@ -673,129 +503,6 @@ struct PTO2SchedulerState { // Inline hot-path methods // ========================================================================= - /** - * Drain wiring queue: pop submitted tasks and wire their fanout edges. - * Called by scheduler thread 0 each loop iteration. Sets fanin_count, - * acquires fanout_lock per producer, allocates dep_pool entries, and - * pushes ready tasks to the appropriate ready queue. - * - * @return Number of tasks wired this call. - */ - - int drain_wiring_queue(bool force_drain = false) { - int wired = 0; - - // Refill local batch buffer when exhausted. - if (wiring.batch_index >= wiring.batch_count) { - // Backoff: defer pop when queue holds fewer than a full batch, - // unless force_drain, orch_needs_drain, or backoff limit reached. - if (!force_drain && wiring.queue.size() < WiringState::BATCH_SIZE) { - if (!wiring.orch_needs_drain.load(std::memory_order_acquire) && - wiring.backoff_counter < WiringState::BACKOFF_LIMIT) { - wiring.backoff_counter++; - return 0; - } - } - wiring.backoff_counter = 0; - wiring.batch_count = wiring.queue.pop_batch(wiring.batch, WiringState::BATCH_SIZE); - wiring.batch_index = 0; - if (wiring.batch_count == 0) return 0; - } - - // Process tasks from local buffer in strict FIFO order. - while (wiring.batch_index < wiring.batch_count) { - PTO2TaskSlotState *ws = wiring.batch[wiring.batch_index]; - int ring_id = ws->ring_id; - auto &rss = ring_sched_states[ring_id]; - int32_t wfanin = ws->payload->fanin_actual_count; - - if (wfanin > 0 && rss.dep_pool.available() < wfanin) { - rss.dep_pool.reclaim(*rss.ring, rss.last_task_alive); - if (rss.dep_pool.available() < wfanin) { -#if PTO2_PROFILING - if (is_scope_stats_enabled()) { - rss.publish_dep_pool_snapshot(); - } -#endif - // dep_pool can't reclaim because the reclaim watermark is - // wedged. This runs on the scheduler thread, so unlike - // alloc()'s detector it cannot self-observe that the - // orchestrator is blocked; wiring.producer_blocked is the - // external certificate -- the orchestrator sets it ONLY while - // it is actually spinning in queue.push() (cleared on a - // successful push), so the "just filled the queue then called - // scope_end" case (push succeeded -> flag stays 0) cannot trip - // a false report. With the producer provably stuck in push - // (program-order before its scope_end) AND the head COMPLETED, - // all consumers released, scope still open (only scope_end - // frees it), scope_end can never run -> provable head-of-line - // deadlock. The producer-blocked gate also pins the head: - // scope_end has not run, so the scope-gated head cannot be - // CONSUMED/reset concurrently while we read it. - if (!rss.dep_deadlock_reported && wiring.producer_blocked.load(std::memory_order_acquire) != 0) { - int32_t last_alive = rss.last_task_alive; - PTO2TaskSlotState &h = rss.ring->get_slot_state_by_task_id(last_alive); - // Read the head under its fanout_lock: fanout_count is a - // lock-protected field, and one snapshot keeps the check - // and the report consistent. - h.lock_fanout(); - int32_t state = h.task_state.load(std::memory_order_acquire); - uint32_t fc = h.fanout_count; - uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); - h.unlock_fanout(); - bool head_scope_gated = (state == PTO2_TASK_COMPLETED) && (rc == (fc & ~PTO2_FANOUT_SCOPE_BIT)); - if (head_scope_gated) { - rss.dep_deadlock_reported = true; - report_wiring_deadlock(rss, wfanin, last_alive, state, fc, rc); - // Latch the shared fatal so both sides exit fast off - // one error code: the scheduler cold-path poll - // (handle_orchestrator_exit) emergency_shutdowns, and - // the orchestrator's push spin breaks out and unwinds. - if (rss.dep_pool.error_code_ptr != nullptr) { - int32_t expected = PTO2_ERROR_NONE; - rss.dep_pool.error_code_ptr->compare_exchange_strong( - expected, PTO2_ERROR_DEP_POOL_OVERFLOW, std::memory_order_acq_rel - ); - } - } - } - break; // not enough dep_pool space — keep remainder for next call - } - } - - wiring.batch_index++; - wire_task(rss, ws, wfanin); - wired++; - } - - return wired; - } - - // Tier-1 structural diagnostic for a provable wiring-queue deadlock (head - // COMPLETED + all consumers released + scope still open, dep_pool exhausted, - // orchestrator provably blocked in push). The head snapshot (state/fc/rc) is - // taken under fanout_lock by the caller and passed in, so the report agrees - // with the check and reads no lock-protected field unlocked. - void report_wiring_deadlock( - RingSchedState &rss, int32_t wfanin, int32_t last_alive, int32_t state, uint32_t fc, uint32_t rc - ) { - LOG_ERROR("========================================"); - LOG_ERROR("FATAL: Wiring-Queue Deadlock - Dep Pool Exhausted!"); - LOG_ERROR("========================================"); - LOG_ERROR("Head task %d COMPLETED, all consumers released, scope still open ->", last_alive); - LOG_ERROR("only scope_end can free it, but the orchestrator is blocked on a full wiring"); - LOG_ERROR("queue (in push, before its scope_end). Provable head-of-line deadlock."); - LOG_ERROR( - " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, state, - rc & ~PTO2_FANOUT_SCOPE_BIT, fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 - ); - LOG_ERROR(" Dep pool: used=%d/%d, needed=%d entries", rss.dep_pool.used(), rss.dep_pool.capacity, wfanin); - LOG_ERROR("Solution:"); - LOG_ERROR(" The open scope's fanout exceeds the dep pool. Either split the scope, or"); - LOG_ERROR(" raise PTO2_RING_DEP_POOL (compile-time PTO2_DEP_LIST_POOL_SIZE)."); - LOG_ERROR("========================================"); - } - // Route a ready slot to the right global queue. Dummy tasks (empty // active_mask) live in dummy_ready_queue; everything else goes to the // per-shape ready_queues[]. @@ -808,51 +515,21 @@ struct PTO2SchedulerState { } } - /** - * Wire fanout edges for a single task. Sets fanin_count, acquires each - * producer's fanout_lock, allocates dep_pool entries for live producers, - * pushes the task to the ready queue once its fanin refcount is satisfied. - */ - void wire_task(RingSchedState &rss, PTO2TaskSlotState *ws, int32_t wfanin) { - PTO2TaskPayload *wp = ws->payload; - ws->fanin_count = wfanin + 1; - - if (wfanin != 0) { - int32_t early_finished = 0; - for_each_fanin_slot_state(*wp, [&](PTO2TaskSlotState *producer) { - producer->lock_fanout(); - int32_t pstate = producer->task_state.load(std::memory_order_acquire); - if (pstate >= PTO2_TASK_COMPLETED) { - early_finished++; - } else { - producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, ws); - } - producer->unlock_fanout(); - }); - - // Seed dispatch_fanin with producers already complete at wiring - // time (e.g. buffer-creator tasks that finished before this - // consumer entered the graph). Such producers never dispatch at - // runtime, so they can never bump dispatch_fanin via the fanout - // walk; without this seed the candidate compare - // (dispatch_fanin == fanin_actual_count) would be unreachable - // whenever any producer is pre-completed. Mirrors the - // early_finished seed that ready_fanin gets via init_rc. - if (early_finished != 0) { - wp->dispatch_fanin.fetch_add(early_finished, std::memory_order_acq_rel); - } - - int32_t init_rc = early_finished + 1; - int32_t new_rc = ws->fanin_refcount.fetch_add(init_rc, std::memory_order_acq_rel) + init_rc; - if (new_rc >= ws->fanin_count) { - push_ready_routed(ws); + bool try_claim_ready_once(PTO2TaskSlotState &slot_state) { + uint8_t state = slot_state.ready_state.load(std::memory_order_acquire); + for (;;) { + if ((state & PTO2_READY_CLAIMED) != 0) return false; + uint8_t desired = state | PTO2_READY_CLAIMED; + if (slot_state.ready_state.compare_exchange_weak( + state, desired, std::memory_order_acq_rel, std::memory_order_acquire + )) { + return true; } - } else { - ws->fanin_refcount.fetch_add(1, std::memory_order_acq_rel); - push_ready_routed(ws); } + } - ws->dep_pool_mark = rss.dep_pool.top; + void mark_dep_pool_position(RingSchedState &rss, PTO2TaskSlotState &slot_state) { + slot_state.dep_pool_mark = rss.dep_pool.top; #if PTO2_PROFILING if (is_scope_stats_enabled()) { rss.publish_dep_pool_snapshot(); @@ -860,6 +537,41 @@ struct PTO2SchedulerState { #endif } + /** + * Wire live fanin edges from the orchestrator thread. The caller must have + * ensured rss.dep_pool has room for wfanin. Dep-pool mutation is Orch-only; + * producer fanout lists are still protected by each producer's fanout_lock. + */ + void wire_fanin_task(RingSchedState &rss, PTO2TaskSlotState &slot_state, int32_t wfanin) { + PTO2TaskPayload *payload = slot_state.payload; + slot_state.fanin_count = wfanin + 1; + + int32_t early_finished = 0; + for_each_fanin_slot_state(*payload, [&](PTO2TaskSlotState *producer) { + producer->lock_fanout(); + int32_t pstate = producer->task_state.load(std::memory_order_acquire); + if (pstate >= PTO2_TASK_COMPLETED) { + early_finished++; + } else { + producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, &slot_state); + } + producer->unlock_fanout(); + }); + + // Producers that completed before this consumer was wired will not walk a + // fanout edge for it, so seed both dispatch and ready fanin here. + if (early_finished != 0) { + payload->dispatch_fanin.fetch_add(early_finished, std::memory_order_acq_rel); + } + + int32_t init_rc = early_finished + 1; + int32_t new_rc = slot_state.fanin_refcount.fetch_add(init_rc, std::memory_order_acq_rel) + init_rc; + mark_dep_pool_position(rss, slot_state); + if (new_rc >= slot_state.fanin_count) { + route_ready_once(slot_state); + } + } + void check_and_handle_consumed(PTO2TaskSlotState &slot_state) { // Read fanout_refcount/fanout_count and flip COMPLETED->CONSUMED under // fanout_lock. The orchestrator claims producers (fanout_count++) under the @@ -1119,6 +831,54 @@ struct PTO2SchedulerState { return slot_state.next_block_idx.load(std::memory_order_seq_cst) >= slot_state.logical_block_num; } + bool route_ready_once(PTO2TaskSlotState &slot_state, SpecReleaseSink *sink = nullptr) { + if (!try_claim_ready_once(slot_state)) return false; + + // Speculative early-dispatch: pre-staged tasks are released by doorbell + // here, skipping the ready-queue round-trip entirely. + if (try_speculative_release(slot_state, sink)) return true; + + PTO2ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == PTO2ResourceShape::DUMMY) { + dummy_ready_queue.push(&slot_state); + } else { + ready_queues[static_cast(shape)].push(&slot_state); + } + return true; + } + +#if PTO2_ORCH_PROFILING || PTO2_SCHED_PROFILING + bool route_ready_once( + PTO2TaskSlotState &slot_state, uint64_t &atomic_count, uint64_t &push_wait, SpecReleaseSink *sink = nullptr + ) { + uint8_t state = slot_state.ready_state.load(std::memory_order_acquire); + atomic_count += 1; // ready_state load + for (;;) { + if ((state & PTO2_READY_CLAIMED) != 0) return false; + uint8_t desired = state | PTO2_READY_CLAIMED; + if (slot_state.ready_state.compare_exchange_weak( + state, desired, std::memory_order_acq_rel, std::memory_order_acquire + )) { + atomic_count += 1; // ready_state CAS + break; + } + atomic_count += 1; // failed ready_state CAS + } + + // Speculative early-dispatch: pre-staged tasks are released by doorbell + // here, skipping the ready-queue round-trip entirely. + if (try_speculative_release(slot_state, sink)) return true; + + PTO2ResourceShape shape = slot_state.active_mask.to_shape(); + if (shape == PTO2ResourceShape::DUMMY) { + dummy_ready_queue.push(&slot_state, atomic_count, push_wait); + } else { + ready_queues[static_cast(shape)].push(&slot_state, atomic_count, push_wait); + } + return true; + } +#endif + bool release_fanin_and_check_ready(PTO2TaskSlotState &slot_state, SpecReleaseSink *sink = nullptr) { // Atomically increment fanin_refcount and check if all producers are done // ACQ_REL on fanin_refcount already synchronizes with the orchestrator's @@ -1126,11 +886,7 @@ struct PTO2SchedulerState { int32_t new_refcount = slot_state.fanin_refcount.fetch_add(1, std::memory_order_acq_rel) + 1; if (new_refcount == slot_state.fanin_count) { - // Speculative early-dispatch: pre-staged tasks are released by doorbell - // here, skipping the ready-queue round-trip entirely. - if (try_speculative_release(slot_state, sink)) return true; - push_ready_routed(&slot_state); - return true; + return route_ready_once(slot_state, sink); } return false; } @@ -1143,19 +899,7 @@ struct PTO2SchedulerState { atomic_count += 1; // fanin_refcount.fetch_add if (new_refcount == slot_state.fanin_count) { - // Speculative early-dispatch: pre-staged tasks are released by doorbell - // here, skipping the ready-queue round-trip entirely. - if (try_speculative_release(slot_state, sink)) return true; - // Dummy slots go to dummy_ready_queue; everything else to the per-shape - // ready_queues[]. Use the profiling-aware push so atomic_count / push_wait - // stay consistent with the non-dummy path. - PTO2ResourceShape shape = slot_state.active_mask.to_shape(); - if (shape == PTO2ResourceShape::DUMMY) { - dummy_ready_queue.push(&slot_state, atomic_count, push_wait); - } else { - ready_queues[static_cast(shape)].push(&slot_state, atomic_count, push_wait); - } - return true; + return route_ready_once(slot_state, atomic_count, push_wait, sink); } return false; } @@ -1245,12 +989,12 @@ struct PTO2SchedulerState { #else slot_state.lock_fanout(); #endif - slot_state.task_state.store(PTO2_TASK_COMPLETED, std::memory_order_release); + slot_state.mark_completed(); PTO2DepListEntry *current = slot_state.fanout_head; // Protected by fanout_lock slot_state.unlock_fanout(); #if PTO2_SCHED_PROFILING - lock_atomics += 2; // state.store + unlock.store + lock_atomics += 3; // task_state.store + ready_state.fetch_or + unlock.store g_sched_lock_atomic_count[thread_idx] += lock_atomics; g_sched_lock_wait_cycle[thread_idx] += lock_wait; PTO2_SCHED_CYCLE_LAP(g_sched_lock_cycle[thread_idx]); @@ -1348,12 +1092,16 @@ struct PTO2SchedulerState { // === Cold-path API (defined in pto_scheduler.cpp) === // Phase 1: declare every sub-region (ready_queue slots, dummy queue slots, - // per-ring dep_pool entries, wiring SPSC buffer) on the supplied arena. + // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. static PTO2SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); static PTO2SchedulerLayout reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]); + static PTO2SchedulerLayout reserve_layout( + DeviceArena &arena, const int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH], + const int32_t task_window_sizes[PTO2_MAX_RING_DEPTH] + ); // Phase 3a: write everything *except* arena-internal pointer fields. // `sm_dev_base` is the device address of the SM (only stored, never @@ -1367,7 +1115,7 @@ struct PTO2SchedulerState { // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, dep_pool.base for each - // ring, wiring.queue.buffer_). Called on both host and device sides. + // ring). Called on both host and device sides. void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); // Forget per-region pointers; arena owns the backing memory. diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 8e1813367..3af69a982 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -507,8 +507,8 @@ void SchedulerContext::log_l2_swimlane_summary(int32_t thread_idx, [[maybe_unuse cycles_to_us(sched_end_ts - l2_swimlane.sched_start_ts) ); - uint64_t sched_total = l2_swimlane.sched_wiring_cycle + l2_swimlane.sched_complete_cycle + - l2_swimlane.sched_dispatch_cycle + l2_swimlane.sched_idle_cycle; + uint64_t sched_total = + l2_swimlane.sched_complete_cycle + l2_swimlane.sched_dispatch_cycle + l2_swimlane.sched_idle_cycle; if (sched_total == 0) sched_total = 1; { @@ -600,19 +600,6 @@ void SchedulerContext::log_l2_swimlane_summary(int32_t thread_idx, [[maybe_unuse l2_swimlane.sched_dispatch_setup_cycle * 100.0 / d_parent ); -#if PTO2_SCHED_PROFILING - LOG_INFO_V9( - "Thread %d: wiring : %.3fus (%.1f%%) tasks=%d", thread_idx, - cycles_to_us(l2_swimlane.sched_wiring_cycle), l2_swimlane.sched_wiring_cycle * 100.0 / sched_total, - l2_swimlane.phase_wiring_count - ); -#else - LOG_INFO_V9( - "Thread %d: wiring : %.3fus (%.1f%%)", thread_idx, cycles_to_us(l2_swimlane.sched_wiring_cycle), - l2_swimlane.sched_wiring_cycle * 100.0 / sched_total - ); -#endif - LOG_INFO_V9( "Thread %d: idle : %.3fus (%.1f%%)", thread_idx, cycles_to_us(l2_swimlane.sched_idle_cycle), l2_swimlane.sched_idle_cycle * 100.0 / sched_total @@ -1037,15 +1024,6 @@ void SchedulerContext::bind_runtime(PTO2Runtime *rt) { void SchedulerContext::wait_for_orchestration_done_before_dispatch(Runtime *runtime, int32_t thread_idx) { while (!orchestration_done() && !completed_.load(std::memory_order_acquire)) { - if (thread_idx == 0 && sched_ != nullptr) { - // Use the wiring subsystem's normal batch/backoff policy while - // waiting. This still honors orch_needs_drain/producer_blocked - // signals without force-draining an empty queue every spin. - int wired = sched_->drain_wiring_queue(/*force_drain=*/false); - if (wired > 0) { - continue; - } - } if (sched_ != nullptr && sched_->sm_header != nullptr && check_idle_fatal_error(thread_idx, sched_->sm_header, runtime) == LoopAction::BREAK_LOOP) { break; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 9ce98ac5f..ed3d20e03 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -93,9 +93,8 @@ class SchedulerContext { // mode where rt is created by the orchestrator thread after init(). void bind_runtime(PTO2Runtime *rt); - // Serial orch->sched mode pre-dispatch wait. Thread 0 may drain deferred - // wiring to keep the bounded wiring queue from back-pressuring orchestration, - // but no AICore dispatch happens before orchestrator_done_. + // Serial orch->sched mode pre-dispatch wait. No AICore dispatch happens + // before orchestrator_done_. void wait_for_orchestration_done_before_dispatch(Runtime *runtime, int32_t thread_idx); // ========================================================================= @@ -297,8 +296,7 @@ class SchedulerContext { // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see pto_scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent - // push/pop. Don't confuse with PTO2SpscQueue::size(), which uses acquire - // loads — that one isn't on this path. A stale read here causes at most one + // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. bool has_residual_mix() const { return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index b7e3fa0f7..c9cba53eb 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -777,7 +777,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ for (int s = 0; s < L2SWIMLANE_NUM_QUEUE_SHAPES; s++) shared_out[s] = shared_cached[s]; }; - // Queue-mutating phases (Complete / Wire / Dummy) push newly-ready consumers + // Queue-mutating phases (Complete / Dummy) push newly-ready consumers // straight into the shared ready_queues[] (the local-first buffer is gone), // so their end-of-phase shared depth differs from their start. Force a fresh // re-sample for those emits — this also refreshes the per-iter cache so the @@ -931,44 +931,15 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ continue; } - // Phase 3: Drain wiring queue (thread 0 only) - int wired = 0; - if (thread_idx == 0) { - wired = sched_->drain_wiring_queue(orchestrator_done_.load(std::memory_order_relaxed)); - if (wired > 0) { - made_progress = true; -#if PTO2_SCHED_PROFILING - l2_swimlane.phase_wiring_count += wired; -#endif - } - } -#if PTO2_PROFILING - CYCLE_COUNT_LAP(l2_swimlane.sched_wiring_cycle); - // Wire outer phase: emit one bar covering this iter's drain_wiring_queue - // pass when it wired any tasks. tasks_processed = wired count. Resolve - // does NOT nest under Wire — wiring only enqueues, the consumer release - // happens later in Complete/Dummy. - if (l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES && wired > 0) { - int16_t phase_end_shared[L2SWIMLANE_NUM_QUEUE_SHAPES]; - capture_phase_end_fresh(phase_end_shared); - l2_swimlane_aicpu_record_sched_phase( - thread_idx, L2SwimlaneSchedPhaseKind::Wire, _t0_phase, _t1, l2_swimlane.sched_loop_count, - static_cast(wired), /*pop_hit=*/0, /*pop_miss=*/0, phase_start_shared, phase_end_shared - ); - for (int s = 0; s < L2SWIMLANE_NUM_QUEUE_SHAPES; s++) - phase_start_shared[s] = phase_end_shared[s]; - _t0_phase = _t1; - } -#endif - - // Phase 3b: Drain dummy ready queue (thread 0 only). + // Phase 3: Drain dummy ready queue (S0/S1/S2). // // Dependency-only tasks bypass AICore dispatch: they go through the // scheduler so fanin/fanout edges stay consistent, but completion is - // signalled inline here. Pinned to thread 0 to avoid cross-thread - // races and to keep cache hot near the wiring drain above. - if (thread_idx == 0) { - constexpr int DUMMY_DRAIN_BATCH = 16; + // signalled inline here. The ready queue is MPMC, and the fanout path + // uses per-slot locks/atomics, so multiple scheduler threads can share + // the dependency-only resolve work. + if (thread_idx < 3) { + constexpr int DUMMY_DRAIN_BATCH = 8; PTO2TaskSlotState *dummy_batch[DUMMY_DRAIN_BATCH]; int dummy_got = sched_->dummy_ready_queue.pop_batch(dummy_batch, DUMMY_DRAIN_BATCH); #if PTO2_PROFILING diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index 445c46a56..5040e0257 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -419,7 +419,6 @@ struct alignas(64) SchedL2SwimlaneCounters { uint64_t sched_start_ts{0}; uint64_t sched_complete_cycle{0}; uint64_t sched_dispatch_cycle{0}; - uint64_t sched_wiring_cycle{0}; uint64_t sched_idle_cycle{0}; uint64_t sched_loop_count{0}; uint32_t phase_complete_count{0}; @@ -437,7 +436,6 @@ struct alignas(64) SchedL2SwimlaneCounters { uint64_t pop_hit_at_last_emit{0}; uint64_t pop_miss_at_last_emit{0}; #if PTO2_SCHED_PROFILING - uint32_t phase_wiring_count{0}; uint64_t complete_probe_count{0}; uint64_t complete_hit_count{0}; uint64_t sched_complete_perf_cycle{0}; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp index 1561acc56..9ac6f99e5 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp @@ -110,7 +110,6 @@ void PTO2SchedulerState::RingSchedState::reset_for_reuse( ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; advance_lock.store(0, std::memory_order_relaxed); - dep_deadlock_reported = false; dep_pool.reset_for_reuse(orch_err); #if PTO2_PROFILING dep_pool_snapshot_tail.store(1, std::memory_order_relaxed); @@ -122,17 +121,30 @@ void PTO2SchedulerState::RingSchedState::destroy() { ring = nullptr; } PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity) { int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]; + int32_t task_window_sizes[PTO2_MAX_RING_DEPTH]; for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { dep_pool_capacities[r] = dep_pool_capacity; + task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; } - return reserve_layout(arena, dep_pool_capacities); + return reserve_layout(arena, dep_pool_capacities, task_window_sizes); } PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]) { + int32_t task_window_sizes[PTO2_MAX_RING_DEPTH]; + for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { + task_window_sizes[r] = PTO2_TASK_WINDOW_SIZE; + } + return reserve_layout(arena, dep_pool_capacities, task_window_sizes); +} + +PTO2SchedulerLayout PTO2SchedulerState::reserve_layout( + DeviceArena &arena, const int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH], + const int32_t task_window_sizes[PTO2_MAX_RING_DEPTH] +) { + (void)task_window_sizes; PTO2SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; - layout.spsc_capacity = PTO2_WRIRING_QUEUE_SIZE; for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } @@ -143,13 +155,11 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); layout.off_early_dispatch_queue_slots = ready_queue_reserve_layout(arena, PTO2_EARLY_DISPATCH_QUEUE_SIZE); for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { - // Force a cache-line base so writes from scheduler thread 0 (sole - // writer of this ring's dep_pool) do not invalidate adjacent - // multi-threaded regions like ready_queue.slots. + // Force a cache-line base so Orch-side dep_pool writes do not invalidate + // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(PTO2DepListEntry), PTO2_ALIGN_SIZE); } - layout.off_wiring_spsc_buffer = PTO2SpscQueue::reserve_layout(arena, PTO2_WRIRING_QUEUE_SIZE); return layout; } @@ -194,13 +204,6 @@ bool PTO2SchedulerState::init_data_from_layout( sched->ring_sched_states[r].dep_pool.init(dep_entries, layout.dep_pool_capacities[r], orch_err); } - if (!sched->wiring.queue.init_data_from_layout(arena, layout.off_wiring_spsc_buffer, layout.spsc_capacity)) { - return false; - } - sched->wiring.batch_count = 0; - sched->wiring.batch_index = 0; - sched->wiring.backoff_counter = 0; - return true; } @@ -223,12 +226,6 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void sched->dummy_ready_queue.reset_for_reuse(); sched->early_dispatch_queue.reset_for_reuse(); - sched->wiring.queue.reset_for_reuse(); - sched->wiring.batch_count = 0; - sched->wiring.batch_index = 0; - sched->wiring.backoff_counter = 0; - sched->wiring.orch_needs_drain.store(false, std::memory_order_relaxed); - sched->wiring.producer_blocked.store(0, std::memory_order_relaxed); sched->async_wait_list.reset_for_reuse(); (void)layout; } @@ -244,7 +241,6 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, sched->ring_sched_states[r].dep_pool.base = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); } - sched->wiring.queue.wire_arena_pointers(arena, layout.off_wiring_spsc_buffer); } void PTO2SchedulerState::destroy() { @@ -253,7 +249,6 @@ void PTO2SchedulerState::destroy() { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; } - sched->wiring.queue.destroy(); for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } @@ -513,7 +508,7 @@ PTO2RuntimeArenaLayout runtime_reserve_layout( task_window_sizes_i32[r] = static_cast(task_window_sizes[r]); } layout.offsets.orch = PTO2OrchestratorState::reserve_layout(arena, task_window_sizes_i32, dep_pool_capacities); - layout.offsets.sched = PTO2SchedulerState::reserve_layout(arena, dep_pool_capacities); + layout.offsets.sched = PTO2SchedulerState::reserve_layout(arena, dep_pool_capacities, task_window_sizes_i32); layout.offsets.off_runtime = arena.reserve(sizeof(PTO2Runtime), PTO2_ALIGN_SIZE); layout.offsets.off_mailbox = arena.reserve(sizeof(AICoreCompletionMailbox), alignof(AICoreCompletionMailbox)); diff --git a/src/a5/platform/include/common/l2_swimlane_profiling.h b/src/a5/platform/include/common/l2_swimlane_profiling.h index 04c6a1837..515a172bb 100644 --- a/src/a5/platform/include/common/l2_swimlane_profiling.h +++ b/src/a5/platform/include/common/l2_swimlane_profiling.h @@ -504,8 +504,7 @@ enum class L2SwimlaneSchedPhaseKind : uint32_t { // tasks_processed = subtasks published this iter. Release = 2, // Deferred-release drain (on_task_release work). // tasks_processed = slots released this iter. - Wire = 3, // drain_wiring_queue: pop wired tasks into ready queues. - // tasks_processed = wired count. + Wire = 3, // Legacy scheduler wiring phase, kept for old traces. Dummy = 4, // dummy_drain outer bar: covers handling of all dummies // popped this iter. tasks_processed = dummy_got count. EarlyDispatch = 5, // try_speculative_early_dispatch: speculative pre-staging diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md index 42f64299e..96ed56074 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/MULTI_RING.md @@ -135,12 +135,11 @@ struct RingSchedState { int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS - // Cache Line 1+: Thread 0 only (wiring dep_pool, cache-isolated) + // Cache Line 1+: Orch-side wiring dep_pool, cache-isolated alignas(64) PTO2DepListPool dep_pool; }; RingSchedState ring_sched_states[PTO2_MAX_RING_DEPTH]; -PTO2SpscQueue wiring_queue; // global SPSC queue: orchestrator pushes, scheduler thread 0 drains ``` `slot_states`, `task_window_size`, and `task_window_mask` are no longer duplicated — callers access them via `ring->get_slot_state_by_*()` and other ring header accessors. The ring pointer shares cache line 0 with `last_task_alive` and `advance_lock`. @@ -198,10 +197,10 @@ Dependency edges use `PTO2TaskSlotState*` pointers, which naturally span rings: ### 5.3 DepPool Reclamation -DepPool is exclusively managed by scheduler thread 0 (allocation during wiring, reclamation during watermark advancement): +DepPool entries are allocated by the orchestrator during Orch-side wiring and reclaimed during watermark advancement: ```text -// Called by scheduler thread 0 during wiring_queue drain: +// Called during ring watermark advancement: dep_pool_reclaim(ring_id): la = ring->fc.last_task_alive newest_consumed = la - 1 diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 363a8d1bb..25b9308fa 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -418,7 +418,7 @@ Key members: - `rings[PTO2_MAX_RING_DEPTH]`: per-ring `PTO2RingSet` (HeapRing + TaskRing + FaninPool). See [MULTI_RING.md §4.2](MULTI_RING.md). - `tensor_map`, `tensor_pool`: dependency tracking - `scope_tasks[]`, `scope_begins[]`, `scope_stack_top`: scope nesting stack (flat buffer partitioned by level) -- `scheduler`: pointer to scheduler state (for wiring queue and ready queue access) +- `scheduler`: pointer to scheduler state (for Orch-side wiring helpers and ready queue access) - `gm_heap_base`, `gm_heap_size`: GM heap for output buffers ### 7.2 Task Submission Flow (`PTO2OrchestratorState::submit_task`) @@ -430,17 +430,15 @@ Key members: | 2 | Initialize task descriptor + slot state, copy parameters | | 3 | **Lookup**: for each INPUT/INOUT param, search TensorMap for producers; collect producer pointers in `PTO2FaninBuilder` | | 4 | **Insert**: register OUTPUT/INOUT args in TensorMap | -| 5 | **Record fanin metadata**: store producer pointers in `payload->fanin_inline_slot_states[]` (+ spill pool if >64); increment each producer's `fanout_count` (no lock needed — single writer). This step runs **before** `payload.init()`. | -| 6 | **Push to wiring queue**: push to global `PTO2SpscQueue`; scheduler thread 0 asynchronously wires fanout edges (lock + dep_pool + early_finished check + ready push) | +| 5 | **Record fanin metadata**: store producer pointers in `payload->fanin_inline_slot_states[]` (+ spill pool if >64); claim each live producer by incrementing `fanout_count` under that producer's `fanout_lock`. This step runs **before** `payload.init()`. | +| 6 | **Orch-side wiring / ready publish**: the orchestrator wires live fanout edges into the per-ring dep_pool; zero-fanin and already-completed fanin tasks publish directly to ready queues | -> **Note**: Fanout wiring (Steps 4–7 in earlier versions) has been moved from the -> orchestrator submit hot path to the scheduler's global `wiring_queue` (SPSC). This reduces the -> orchestrator's shared L2 cache / memory bus pressure, as the orchestrator no longer -> acquires `fanout_lock` or allocates from `dep_pool` during submission. +> **Note**: Fanout wiring is now completed before publish in the orchestrator submit path. +> Scheduler threads consume ready queues directly. -### 7.3 Deferred Fanout Wiring (Scheduler Wiring Queue) +### 7.3 Orch-Side Fanout Wiring -The orchestrator pushes each submitted task to the global `scheduler->wiring_queue` (a wait-free SPSC queue). Scheduler thread 0 drains this queue in batches, deferring if the queue holds fewer than a full batch of items to reduce contention (unless a final flush is needed at end of execution). For each task: +The orchestrator completes fanout wiring before publishing a task to the ready queues. For each task with live producers: 1. Sets `fanin_count = N + 1` (+1 redundance to prevent premature readiness) 2. For each producer in `payload->fanin_slot_states[]`: @@ -449,7 +447,9 @@ The orchestrator pushes each submitted task to the global `scheduler->wiring_que - If not completed: prepends consumer to producer's `fanout_head` via `dep_pool.prepend` - **Releases** `fanout_lock` 3. Atomically releases the +1 redundance + early_finished count via `fanin_refcount.fetch_add` -4. If all deps satisfied: pushes task to ready queue +4. If all deps satisfied: pushes task to the routed ready queue + +Zero-fanin tasks and tasks whose claimed producers are already completed skip dep_pool entry allocation and publish directly to the routed ready queue. The scheduler's completion handler mirrors this: @@ -700,11 +700,9 @@ With `PTO2_SERIAL_ORCH_SCHED=1`, scheduler threads still wait for `runtime_init_ready_` first, then additionally wait for `orchestrator_done_` before entering `resolve_and_dispatch()`. The default is off, preserving the current overlapped orch/sched pipeline. Serial mode is intended for measurement -and debugging. During the serial wait, scheduler thread 0 may drain deferred -wiring records to prevent bounded wiring-queue backpressure, but it does not -dispatch AICore work until `orchestrator_done_` is set. Large graphs may still -require larger task-ring, heap, or dependency-pool capacity because no task -execution/reclaim happens during graph build. +and debugging. It does not dispatch AICore work until `orchestrator_done_` is +set. Large graphs may still require larger task-ring, heap, or dependency-pool +capacity because no task execution/reclaim happens during graph build. --- diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 50f734fee..1d8ee744b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -163,7 +163,7 @@ This project-defined flattened numbering is kept unchanged. 1. Validate submit arguments. 2. Allocate mixed-task ID and initialize descriptor/payload/slot_state once. 3. Lookup producers via TensorMap; collect fanin metadata and increment producers' `fanout_count`. -4. Push task to scheduler's wiring queue (scheduler thread 0 asynchronously wires fanout edges and determines readiness). +4. Wire fanout edges on the orchestrator side and publish ready tasks to the scheduler queues. 5. Dispatch all active lanes atomically when resources allow. 6. Aggregate completion and release downstream once. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp index 31c3e439b..cfdb88b33 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp @@ -274,9 +274,9 @@ static bool append_fanin_or_fail( // producer; ++'ing it would corrupt an unrelated task. // (2) Already CONSUMED in place — finished, output ready, no real edge. // In either case, adding it to the fanin and bumping fanout_count would leave - // a stale ++/release pair (wire_task drops the fanout edge but keeps the fanin - // slot, so on_task_release still release_producer()'s it) that desyncs the - // slot's refcount (rc != fc) and wedges in-order reclaim. Claiming a live + // a stale ++/release pair (Orch-side wiring drops the fanout edge but keeps + // the fanin slot, so on_task_release still release_producer()'s it) that + // desyncs the slot's refcount (rc != fc) and wedges in-order reclaim. Claiming a live // producer under the lock pins it: fanout_count now counts us, so it cannot // reach CONSUMED (rc == fc) until we release it in on_task_release, keeping the // slot's generation stable until then. check_and_handle_consumed flips @@ -340,6 +340,83 @@ static bool append_fanin_or_fail( return true; } +static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { + if (fanin_builder.count == 0) return true; + return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool { + return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED; + }); +} + +static void mark_orch_wired_no_edges(PTO2SchedulerState *sched, PTO2TaskSlotState &slot_state) { + auto &rss = sched->ring_sched_states[slot_state.ring_id]; + sched->mark_dep_pool_position(rss, slot_state); +} + +static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, PTO2TaskSlotState &slot_state, int32_t wfanin) { + PTO2SchedulerState *sched = orch->scheduler; + auto &rss = sched->ring_sched_states[slot_state.ring_id]; + + // dep_pool is orchestrator-exclusive (no lock). Block until space frees. + // A stall here is usually a *symptom* of the scheduler having stalled + // (e.g. a scheduler timeout latches sched_error_code); latching our own + // dep_pool overflow would mask that root cause. So bail without latching + // when any fatal is already set elsewhere, and only declare a genuine + // dep_pool deadlock after a backstop that runs STRICTLY LONGER than the + // scheduler timeout — so a scheduler-side fault (e.g. -100) surfaces first + // — with no forward progress and no other fault. + if (rss.dep_pool.available() >= wfanin) { + sched->wire_fanin_task(rss, slot_state, wfanin); + return true; + } + + // Backstop = scheduler timeout + alloc-deadlock slack. Must exceed the + // scheduler timeout so the scheduler's own fault (-100) is latched and + // observed here before this dep_pool overflow (-4). + const uint64_t backstop_cycles = + static_cast(PLATFORM_SCHEDULER_TIMEOUT_MS) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) + + PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES; + + int spin_count = 0; + int32_t prev_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire); + uint64_t block_cycle0 = 0; + bool block_timing = false; + while (rss.dep_pool.available() < wfanin) { + rss.dep_pool.reclaim(*rss.ring, prev_last_alive); + if (rss.dep_pool.available() >= wfanin) break; + + spin_count++; + int32_t cur_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire); + if (cur_last_alive > prev_last_alive) { + spin_count = 0; + prev_last_alive = cur_last_alive; + block_timing = false; + } else if ((spin_count & 1023) == 0) { + // A fatal latched elsewhere (scheduler timeout -> sched_error_code, + // or an earlier orch_error_code) is the root cause of the stall — + // unwind without latching dep_pool overflow so that code surfaces. + if (orch->sm_header->sched_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE || + orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { + orch->fatal = true; + return false; + } + // Absolute-time backstop, stable across contention. get_sys_cnt_aicpu + // is an MMIO read, so sample it only once per 1024 spins. + uint64_t now = get_sys_cnt_aicpu(); + if (!block_timing) { + block_cycle0 = now; + block_timing = true; + } else if (now - block_cycle0 >= backstop_cycles) { + orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW); + return false; + } + } + SPIN_WAIT_HINT(); + } + + sched->wire_fanin_task(rss, slot_state, wfanin); + return true; +} + static void scope_tasks_push(PTO2OrchestratorState *orch, PTO2TaskSlotState *task_slot_state); struct PTO2PreparedTask { @@ -435,8 +512,8 @@ static bool prepare_task( // Reset the fanout/fanin bookkeeping for this reuse. The allocator only // returns a slot whose previous occupant is CONSUMED and quiescent (alloc // spins until last_task_alive passes it; in-order reclaim + acquire load), - // and the slot is not published to any scheduler thread until the - // wiring.queue.push at the end of submit_task_common — so this reset is + // and the slot is not published to any scheduler thread until the Orch-side + // wiring publish at the end of submit_task_common — so this reset is // race-free. Doing it here (not relying on the scheduler's eager // reset-after-CONSUMED, which only covers the contiguously-reclaimed tail) // makes every reused slot self-clean, which lets the per-boot SM init skip @@ -453,10 +530,9 @@ static bool prepare_task( // here lets RingSchedState::init_data_from_layout() skip the // O(window_size) bind loop. Both writes hit the same 64B slot_state // cache line we're about to dirty below, so the extra cost is two - // stores on an already-hot line. Must precede the scheduler - // wiring.queue.push at the end of submit_task_common — that push is - // the first read of slot_state->task / slot_state->payload by another - // thread. + // stores on an already-hot line. Must precede the Orch-side wiring publish + // at the end of submit_task_common — that publish is the first read of + // slot_state->task / slot_state->payload by scheduler threads. out->slot_state->bind_buffers(out->payload, out->task); // Fields already reset by advance_ring_pointers (eager reset after CONSUMED): @@ -472,7 +548,7 @@ static bool prepare_task( static_cast(block_num * __builtin_popcount(active_mask.core_mask())); out->slot_state->logical_block_num = block_num; out->slot_state->active_mask = active_mask; - // fanin_count is set by scheduler during wiring + // fanin_count is set during Orch-side wiring scope_tasks_push(orch, out->slot_state); return true; @@ -690,8 +766,8 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // Shared body for submit_task / submit_dummy_task. Caller has already validated // args.has_error, decided active_mask (empty for dummy), and resolved the per-slot // kernel_ids (all INVALID_KERNEL_ID for dummy). Performs tensormap sync, fanin -// computation (explicit_deps + auto), output registration, slot init, and pushes -// to the scheduler wiring queue. +// computation (explicit_deps + auto), output registration, slot init, and +// Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id @@ -869,28 +945,25 @@ static TaskOutputTensors submit_task_common( CYCLE_COUNT_LAP(g_orch_args_cycle); - // === STEP 6: push to wiring queue === - // Deferred wiring: orchestrator only stores dependency metadata and increments - // fanout_count. The actual fanout_head wiring (lock + dep_pool + early_finished) - // is handled asynchronously by scheduler thread 0 via the wiring queue. - // Push to global wiring queue — scheduler sets fanin_count, wires fanout, checks readiness - if (!sched->wiring.queue.push(&cur_slot_state)) { - // producer_blocked is the wiring deadlock detector's "orchestrator is - // stuck in push" observable: set ONLY while we actually spin (queue - // full), cleared on exit, so the just-filled-then-scope_end case (push - // succeeded, no spin) never trips a false deadlock. Also poll the shared - // orch_error_code so a fatal latched by any party (e.g. that detector) - // breaks this otherwise-unbounded spin and unwinds orchestration. - sched->wiring.producer_blocked.store(1, std::memory_order_release); - while (!sched->wiring.queue.push(&cur_slot_state)) { - if (orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { - orch->fatal = true; - sched->wiring.producer_blocked.store(0, std::memory_order_release); - return result; - } - SPIN_WAIT_HINT(); + // === STEP 6: wire on the orchestrator side and publish readiness === + // Zero-fanin tasks and tasks whose claimed producers are already completed + // do not need fanout links or dep_pool entries. Tasks with live producers + // allocate fanout links here before any scheduler thread can dispatch them. + if (fanin_builder.count == 0) { + cur_slot_state.fanin_count = 1; + cur_slot_state.fanin_refcount.store(1, std::memory_order_release); + mark_orch_wired_no_edges(sched, cur_slot_state); + sched->push_ready_routed(&cur_slot_state); + } else if (all_claimed_fanin_completed(fanin_builder)) { + int32_t ready_seed = fanin_builder.count + 1; + cur_slot_state.fanin_count = ready_seed; + cur_slot_state.fanin_refcount.store(ready_seed, std::memory_order_release); + mark_orch_wired_no_edges(sched, cur_slot_state); + sched->push_ready_routed(&cur_slot_state); + } else { + if (!orch_wire_live_fanin_task(orch, cur_slot_state, fanin_builder.count)) { + return result; } - sched->wiring.producer_blocked.store(0, std::memory_order_release); } CYCLE_COUNT_LAP(g_orch_fanin_cycle); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h index 97f318d40..05fab8762 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_ring_buffer.h @@ -172,8 +172,7 @@ class PTO2TaskAllocator { prev_last_alive = last_alive; block_timing = false; } else if ((spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this otherwise-unbounded spin; the + // A fatal latched elsewhere breaks this otherwise-unbounded spin; the // caller maps the failed alloc to orch_mark_fatal. Polled on the // cold path only -- error_code_ptr_ is orch_error_code. if (error_code_ptr_ != nullptr && error_code_ptr_->load(std::memory_order_acquire) != PTO2_ERROR_NONE) { diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp index 83a44c957..459abafcc 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.cpp @@ -108,7 +108,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa constexpr int kSegmentCap = 64; const PTO2TaskSlotState *seg[kSegmentCap]; int seg_count = 0; - bool signaled = false; bool failed = false; auto wait_one_producer = [&](const PTO2TaskSlotState &slot) { @@ -119,8 +118,7 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa while (slot.task_state.load(std::memory_order_acquire) < PTO2_TASK_COMPLETED) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this wait; cold path only. + // A fatal latched elsewhere breaks this wait; cold path only. if (orch.sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { failed = true; return; @@ -147,8 +145,7 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa (slot.fanout_count & ~PTO2_FANOUT_SCOPE_BIT)) { SPIN_WAIT_HINT(); if ((++spin_count & 1023) == 0) { - // A fatal latched elsewhere (e.g. the scheduler-side wiring - // deadlock detector) breaks this wait; cold path only. + // A fatal latched elsewhere breaks this wait; cold path only. if (orch.sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { failed = true; return; @@ -186,10 +183,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa if (failed) return; } seg[seg_count++] = &s; - if (!signaled) { - orch.scheduler->wiring.orch_needs_drain.store(true, std::memory_order_release); - signaled = true; - } }; auto do_wait = [&]() { @@ -212,9 +205,6 @@ static bool wait_for_tensor_ready(PTO2Runtime *rt, const Tensor &tensor, bool wa }; do_wait(); - if (signaled) { - orch.scheduler->wiring.orch_needs_drain.store(false, std::memory_order_release); - } return !failed; } MAYBE_UNINITIALIZED_END diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h index fbb65a693..4300608c3 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2.h @@ -224,8 +224,8 @@ PTO2Runtime *runtime_init_data_from_layout( /** * Phase 3 — wire every arena-internal pointer field (rt->sm_handle, * rt->aicore_mailbox, orchestrator.{scope_tasks, scope_begins, scheduler, - * tensor_map.*, rings[].fanin_pool.base}, scheduler.{ready_queues, dep_pool, - * wiring.queue}) so each holds arena.base() + offset. Idempotent — runs on + * tensor_map.*, rings[].fanin_pool.base}, scheduler.{ready_queues, dep_pool}) + * so each holds arena.base() + offset. Idempotent — runs on * both host (writing host-mirror addresses) and AICPU (writing device * addresses) sides. */ diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h index 017db57f5..853de50b8 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h @@ -87,9 +87,6 @@ // Ready queue #define PTO2_READY_QUEUE_SIZE 65536 // Per-shape queue size -// Wiring queue -#define PTO2_WRIRING_QUEUE_SIZE 1024 // Per-shape queue size - // Fanin storage #define PTO2_FANIN_INLINE_CAP 64 @@ -342,7 +339,7 @@ struct alignas(64) PTO2TaskSlotState { // and dep_pool_mark to keep PTO2TaskSlotState at 64 bytes. std::atomic any_subtask_deferred{false}; uint8_t _async_pad{0}; - int32_t dep_pool_mark{0}; // Dep pool top after wiring (thread-0-only) + int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring std::atomic completed_subtasks{0}; // Each core completion increments by 1 int16_t total_required_subtasks{0}; // = logical_block_num * popcount(active_mask) diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h index 7cc866c08..98ab20f26 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h @@ -385,135 +385,6 @@ bool ready_queue_init_data_from_layout(PTO2ReadyQueue *queue, DeviceArena &arena void ready_queue_wire_arena_pointers(PTO2ReadyQueue *queue, DeviceArena &arena, size_t slots_off); void ready_queue_destroy(PTO2ReadyQueue *queue); -// ============================================================================= -// SPSC Queue (Single-Producer Single-Consumer, wait-free) -// ============================================================================= -// -// Bounded ring buffer optimized for the wiring queue use case: -// - Producer: orchestrator thread (push) -// - Consumer: scheduler thread 0 (pop_batch) -// -// Design based on Rigtorp's cached-index technique: each side caches -// the other's index locally, avoiding cross-core cache line bouncing -// on the hot path. Only when the local cache says "full" or "empty" -// does the thread issue an acquire load on the remote index. -// -// Memory layout: 5 cache-line-aligned fields ensure zero false sharing. - -struct alignas(64) PTO2SpscQueue { - // --- Producer cache lines (orchestrator thread) --- - alignas(64) std::atomic head_{0}; - alignas(64) uint64_t tail_cached_{0}; - - // --- Consumer cache lines (scheduler thread 0) --- - alignas(64) std::atomic tail_{0}; - alignas(64) uint64_t head_cached_{0}; - - // --- Shared Cacheline (read only) with mask and data ptr (immutable after init) --- - alignas(64) PTO2TaskSlotState **buffer_{nullptr}; - uint64_t mask_{0}; - - // Padding to exactly 5 cache lines - char padding[64 - sizeof(PTO2TaskSlotState **) - sizeof(uint64_t)]; - - // Reserve the backing buffer region on the supplied arena. Returns the - // region offset, to be passed to init_from_layout() after the arena is - // committed. Cache-line aligned: the buffer is shared between the - // orchestrator (push) and scheduler thread 0 (pop_batch), so its base - // must not false-share with neighboring regions. - static size_t reserve_layout(DeviceArena &arena, uint64_t capacity) { - return arena.reserve(capacity * sizeof(uintptr_t), PTO2_ALIGN_SIZE); - } - - // Writes everything except the arena-internal `buffer_` pointer field - // (zeros the slot pointer array, mask/head/tail). The host pre-builds the - // image without storing a host address in buffer_; the AICPU wires - // buffer_ at boot via wire_arena_pointers(). - bool init_data_from_layout(DeviceArena &arena, size_t buffer_off, uint64_t capacity) { - if (capacity == 0 || (capacity & (capacity - 1)) != 0) return false; - auto *buf = static_cast(arena.region_ptr(buffer_off)); - // calloc'd-equivalent: zero the slot pointers so spurious early pops - // observe nullptr. - for (uint64_t i = 0; i < capacity; i++) - buf[i] = nullptr; - mask_ = capacity - 1; - head_.store(0, std::memory_order_relaxed); - tail_.store(0, std::memory_order_relaxed); - tail_cached_ = 0; - head_cached_ = 0; - return true; - } - - // Wire the arena-internal pointer. Called by both host (with host arena) - // and AICPU (with device arena attached to the prebuilt image). - void wire_arena_pointers(DeviceArena &arena, size_t buffer_off) { - buffer_ = static_cast(arena.region_ptr(buffer_off)); - } - - void reset_for_reuse() { - uint64_t h = head_.load(std::memory_order_relaxed); - tail_.store(h, std::memory_order_relaxed); - tail_cached_ = h; - head_cached_ = h; - } - - // Arena owns the buffer; here we only forget our pointer. - void destroy() { buffer_ = nullptr; } - - // Push one item (producer only). Returns false if queue is full. - // Full condition: next_h - tail > mask_ (i.e. > capacity-1), so the - // effective usable capacity is capacity-1 (one slot is wasted as a - // sentinel to distinguish full from empty). uint64_t wrapping is safe - // since head and tail are monotonically increasing and subtraction - // wraps correctly. - bool push(PTO2TaskSlotState *item) { - uint64_t h = head_.load(std::memory_order_relaxed); - uint64_t next_h = h + 1; - if (next_h - tail_cached_ > mask_) { - tail_cached_ = tail_.load(std::memory_order_acquire); - if (next_h - tail_cached_ > mask_) { - return false; - } - } - buffer_[h & mask_] = item; - head_.store(next_h, std::memory_order_release); - return true; - } - - // Pop up to max_count items (consumer only). Returns actual count. - int pop_batch(PTO2TaskSlotState **out, int max_count) { - uint64_t t = tail_.load(std::memory_order_relaxed); - uint64_t avail = head_cached_ - t; - if (avail < static_cast(max_count)) { - head_cached_ = head_.load(std::memory_order_acquire); - avail = head_cached_ - t; - if (avail == 0) return 0; - } - int count = (avail < static_cast(max_count)) ? static_cast(avail) : max_count; - for (int i = 0; i < count; i++) { - out[i] = buffer_[(t + i) & mask_]; - } - tail_.store(t + count, std::memory_order_release); - return count; - } - - // Approximate size (used for backoff decisions, not exact). - uint64_t size() const { - uint64_t h = head_.load(std::memory_order_acquire); - uint64_t t = tail_.load(std::memory_order_acquire); - return h - t; - } - - // Full ⟺ the producer's next push() would fail: size has reached the - // usable capacity (mask_ = capacity - 1, one slot reserved as sentinel). - // Used by the wiring-queue deadlock detector to prove the orchestrator is - // blocked in push(). - bool full() const { return size() >= mask_; } -}; - -static_assert(sizeof(PTO2SpscQueue) == 5 * 64, "PTO2SpscQueue must be exactly 5 cache lines (320B)"); -// ============================================================================= - /** * Statistics returned by mixed-task completion processing */ @@ -533,9 +404,7 @@ struct PTO2SchedulerLayout { size_t off_ready_queue_slots[PTO2_NUM_RESOURCE_SHAPES]; size_t off_dummy_ready_queue_slots; size_t off_dep_pool_entries[PTO2_MAX_RING_DEPTH]; - size_t off_wiring_spsc_buffer; uint64_t ready_queue_capacity; - uint64_t spsc_capacity; int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]; }; @@ -557,12 +426,8 @@ struct PTO2SchedulerState { int32_t last_task_alive; std::atomic advance_lock; // multi-thread CAS - // --- Cache Line 1+: Thread 0 only (wiring dep_pool) --- + // --- Cache Line 1+: Orch-side wiring dep_pool --- alignas(64) PTO2DepListPool dep_pool; - // One-shot latch for the wiring-queue deadlock report (thread 0 only): - // the drain breaks on dep_pool exhaustion every call while wedged, so - // the tier-1 structural diagnostic is emitted once, not per call. - bool dep_deadlock_reported = false; #if PTO2_PROFILING // Published only for scope_stats; orchestrator must not read dep_pool's non-atomic counters directly. alignas(64) std::atomic dep_pool_snapshot_tail; @@ -625,41 +490,6 @@ struct PTO2SchedulerState { // the dispatch loop and completed inline -- never goes to AICore. PTO2ReadyQueue dummy_ready_queue; - // Wiring subsystem — groups all wiring-related state for cache-line isolation. - // - // Three cache-line regions by writer: - // 1. batch_* / backoff — thread 0 exclusive (local batch buffer) - // 2. queue — SPSC: orchestrator push, thread 0 pop - // 3. orch_needs_drain — orchestrator write, thread 0 read - struct alignas(64) WiringState { - static constexpr uint64_t BATCH_SIZE = 30; - static constexpr int BACKOFF_LIMIT = 32; - - // --- Thread 0 exclusive: local batch buffer + backoff --- - int batch_count = 0; - int batch_index = 0; - int backoff_counter = 0; - PTO2TaskSlotState *batch[BATCH_SIZE]; - - // --- SPSC queue: orchestrator (push) ↔ thread 0 (pop) --- - alignas(64) PTO2SpscQueue queue; - - // --- Orchestrator write, thread 0 read --- - alignas(64) std::atomic orch_needs_drain{false}; - // Set to 1 only while the orchestrator is actually spinning in - // queue.push() (queue full), cleared on a successful push. The wiring - // deadlock detector reads this as the producer-blocked observable: it - // proves the orchestrator is stuck BEFORE its scope_end, as opposed to - // having just filled the queue with its last in-scope push and being - // about to call scope_end (which would release the head -> no deadlock). - std::atomic producer_blocked{0}; - } wiring; - - static_assert( - offsetof(WiringState, queue) == 256, "WiringState: batch region must be exactly 4 cache lines before queue" - ); - static_assert(sizeof(WiringState) == 640, "WiringState must be exactly 10 cache lines (640B)"); - alignas(64) AsyncWaitList async_wait_list; // Statistics (cold path, isolated from hot-path fields) @@ -671,129 +501,6 @@ struct PTO2SchedulerState { // Inline hot-path methods // ========================================================================= - /** - * Drain wiring queue: pop submitted tasks and wire their fanout edges. - * Called by scheduler thread 0 each loop iteration. Sets fanin_count, - * acquires fanout_lock per producer, allocates dep_pool entries, and - * pushes ready tasks to the appropriate ready queue. - * - * @return Number of tasks wired this call. - */ - - int drain_wiring_queue(bool force_drain = false) { - int wired = 0; - - // Refill local batch buffer when exhausted. - if (wiring.batch_index >= wiring.batch_count) { - // Backoff: defer pop when queue holds fewer than a full batch, - // unless force_drain, orch_needs_drain, or backoff limit reached. - if (!force_drain && wiring.queue.size() < WiringState::BATCH_SIZE) { - if (!wiring.orch_needs_drain.load(std::memory_order_acquire) && - wiring.backoff_counter < WiringState::BACKOFF_LIMIT) { - wiring.backoff_counter++; - return 0; - } - } - wiring.backoff_counter = 0; - wiring.batch_count = wiring.queue.pop_batch(wiring.batch, WiringState::BATCH_SIZE); - wiring.batch_index = 0; - if (wiring.batch_count == 0) return 0; - } - - // Process tasks from local buffer in strict FIFO order. - while (wiring.batch_index < wiring.batch_count) { - PTO2TaskSlotState *ws = wiring.batch[wiring.batch_index]; - int ring_id = ws->ring_id; - auto &rss = ring_sched_states[ring_id]; - int32_t wfanin = ws->payload->fanin_actual_count; - - if (wfanin > 0 && rss.dep_pool.available() < wfanin) { - rss.dep_pool.reclaim(*rss.ring, rss.last_task_alive); - if (rss.dep_pool.available() < wfanin) { -#if PTO2_PROFILING - if (is_scope_stats_enabled()) { - rss.publish_dep_pool_snapshot(); - } -#endif - // dep_pool can't reclaim because the reclaim watermark is - // wedged. This runs on the scheduler thread, so unlike - // alloc()'s detector it cannot self-observe that the - // orchestrator is blocked; wiring.producer_blocked is the - // external certificate -- the orchestrator sets it ONLY while - // it is actually spinning in queue.push() (cleared on a - // successful push), so the "just filled the queue then called - // scope_end" case (push succeeded -> flag stays 0) cannot trip - // a false report. With the producer provably stuck in push - // (program-order before its scope_end) AND the head COMPLETED, - // all consumers released, scope still open (only scope_end - // frees it), scope_end can never run -> provable head-of-line - // deadlock. The producer-blocked gate also pins the head: - // scope_end has not run, so the scope-gated head cannot be - // CONSUMED/reset concurrently while we read it. - if (!rss.dep_deadlock_reported && wiring.producer_blocked.load(std::memory_order_acquire) != 0) { - int32_t last_alive = rss.last_task_alive; - PTO2TaskSlotState &h = rss.ring->get_slot_state_by_task_id(last_alive); - // Read the head under its fanout_lock: fanout_count is a - // lock-protected field, and one snapshot keeps the check - // and the report consistent. - h.lock_fanout(); - int32_t state = h.task_state.load(std::memory_order_acquire); - uint32_t fc = h.fanout_count; - uint32_t rc = h.fanout_refcount.load(std::memory_order_acquire); - h.unlock_fanout(); - bool head_scope_gated = (state == PTO2_TASK_COMPLETED) && (rc == (fc & ~PTO2_FANOUT_SCOPE_BIT)); - if (head_scope_gated) { - rss.dep_deadlock_reported = true; - report_wiring_deadlock(rss, wfanin, last_alive, state, fc, rc); - // Latch the shared fatal so both sides exit fast off - // one error code: the scheduler cold-path poll - // (handle_orchestrator_exit) emergency_shutdowns, and - // the orchestrator's push spin breaks out and unwinds. - if (rss.dep_pool.error_code_ptr != nullptr) { - int32_t expected = PTO2_ERROR_NONE; - rss.dep_pool.error_code_ptr->compare_exchange_strong( - expected, PTO2_ERROR_DEP_POOL_OVERFLOW, std::memory_order_acq_rel - ); - } - } - } - break; // not enough dep_pool space — keep remainder for next call - } - } - - wiring.batch_index++; - wire_task(rss, ws, wfanin); - wired++; - } - - return wired; - } - - // Tier-1 structural diagnostic for a provable wiring-queue deadlock (head - // COMPLETED + all consumers released + scope still open, dep_pool exhausted, - // orchestrator provably blocked in push). The head snapshot (state/fc/rc) is - // taken under fanout_lock by the caller and passed in, so the report agrees - // with the check and reads no lock-protected field unlocked. - void report_wiring_deadlock( - RingSchedState &rss, int32_t wfanin, int32_t last_alive, int32_t state, uint32_t fc, uint32_t rc - ) { - LOG_ERROR("========================================"); - LOG_ERROR("FATAL: Wiring-Queue Deadlock - Dep Pool Exhausted!"); - LOG_ERROR("========================================"); - LOG_ERROR("Head task %d COMPLETED, all consumers released, scope still open ->", last_alive); - LOG_ERROR("only scope_end can free it, but the orchestrator is blocked on a full wiring"); - LOG_ERROR("queue (in push, before its scope_end). Provable head-of-line deadlock."); - LOG_ERROR( - " Head task %d: state=%d, consumers=%u/%u, scope_released=%d", last_alive, state, - rc & ~PTO2_FANOUT_SCOPE_BIT, fc & ~PTO2_FANOUT_SCOPE_BIT, (rc & PTO2_FANOUT_SCOPE_BIT) ? 1 : 0 - ); - LOG_ERROR(" Dep pool: used=%d/%d, needed=%d entries", rss.dep_pool.used(), rss.dep_pool.capacity, wfanin); - LOG_ERROR("Solution:"); - LOG_ERROR(" The open scope's fanout exceeds the dep pool. Either split the scope, or"); - LOG_ERROR(" raise PTO2_RING_DEP_POOL (compile-time PTO2_DEP_LIST_POOL_SIZE)."); - LOG_ERROR("========================================"); - } - // Route a ready slot to the right global queue. Dummy tasks (empty // active_mask) live in dummy_ready_queue; everything else goes to the // per-shape ready_queues[]. @@ -806,39 +513,8 @@ struct PTO2SchedulerState { } } - /** - * Wire fanout edges for a single task. Sets fanin_count, acquires each - * producer's fanout_lock, allocates dep_pool entries for live producers, - * pushes the task to the ready queue once its fanin refcount is satisfied. - */ - void wire_task(RingSchedState &rss, PTO2TaskSlotState *ws, int32_t wfanin) { - PTO2TaskPayload *wp = ws->payload; - ws->fanin_count = wfanin + 1; - - if (wfanin != 0) { - int32_t early_finished = 0; - for_each_fanin_slot_state(*wp, [&](PTO2TaskSlotState *producer) { - producer->lock_fanout(); - int32_t pstate = producer->task_state.load(std::memory_order_acquire); - if (pstate >= PTO2_TASK_COMPLETED) { - early_finished++; - } else { - producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, ws); - } - producer->unlock_fanout(); - }); - - int32_t init_rc = early_finished + 1; - int32_t new_rc = ws->fanin_refcount.fetch_add(init_rc, std::memory_order_acq_rel) + init_rc; - if (new_rc >= ws->fanin_count) { - push_ready_routed(ws); - } - } else { - ws->fanin_refcount.fetch_add(1, std::memory_order_acq_rel); - push_ready_routed(ws); - } - - ws->dep_pool_mark = rss.dep_pool.top; + void mark_dep_pool_position(RingSchedState &rss, PTO2TaskSlotState &slot_state) { + slot_state.dep_pool_mark = rss.dep_pool.top; #if PTO2_PROFILING if (is_scope_stats_enabled()) { rss.publish_dep_pool_snapshot(); @@ -846,6 +522,35 @@ struct PTO2SchedulerState { #endif } + /** + * Wire live fanin edges from the orchestrator thread. The caller must have + * ensured rss.dep_pool has room for wfanin. Dep-pool mutation is Orch-only; + * producer fanout lists are still protected by each producer's fanout_lock. + */ + void wire_fanin_task(RingSchedState &rss, PTO2TaskSlotState &slot_state, int32_t wfanin) { + PTO2TaskPayload *payload = slot_state.payload; + slot_state.fanin_count = wfanin + 1; + + int32_t early_finished = 0; + for_each_fanin_slot_state(*payload, [&](PTO2TaskSlotState *producer) { + producer->lock_fanout(); + int32_t pstate = producer->task_state.load(std::memory_order_acquire); + if (pstate >= PTO2_TASK_COMPLETED) { + early_finished++; + } else { + producer->fanout_head = rss.dep_pool.prepend(producer->fanout_head, &slot_state); + } + producer->unlock_fanout(); + }); + + int32_t init_rc = early_finished + 1; + int32_t new_rc = slot_state.fanin_refcount.fetch_add(init_rc, std::memory_order_acq_rel) + init_rc; + mark_dep_pool_position(rss, slot_state); + if (new_rc >= slot_state.fanin_count) { + push_ready_routed(&slot_state); + } + } + void check_and_handle_consumed(PTO2TaskSlotState &slot_state) { // Read fanout_refcount/fanout_count and flip COMPLETED->CONSUMED under // fanout_lock. The orchestrator claims producers (fanout_count++) under the @@ -1146,7 +851,7 @@ struct PTO2SchedulerState { // === Cold-path API (defined in pto_scheduler.cpp) === // Phase 1: declare every sub-region (ready_queue slots, dummy queue slots, - // per-ring dep_pool entries, wiring SPSC buffer) on the supplied arena. + // per-ring dep_pool entries) on the supplied arena. // Capacities are baked into the returned layout; init_data_from_layout uses // the same values. static PTO2SchedulerLayout reserve_layout(DeviceArena &arena, int32_t dep_pool_capacity = PTO2_DEP_LIST_POOL_SIZE); @@ -1165,7 +870,7 @@ struct PTO2SchedulerState { // Phase 3b: write the arena-internal pointer fields // (ready_queues[].slots, dummy_ready_queue.slots, dep_pool.base for each - // ring, wiring.queue.buffer_). Called on both host and device sides. + // ring). Called on both host and device sides. void wire_arena_pointers(const PTO2SchedulerLayout &layout, DeviceArena &arena); // Forget per-region pointers; arena owns the backing memory. diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index 337cc5df4..b8ce43217 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -504,9 +504,8 @@ void SchedulerContext::log_l2_swimlane_summary(int32_t thread_idx, [[maybe_unuse cycles_to_us(sched_end_ts - l2_swimlane.sched_start_ts) ); - uint64_t sched_total = l2_swimlane.sched_wiring_cycle + l2_swimlane.sched_complete_cycle + - l2_swimlane.sched_scan_cycle + l2_swimlane.sched_dispatch_cycle + - l2_swimlane.sched_idle_cycle; + uint64_t sched_total = l2_swimlane.sched_complete_cycle + l2_swimlane.sched_scan_cycle + + l2_swimlane.sched_dispatch_cycle + l2_swimlane.sched_idle_cycle; if (sched_total == 0) sched_total = 1; { @@ -603,19 +602,6 @@ void SchedulerContext::log_l2_swimlane_summary(int32_t thread_idx, [[maybe_unuse l2_swimlane.sched_scan_cycle * 100.0 / sched_total ); -#if PTO2_SCHED_PROFILING - LOG_INFO_V9( - "Thread %d: wiring : %.3fus (%.1f%%) tasks=%d", thread_idx, - cycles_to_us(l2_swimlane.sched_wiring_cycle), l2_swimlane.sched_wiring_cycle * 100.0 / sched_total, - l2_swimlane.phase_wiring_count - ); -#else - LOG_INFO_V9( - "Thread %d: wiring : %.3fus (%.1f%%)", thread_idx, cycles_to_us(l2_swimlane.sched_wiring_cycle), - l2_swimlane.sched_wiring_cycle * 100.0 / sched_total - ); -#endif - LOG_INFO_V9( "Thread %d: idle : %.3fus (%.1f%%)", thread_idx, cycles_to_us(l2_swimlane.sched_idle_cycle), l2_swimlane.sched_idle_cycle * 100.0 / sched_total @@ -1044,15 +1030,6 @@ void SchedulerContext::bind_runtime(PTO2Runtime *rt) { void SchedulerContext::wait_for_orchestration_done_before_dispatch(Runtime *runtime, int32_t thread_idx) { while (!orchestration_done() && !completed_.load(std::memory_order_acquire)) { - if (thread_idx == 0 && sched_ != nullptr) { - // Use the wiring subsystem's normal batch/backoff policy while - // waiting. This still honors orch_needs_drain/producer_blocked - // signals without force-draining an empty queue every spin. - int wired = sched_->drain_wiring_queue(/*force_drain=*/false); - if (wired > 0) { - continue; - } - } if (sched_ != nullptr && sched_->sm_header != nullptr && check_idle_fatal_error(thread_idx, sched_->sm_header, runtime) == LoopAction::BREAK_LOOP) { break; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index 2ec464587..4cd58b7df 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -91,9 +91,8 @@ class SchedulerContext { // mode where rt is created by the orchestrator thread after init(). void bind_runtime(PTO2Runtime *rt); - // Serial orch->sched mode pre-dispatch wait. Thread 0 may drain deferred - // wiring to keep the bounded wiring queue from back-pressuring orchestration, - // but no AICore dispatch happens before orchestrator_done_. + // Serial orch->sched mode pre-dispatch wait. No AICore dispatch happens + // before orchestrator_done_. void wait_for_orchestration_done_before_dispatch(Runtime *runtime, int32_t thread_idx); // ========================================================================= @@ -261,8 +260,7 @@ class SchedulerContext { // True if mix tasks remain in the global MIX ready queue. Approximate — // PTO2ReadyQueue::size() (see pto_scheduler.h) snapshots its enqueue/dequeue // positions with std::memory_order_relaxed and may interleave with concurrent - // push/pop. Don't confuse with PTO2SpscQueue::size(), which uses acquire - // loads — that one isn't on this path. A stale read here causes at most one + // push/pop. A stale read here causes at most one // extra/missed AIC/AIV skip and self-corrects on the next loop iteration. bool has_residual_mix() const { return sched_->ready_queues[static_cast(PTO2ResourceShape::MIX)].size() > 0; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 1b6ad25d3..6d570baab 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -542,7 +542,7 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ for (int s = 0; s < L2SWIMLANE_NUM_QUEUE_SHAPES; s++) shared_out[s] = shared_cached[s]; }; - // Queue-mutating phases (Complete / Wire / Dummy) push newly-ready consumers + // Queue-mutating phases (Complete / Dummy) push newly-ready consumers // straight into the shared ready_queues[] (the local-first buffer is gone), // so their end-of-phase shared depth differs from their start. Force a fresh // re-sample for those emits — this also refreshes the per-iter cache so the @@ -685,40 +685,11 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ continue; } - // Phase 3: Drain wiring queue (thread 0 only) - int wired = 0; - if (thread_idx == 0) { - wired = sched_->drain_wiring_queue(orchestrator_done_.load(std::memory_order_relaxed)); - if (wired > 0) { - made_progress = true; -#if PTO2_SCHED_PROFILING - l2_swimlane.phase_wiring_count += wired; -#endif - } - } -#if PTO2_PROFILING - CYCLE_COUNT_LAP(l2_swimlane.sched_wiring_cycle); - // Wire outer phase: emit one bar covering this iter's drain_wiring_queue - // pass when it wired any tasks. tasks_processed = wired count. - if (l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES && wired > 0) { - int16_t phase_end_shared[L2SWIMLANE_NUM_QUEUE_SHAPES]; - capture_phase_end_fresh(phase_end_shared); - l2_swimlane_aicpu_record_sched_phase( - thread_idx, L2SwimlaneSchedPhaseKind::Wire, _t0_phase, _t1, l2_swimlane.sched_loop_count, - static_cast(wired), /*pop_hit=*/0, /*pop_miss=*/0, phase_start_shared, phase_end_shared - ); - for (int s = 0; s < L2SWIMLANE_NUM_QUEUE_SHAPES; s++) - phase_start_shared[s] = phase_end_shared[s]; - _t0_phase = _t1; - } -#endif - - // Phase 3b: Drain dummy ready queue (thread 0 only). + // Phase 3: Drain dummy ready queue (thread 0 only). // // Dependency-only tasks bypass AICore dispatch: they go through the // scheduler so fanin/fanout edges stay consistent, but completion is - // signalled inline here. Pinned to thread 0 to avoid cross-thread - // races and to keep cache hot near the wiring drain above. + // signalled inline here. Pinned to thread 0 to avoid cross-thread races. if (thread_idx == 0) { constexpr int DUMMY_DRAIN_BATCH = 16; PTO2TaskSlotState *dummy_batch[DUMMY_DRAIN_BATCH]; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h index 9ebd0552f..101c6fbc0 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_types.h @@ -421,7 +421,6 @@ struct alignas(64) SchedL2SwimlaneCounters { uint64_t sched_scan_cycle{0}; uint64_t sched_complete_cycle{0}; uint64_t sched_dispatch_cycle{0}; - uint64_t sched_wiring_cycle{0}; uint64_t sched_idle_cycle{0}; uint64_t sched_loop_count{0}; uint32_t phase_complete_count{0}; @@ -433,7 +432,6 @@ struct alignas(64) SchedL2SwimlaneCounters { uint64_t pop_hit_at_last_emit{0}; uint64_t pop_miss_at_last_emit{0}; #if PTO2_SCHED_PROFILING - uint32_t phase_wiring_count{0}; uint64_t complete_probe_count{0}; uint64_t complete_hit_count{0}; uint64_t sched_complete_perf_cycle{0}; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp index 707257626..f40251c5b 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/shared/pto_runtime2_init.cpp @@ -110,7 +110,6 @@ void PTO2SchedulerState::RingSchedState::reset_for_reuse( ring = pto2_sm_layout::ring_header_addr(sm_dev_base, ring_id); last_task_alive = 0; advance_lock.store(0, std::memory_order_relaxed); - dep_deadlock_reported = false; dep_pool.reset_for_reuse(orch_err); #if PTO2_PROFILING dep_pool_snapshot_tail.store(1, std::memory_order_relaxed); @@ -132,7 +131,6 @@ PTO2SchedulerLayout PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_capacities[PTO2_MAX_RING_DEPTH]) { PTO2SchedulerLayout layout{}; layout.ready_queue_capacity = PTO2_READY_QUEUE_SIZE; - layout.spsc_capacity = PTO2_WRIRING_QUEUE_SIZE; for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { layout.dep_pool_capacities[r] = dep_pool_capacities[r]; } @@ -142,13 +140,11 @@ PTO2SchedulerState::reserve_layout(DeviceArena &arena, const int32_t dep_pool_ca } layout.off_dummy_ready_queue_slots = ready_queue_reserve_layout(arena, PTO2_READY_QUEUE_SIZE); for (int r = 0; r < PTO2_MAX_RING_DEPTH; r++) { - // Force a cache-line base so writes from scheduler thread 0 (sole - // writer of this ring's dep_pool) do not invalidate adjacent - // multi-threaded regions like ready_queue.slots. + // Force a cache-line base so Orch-side dep_pool writes do not invalidate + // adjacent multi-threaded regions like ready_queue.slots. layout.off_dep_pool_entries[r] = arena.reserve(static_cast(dep_pool_capacities[r]) * sizeof(PTO2DepListEntry), PTO2_ALIGN_SIZE); } - layout.off_wiring_spsc_buffer = PTO2SpscQueue::reserve_layout(arena, PTO2_WRIRING_QUEUE_SIZE); return layout; } @@ -188,13 +184,6 @@ bool PTO2SchedulerState::init_data_from_layout( sched->ring_sched_states[r].dep_pool.init(dep_entries, layout.dep_pool_capacities[r], orch_err); } - if (!sched->wiring.queue.init_data_from_layout(arena, layout.off_wiring_spsc_buffer, layout.spsc_capacity)) { - return false; - } - sched->wiring.batch_count = 0; - sched->wiring.batch_index = 0; - sched->wiring.backoff_counter = 0; - return true; } @@ -216,12 +205,6 @@ void PTO2SchedulerState::reset_for_reuse(const PTO2SchedulerLayout &layout, void } sched->dummy_ready_queue.reset_for_reuse(); - sched->wiring.queue.reset_for_reuse(); - sched->wiring.batch_count = 0; - sched->wiring.batch_index = 0; - sched->wiring.backoff_counter = 0; - sched->wiring.orch_needs_drain.store(false, std::memory_order_relaxed); - sched->wiring.producer_blocked.store(0, std::memory_order_relaxed); sched->async_wait_list.reset_for_reuse(); (void)layout; } @@ -236,7 +219,6 @@ void PTO2SchedulerState::wire_arena_pointers(const PTO2SchedulerLayout &layout, sched->ring_sched_states[r].dep_pool.base = static_cast(arena.region_ptr(layout.off_dep_pool_entries[r])); } - sched->wiring.queue.wire_arena_pointers(arena, layout.off_wiring_spsc_buffer); } void PTO2SchedulerState::destroy() { @@ -245,7 +227,6 @@ void PTO2SchedulerState::destroy() { sched->ring_sched_states[r].destroy(); sched->ring_sched_states[r].dep_pool.base = nullptr; } - sched->wiring.queue.destroy(); for (int i = 0; i < PTO2_NUM_RESOURCE_SHAPES; i++) { ready_queue_destroy(&sched->ready_queues[i]); } diff --git a/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp b/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp index ea82e7703..0beebf601 100644 --- a/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp +++ b/tests/st/runtime_fatal_codes/kernels/orchestration/dep_pool_overflow_orch.cpp @@ -20,10 +20,11 @@ * spill allocator exhausts and the orchestrator latches DEP_POOL_OVERFLOW. * * PRODUCER_COUNT must exceed PTO2_FANIN_INLINE_CAP, otherwise every edge fits - * inline, the spill pool is never touched, and the run stalls into - * SCHEDULER_TIMEOUT (code 100) instead — see scheduler_timeout_orch.cpp, which - * pins exactly that boundary. The scope keeps all producer ids live so the - * consumer really takes PRODUCER_COUNT distinct fanin edges. + * inline and the spill pool is never touched. With Orch-side dependency wiring, + * already-completed producer fanins bypass the dep pool entirely, so the old + * inline-boundary scheduler-timeout fixture is no longer a stable public-API + * repro. The scope keeps all producer ids live so the consumer really takes + * PRODUCER_COUNT distinct fanin edges. */ #include diff --git a/tests/st/runtime_fatal_codes/kernels/orchestration/scheduler_timeout_orch.cpp b/tests/st/runtime_fatal_codes/kernels/orchestration/scheduler_timeout_orch.cpp deleted file mode 100644 index 294b9cef9..000000000 --- a/tests/st/runtime_fatal_codes/kernels/orchestration/scheduler_timeout_orch.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ - -/** - * Negative ST orchestration: PTO2_ERROR_SCHEDULER_TIMEOUT (code 100), sub_class=S3. - * - * Submits exactly PTO2_FANIN_INLINE_CAP (64) producer dummy tasks inside one - * open scope (so their ids stay live), then one consumer depending on every - * producer, with the dep pool pinned tiny via CallConfig.runtime_env. The count - * is chosen at the inline-cap boundary on purpose: all 64 fanin edges fit the - * consumer's inline fanin slots, so the fanin spill pool is NEVER allocated and - * this does not (and must not) latch DEP_POOL_OVERFLOW(4) — that needs > 64 - * edges, see dep_pool_overflow_orch.cpp. Instead the tiny ring_dep_pool starves - * the scheduler: the consumer's producers all retire (completed=64/65) and the - * consumer goes READY, but the scheduler cannot make forward progress - * dispatching it from the undersized pool, so it sits ready-but-idle and the - * AICPU no-progress watchdog latches SCHEDULER_TIMEOUT, classified S3 - * (ready-but-all-idle) — exactly the stall issue #1180's sub-classification - * diagnoses. Verified deterministic: it stays code 100 even with the watchdog - * raised to 30 s, and a generously sized dep pool completes cleanly. Window and - * heap are left ample so only the dep pool is the bottleneck. The test lowers - * PTO2_SCHEDULER_TIMEOUT_MS so the watchdog fires quickly. - */ - -#include - -#include "pto_orchestration_api.h" // NOLINT(build/include_subdir) - -static constexpr int32_t PRODUCER_COUNT = 64; // == PTO2_FANIN_INLINE_CAP (boundary: no spill) - -extern "C" { - -__attribute__((visibility("default"))) PTO2OrchestrationConfig aicpu_orchestration_config(const L2TaskArgs &orch_args) { - (void)orch_args; - return PTO2OrchestrationConfig{ - .expected_arg_count = 0, - }; -} - -__attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2TaskArgs &orch_args) { - (void)orch_args; - - uint32_t shape[1] = {1}; - TensorCreateInfo ci(shape, 1, DataType::INT32); - - PTO2_SCOPE() { - PTO2TaskId producers[PRODUCER_COUNT]; - for (int32_t i = 0; i < PRODUCER_COUNT; i++) { - L0TaskArgs args; - args.add_output(ci); - producers[i] = rt_submit_dummy_task(args).task_id(); - } - - // One consumer depending on every producer. PRODUCER_COUNT fanin edges - // fit inline (== cap), so they never spill; the tiny dep pool instead - // starves the scheduler's dispatch of the ready consumer. - L0TaskArgs consumer; - consumer.set_dependencies(producers, PRODUCER_COUNT); - rt_submit_dummy_task(consumer); - } -} - -} // extern "C" diff --git a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py index c7a88a37c..060677cd5 100644 --- a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py +++ b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py @@ -94,15 +94,6 @@ kernel="aiv/kernel_noop.cpp", marker="orch_error_code=7", ), - "scheduler_timeout": dict( - orch="scheduler_timeout_orch.cpp", - code=100, - runtime_env={"ring_dep_pool": 4}, - kernel=None, - # Fire the no-progress watchdog in well under a second (default 10 s). - env={"PTO2_SCHEDULER_TIMEOUT_MS": 500}, - marker="sub_class=S3", - ), "aicore_hang": dict( orch="aicore_hang_orch.cpp", code=100, @@ -181,20 +172,25 @@ # the ring capacity together, so the constant gap holds. # # -- Design-unreachable from the public API (defensive classifier branches) -- - # * SCHEDULER_TIMEOUT sub-classes S4/S5/UNKNOWN: the pure classifier is unit- + # * SCHEDULER_TIMEOUT sub-classes S3/S4/S5/UNKNOWN: the pure classifier is unit- # tested for all of them (classify_stall_detail priority table in # tests/ut/cpp/.../test_shared_memory.cpp), but the live states cannot be - # produced through the public API. S4 (dep-deadlock, pure WAIT) needs a - # dependency that never resolves with nothing running/ready; set_dependencies - # only references already-submitted tasks, so cycles are inexpressible, and a - # stuck producer is RUNNING (-> S1), never pure WAIT. S5 (orch-starvation) - # needs completed < total with empty rings, but total_tasks_ is the count of - # *submitted* tasks (sum of current_task_index), so once they retire - # completed == total and the watchdog never fires — a while-loop in the orch - # does not help. UNKNOWN is a bookkeeping-invariant violation (corruption). - # These are kept as defensive labels: if a future bug ever produces the state, - # the classifier names it correctly instead of mislabeling. S1/S3 are the - # reproducible ones (aicore_hang_orch.cpp / scheduler_timeout_orch.cpp). + # produced stably through the public API after Orch-side wiring. S3 used to + # be reachable by under-sizing the fanout dep_pool for a ready dummy consumer, + # but completed-producer fanins now bypass dep_pool entirely, while live + # producers block in Orch-side prewire and either recover or latch the + # orchestrator dep_pool error before the scheduler has an orch-done total to + # classify as ready-but-idle. S4 (dep-deadlock, pure WAIT) needs a dependency + # that never resolves with nothing running/ready; set_dependencies only + # references already-submitted tasks, so cycles are inexpressible, and a stuck + # producer is RUNNING (-> S1), never pure WAIT. S5 (orch-starvation) needs + # completed < total with empty rings, but total_tasks_ is the count of + # *submitted* tasks (sum of current_task_index), so once they retire completed + # == total and the watchdog never fires — a while-loop in the orch does not + # help. UNKNOWN is a bookkeeping-invariant violation (corruption). These are + # kept as defensive labels: if a future bug ever produces the state, the + # classifier names it correctly instead of mislabeling. S1 remains the + # reproducible scheduler-timeout e2e case (aicore_hang_orch.cpp). # # -- Reachable only by exhausting a fixed compile-time cap -- # * TENSORMAP_OVERFLOW (11): the tensormap entry pool (PTO2_TENSORMAP_POOL_SIZE diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 3df7fe08a..caa741177 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -547,7 +547,6 @@ add_a2a3_runtime_test(test_shared_memory a2a3/test_shared_memory.cpp) add_a2a3_runtime_test(test_a2a3_tensormap a2a3/test_tensormap.cpp) add_a2a3_runtime_test(test_fanin_pool a2a3/test_fanin_pool.cpp) add_a2a3_orchestrator_runtime_test(test_a2a3_orchestrator_fanin a2a3/test_orchestrator_fanin.cpp) -add_a2a3_runtime_test(test_spsc_queue a2a3/test_spsc_queue.cpp) add_a2a3_runtime_test(test_wiring a2a3/test_wiring.cpp) add_a2a3_runtime_test(test_aicore_completion_mailbox a2a3/test_aicore_completion_mailbox.cpp) add_a2a3_runtime_test(test_a2a3_scope_stats_collector a2a3/test_scope_stats_collector.cpp) @@ -570,7 +569,6 @@ add_a5_runtime_test(test_a5_shared_memory a5/test_shared_memory.cpp) add_a5_runtime_test(test_a5_tensormap a5/test_tensormap.cpp) add_a5_runtime_test(test_a5_fanin_pool a5/test_fanin_pool.cpp) add_a5_orchestrator_runtime_test(test_a5_orchestrator_fanin a5/test_orchestrator_fanin.cpp) -add_a5_runtime_test(test_a5_spsc_queue a5/test_spsc_queue.cpp) add_a5_runtime_test(test_a5_wiring a5/test_wiring.cpp) add_a5_runtime_test(test_a5_aicore_completion_mailbox a5/test_aicore_completion_mailbox.cpp) diff --git a/tests/ut/cpp/a2a3/test_spsc_queue.cpp b/tests/ut/cpp/a2a3/test_spsc_queue.cpp deleted file mode 100644 index 664342d01..000000000 --- a/tests/ut/cpp/a2a3/test_spsc_queue.cpp +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ -/** - * Unit tests for PTO2SpscQueue from pto_scheduler.h - * - * Tests the Rigtorp cached-index SPSC queue used as the orchestrator → - * scheduler wiring channel: - * - Basic push / pop_batch correctness - * - Full / empty detection (including cached-index lazy refresh) - * - Wrap-around via modulo indexing - * - Capacity is capacity-1 (one sentinel slot) - * - pop_batch partial reads - * - size() accuracy - */ - -#include - -#include -#include -#include - -#include "utils/device_arena.h" -#include "scheduler/pto_scheduler.h" - -// ============================================================================= -// Fixture -// ============================================================================= - -class SpscQueueTest : public ::testing::Test { -protected: - static constexpr uint64_t CAPACITY = 16; // must be power of 2 - - PTO2SpscQueue queue{}; - DeviceArena arena; - // Dummy slot states used as push values - alignas(64) PTO2TaskSlotState slots[64]{}; - - void SetUp() override { - memset(&queue, 0, sizeof(queue)); - const size_t off = PTO2SpscQueue::reserve_layout(arena, CAPACITY); - ASSERT_NE(arena.commit(), nullptr); - ASSERT_TRUE(queue.init_data_from_layout(arena, off, CAPACITY)); - queue.wire_arena_pointers(arena, off); - } - - void TearDown() override { - queue.destroy(); - arena.release(); - } -}; - -// ============================================================================= -// Initialization -// ============================================================================= - -TEST_F(SpscQueueTest, InitValidState) { - EXPECT_EQ(queue.size(), 0u); - EXPECT_EQ(queue.mask_, CAPACITY - 1); - EXPECT_NE(queue.buffer_, nullptr); -} - -TEST_F(SpscQueueTest, InitRejectsNonPowerOfTwo) { - // init_from_layout rejects non-power-of-two capacities. Use a fresh arena - // each time since reserve runs before commit. - PTO2SpscQueue bad{}; - DeviceArena local; - const size_t off = PTO2SpscQueue::reserve_layout(local, 1); // dummy reservation so commit succeeds - (void)off; - ASSERT_NE(local.commit(), nullptr); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 3)); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 7)); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 0)); -} - -TEST_F(SpscQueueTest, InitAcceptsPowerOfTwo) { - PTO2SpscQueue q{}; - DeviceArena local; - const size_t off4 = PTO2SpscQueue::reserve_layout(local, 4); - const size_t off1024 = PTO2SpscQueue::reserve_layout(local, 1024); - ASSERT_NE(local.commit(), nullptr); - EXPECT_TRUE(q.init_data_from_layout(local, off4, 4)); - q.destroy(); - EXPECT_TRUE(q.init_data_from_layout(local, off1024, 1024)); - q.destroy(); -} - -// ============================================================================= -// Basic push / pop -// ============================================================================= - -TEST_F(SpscQueueTest, PushPopSingle) { - EXPECT_TRUE(queue.push(&slots[0])); - EXPECT_EQ(queue.size(), 1u); - - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1); - EXPECT_EQ(out[0], &slots[0]); - EXPECT_EQ(queue.size(), 0u); -} - -TEST_F(SpscQueueTest, FIFOOrdering) { - for (int i = 0; i < 5; i++) { - ASSERT_TRUE(queue.push(&slots[i])); - } - - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - ASSERT_EQ(count, 5); - for (int i = 0; i < 5; i++) { - EXPECT_EQ(out[i], &slots[i]) << "FIFO order violated at i=" << i; - } -} - -TEST_F(SpscQueueTest, PopBatchPartial) { - for (int i = 0; i < 3; i++) { - queue.push(&slots[i]); - } - - // Request 5 but only 3 available - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - EXPECT_EQ(count, 3); -} - -TEST_F(SpscQueueTest, PopBatchEmpty) { - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - EXPECT_EQ(count, 0); -} - -// ============================================================================= -// Full detection -// ============================================================================= - -TEST_F(SpscQueueTest, FullReturnsFalse) { - // Usable capacity = CAPACITY - 1 = 15 - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - ASSERT_TRUE(queue.push(&slots[i])) << "push failed at i=" << i; - } - EXPECT_EQ(queue.size(), CAPACITY - 1); - - // Queue full - EXPECT_FALSE(queue.push(&slots[CAPACITY - 1])) << "Push to full queue must return false"; -} - -TEST_F(SpscQueueTest, UsableCapacityIsCapacityMinusOne) { - int pushed = 0; - while (queue.push(&slots[pushed % 64])) { - pushed++; - if (pushed > 100) break; // safety - } - EXPECT_EQ(pushed, static_cast(CAPACITY - 1)); -} - -// ============================================================================= -// Full then recover -// ============================================================================= - -TEST_F(SpscQueueTest, FullThenPopThenPush) { - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - queue.push(&slots[i]); - } - EXPECT_FALSE(queue.push(&slots[0])); - - // Pop one - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1); - - // Now push should succeed - EXPECT_TRUE(queue.push(&slots[0])); -} - -// ============================================================================= -// Wrap-around -// ============================================================================= - -TEST_F(SpscQueueTest, WrapAroundCorrectness) { - // Push-pop cycles to advance head/tail past capacity boundary - for (int cycle = 0; cycle < 100; cycle++) { - ASSERT_TRUE(queue.push(&slots[cycle % 64])) << "push failed at cycle=" << cycle; - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1) << "pop_batch failed at cycle=" << cycle; - EXPECT_EQ(out[0], &slots[cycle % 64]); - } - EXPECT_EQ(queue.size(), 0u); -} - -TEST_F(SpscQueueTest, WrapAroundBatchCorrectness) { - // Multiple cycles of batch push/pop across wrap boundary - for (int cycle = 0; cycle < 20; cycle++) { - int batch = 5; - for (int i = 0; i < batch; i++) { - ASSERT_TRUE(queue.push(&slots[(cycle * batch + i) % 64])); - } - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, batch); - ASSERT_EQ(count, batch); - for (int i = 0; i < batch; i++) { - EXPECT_EQ(out[i], &slots[(cycle * batch + i) % 64]); - } - } -} - -// ============================================================================= -// size() accuracy -// ============================================================================= - -TEST_F(SpscQueueTest, SizeTracksOperations) { - EXPECT_EQ(queue.size(), 0u); - - queue.push(&slots[0]); - EXPECT_EQ(queue.size(), 1u); - - queue.push(&slots[1]); - queue.push(&slots[2]); - EXPECT_EQ(queue.size(), 3u); - - PTO2TaskSlotState *out[2]; - queue.pop_batch(out, 2); - EXPECT_EQ(queue.size(), 1u); - - queue.pop_batch(out, 1); - EXPECT_EQ(queue.size(), 0u); -} - -// ============================================================================= -// Producer-consumer (two threads) -// ============================================================================= - -TEST_F(SpscQueueTest, TwoThreadProducerConsumer) { - constexpr int TOTAL = 10000; - std::vector consumed; - consumed.reserve(TOTAL); - - // Use a large pool of slot states for unique pointers - std::vector big_pool(TOTAL); - - std::thread producer([&]() { - for (int i = 0; i < TOTAL; i++) { - while (!queue.push(&big_pool[i])) { - // spin - } - } - }); - - std::thread consumer([&]() { - int total = 0; - PTO2TaskSlotState *out[32]; - while (total < TOTAL) { - int count = queue.pop_batch(out, 32); - for (int i = 0; i < count; i++) { - consumed.push_back(out[i]); - } - total += count; - } - }); - - producer.join(); - consumer.join(); - - ASSERT_EQ(consumed.size(), static_cast(TOTAL)); - // Verify FIFO order - for (int i = 0; i < TOTAL; i++) { - EXPECT_EQ(consumed[i], &big_pool[i]) << "FIFO violated at i=" << i; - } -} - -// ============================================================================= -// Cached index behavior -// ============================================================================= - -TEST_F(SpscQueueTest, CachedIndexLazyRefresh) { - // Fill queue - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - queue.push(&slots[i]); - } - - // Consumer pops all - PTO2TaskSlotState *out[16]; - int count = queue.pop_batch(out, CAPACITY); - EXPECT_EQ(count, static_cast(CAPACITY - 1)); - - // Producer's tail_cached_ is stale (still thinks queue is full) - // Next push should refresh tail_cached_ and succeed - EXPECT_TRUE(queue.push(&slots[0])); -} - -TEST_F(SpscQueueTest, CachedIndexConsumerRefresh) { - // Consumer calls pop_batch on empty queue (head_cached_ is 0) - PTO2TaskSlotState *out[1]; - EXPECT_EQ(queue.pop_batch(out, 1), 0); - - // Producer pushes - queue.push(&slots[0]); - - // Consumer's head_cached_ is stale, pop_batch must refresh - int count = queue.pop_batch(out, 1); - EXPECT_EQ(count, 1); - EXPECT_EQ(out[0], &slots[0]); -} diff --git a/tests/ut/cpp/a2a3/test_wiring.cpp b/tests/ut/cpp/a2a3/test_wiring.cpp index 93e1d04e1..0317bec66 100644 --- a/tests/ut/cpp/a2a3/test_wiring.cpp +++ b/tests/ut/cpp/a2a3/test_wiring.cpp @@ -11,7 +11,7 @@ /** * Unit tests for scheduler wiring and completion paths: * - * 1. wire_task() — fanout wiring, early-finished detection, + * 1. Orch-side wiring — fanout wiring, early-finished detection, * fanin_count initialization, ready push * 2. on_task_complete() — COMPLETED transition, fanout traversal, * consumer fanin release @@ -87,13 +87,30 @@ class WiringTest : public ::testing::Test { memset(&slot_pl, 0, sizeof(slot_pl)); slot.payload = &slot_pl; } + + void publish_no_fanin(PTO2TaskSlotState &slot) { + auto &rss = sched.ring_sched_states[slot.ring_id]; + slot.fanin_count = 1; + slot.fanin_refcount.store(1, std::memory_order_release); + sched.mark_dep_pool_position(rss, slot); + sched.push_ready_routed(&slot); + } + + void wire_fanin(PTO2TaskSlotState &slot, int32_t wfanin) { + auto &rss = sched.ring_sched_states[slot.ring_id]; + bool ok = rss.dep_pool.ensure_space(*rss.ring, wfanin); + if (ok) { + sched.wire_fanin_task(rss, slot, wfanin); + } + ASSERT_TRUE(ok); + } }; // ============================================================================= -// wire_task: no fanin (independent task) +// Orch-side publish: no fanin (independent task) // ============================================================================= -TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { +TEST_F(WiringTest, NoFaninTaskBecomesReady) { // A task with 0 actual fanins should immediately be pushed to ready queue alignas(64) PTO2TaskSlotState task_slot; alignas(64) PTO2TaskPayload payload; @@ -105,8 +122,7 @@ TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 0); + publish_no_fanin(task_slot); // fanin_count set to 0 + 1 = 1 (the wiring "+1" sentinel) EXPECT_EQ(task_slot.fanin_count, 1); @@ -120,7 +136,7 @@ TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { } // ============================================================================= -// wire_task: with fanin, all producers already completed (early-finished) +// Orch-side wiring: with fanin, all producers already completed (early-finished) // ============================================================================= TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { @@ -144,8 +160,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 2); + wire_fanin(task_slot, 2); // fanin_count = 2 + 1 = 3 EXPECT_EQ(task_slot.fanin_count, 3); @@ -159,7 +174,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { } // ============================================================================= -// wire_task: with fanin, producers still pending (task NOT ready) +// Orch-side wiring: with fanin, producers still pending (task NOT ready) // ============================================================================= TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { @@ -181,8 +196,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 2); + wire_fanin(task_slot, 2); // fanin_count = 3 (2 + 1) EXPECT_EQ(task_slot.fanin_count, 3); @@ -203,7 +217,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { } // ============================================================================= -// wire_task: mixed early-finished and pending producers +// Orch-side wiring: mixed early-finished and pending producers // ============================================================================= TEST_F(WiringTest, WireTaskMixedProducerStates) { @@ -225,8 +239,7 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 3); + wire_fanin(task_slot, 3); // fanin_count = 4 (3 + 1) EXPECT_EQ(task_slot.fanin_count, 4); @@ -395,11 +408,7 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { EXPECT_EQ(slot.fanout_head, nullptr); } -// ============================================================================= -// drain_wiring_queue: pushes tasks through SPSC queue -// ============================================================================= - -TEST_F(WiringTest, DrainWiringQueueProcessesTasks) { +TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { alignas(64) PTO2TaskSlotState task_slot; alignas(64) PTO2TaskPayload payload; memset(&payload, 0, sizeof(payload)); @@ -410,54 +419,8 @@ TEST_F(WiringTest, DrainWiringQueueProcessesTasks) { task_slot.payload = &payload; task_slot.task = &desc; - // Push into wiring SPSC queue (orchestrator side) - ASSERT_TRUE(sched.wiring.queue.push(&task_slot)); - - // Drain (scheduler thread 0 side) - int wired = sched.drain_wiring_queue(true /* force_drain */); - EXPECT_EQ(wired, 1); - - // Task should be ready - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); - auto *popped = sched.ready_queues[static_cast(shape)].pop(); - EXPECT_EQ(popped, &task_slot); -} - -TEST_F(WiringTest, DrainWiringQueueBackoffDefers) { - alignas(64) PTO2TaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; - memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; - - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); - payload.fanin_actual_count = 0; - task_slot.payload = &payload; - task_slot.task = &desc; - - sched.wiring.queue.push(&task_slot); - - // Without force_drain, single item < BATCH_SIZE → backoff - sched.wiring.backoff_counter = 0; - int wired = sched.drain_wiring_queue(false); - EXPECT_EQ(wired, 0) << "Backoff should defer when queue < BATCH_SIZE"; - EXPECT_EQ(sched.wiring.backoff_counter, 1); -} - -TEST_F(WiringTest, DrainWiringQueueBackoffLimitForcesProcess) { - alignas(64) PTO2TaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; - memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; - - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); - payload.fanin_actual_count = 0; - task_slot.payload = &payload; - task_slot.task = &desc; - - sched.wiring.queue.push(&task_slot); - - // Set backoff at limit → should process - sched.wiring.backoff_counter = PTO2SchedulerState::WiringState::BACKOFF_LIMIT; - int wired = sched.drain_wiring_queue(false); - EXPECT_EQ(wired, 1) << "Backoff limit reached should force processing"; + auto &rss = sched.ring_sched_states[0]; + int32_t before_top = rss.dep_pool.top; + publish_no_fanin(task_slot); + EXPECT_EQ(task_slot.dep_pool_mark, before_top); } diff --git a/tests/ut/cpp/a5/test_spsc_queue.cpp b/tests/ut/cpp/a5/test_spsc_queue.cpp deleted file mode 100644 index 664342d01..000000000 --- a/tests/ut/cpp/a5/test_spsc_queue.cpp +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ -/** - * Unit tests for PTO2SpscQueue from pto_scheduler.h - * - * Tests the Rigtorp cached-index SPSC queue used as the orchestrator → - * scheduler wiring channel: - * - Basic push / pop_batch correctness - * - Full / empty detection (including cached-index lazy refresh) - * - Wrap-around via modulo indexing - * - Capacity is capacity-1 (one sentinel slot) - * - pop_batch partial reads - * - size() accuracy - */ - -#include - -#include -#include -#include - -#include "utils/device_arena.h" -#include "scheduler/pto_scheduler.h" - -// ============================================================================= -// Fixture -// ============================================================================= - -class SpscQueueTest : public ::testing::Test { -protected: - static constexpr uint64_t CAPACITY = 16; // must be power of 2 - - PTO2SpscQueue queue{}; - DeviceArena arena; - // Dummy slot states used as push values - alignas(64) PTO2TaskSlotState slots[64]{}; - - void SetUp() override { - memset(&queue, 0, sizeof(queue)); - const size_t off = PTO2SpscQueue::reserve_layout(arena, CAPACITY); - ASSERT_NE(arena.commit(), nullptr); - ASSERT_TRUE(queue.init_data_from_layout(arena, off, CAPACITY)); - queue.wire_arena_pointers(arena, off); - } - - void TearDown() override { - queue.destroy(); - arena.release(); - } -}; - -// ============================================================================= -// Initialization -// ============================================================================= - -TEST_F(SpscQueueTest, InitValidState) { - EXPECT_EQ(queue.size(), 0u); - EXPECT_EQ(queue.mask_, CAPACITY - 1); - EXPECT_NE(queue.buffer_, nullptr); -} - -TEST_F(SpscQueueTest, InitRejectsNonPowerOfTwo) { - // init_from_layout rejects non-power-of-two capacities. Use a fresh arena - // each time since reserve runs before commit. - PTO2SpscQueue bad{}; - DeviceArena local; - const size_t off = PTO2SpscQueue::reserve_layout(local, 1); // dummy reservation so commit succeeds - (void)off; - ASSERT_NE(local.commit(), nullptr); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 3)); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 7)); - EXPECT_FALSE(bad.init_data_from_layout(local, off, 0)); -} - -TEST_F(SpscQueueTest, InitAcceptsPowerOfTwo) { - PTO2SpscQueue q{}; - DeviceArena local; - const size_t off4 = PTO2SpscQueue::reserve_layout(local, 4); - const size_t off1024 = PTO2SpscQueue::reserve_layout(local, 1024); - ASSERT_NE(local.commit(), nullptr); - EXPECT_TRUE(q.init_data_from_layout(local, off4, 4)); - q.destroy(); - EXPECT_TRUE(q.init_data_from_layout(local, off1024, 1024)); - q.destroy(); -} - -// ============================================================================= -// Basic push / pop -// ============================================================================= - -TEST_F(SpscQueueTest, PushPopSingle) { - EXPECT_TRUE(queue.push(&slots[0])); - EXPECT_EQ(queue.size(), 1u); - - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1); - EXPECT_EQ(out[0], &slots[0]); - EXPECT_EQ(queue.size(), 0u); -} - -TEST_F(SpscQueueTest, FIFOOrdering) { - for (int i = 0; i < 5; i++) { - ASSERT_TRUE(queue.push(&slots[i])); - } - - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - ASSERT_EQ(count, 5); - for (int i = 0; i < 5; i++) { - EXPECT_EQ(out[i], &slots[i]) << "FIFO order violated at i=" << i; - } -} - -TEST_F(SpscQueueTest, PopBatchPartial) { - for (int i = 0; i < 3; i++) { - queue.push(&slots[i]); - } - - // Request 5 but only 3 available - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - EXPECT_EQ(count, 3); -} - -TEST_F(SpscQueueTest, PopBatchEmpty) { - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, 5); - EXPECT_EQ(count, 0); -} - -// ============================================================================= -// Full detection -// ============================================================================= - -TEST_F(SpscQueueTest, FullReturnsFalse) { - // Usable capacity = CAPACITY - 1 = 15 - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - ASSERT_TRUE(queue.push(&slots[i])) << "push failed at i=" << i; - } - EXPECT_EQ(queue.size(), CAPACITY - 1); - - // Queue full - EXPECT_FALSE(queue.push(&slots[CAPACITY - 1])) << "Push to full queue must return false"; -} - -TEST_F(SpscQueueTest, UsableCapacityIsCapacityMinusOne) { - int pushed = 0; - while (queue.push(&slots[pushed % 64])) { - pushed++; - if (pushed > 100) break; // safety - } - EXPECT_EQ(pushed, static_cast(CAPACITY - 1)); -} - -// ============================================================================= -// Full then recover -// ============================================================================= - -TEST_F(SpscQueueTest, FullThenPopThenPush) { - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - queue.push(&slots[i]); - } - EXPECT_FALSE(queue.push(&slots[0])); - - // Pop one - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1); - - // Now push should succeed - EXPECT_TRUE(queue.push(&slots[0])); -} - -// ============================================================================= -// Wrap-around -// ============================================================================= - -TEST_F(SpscQueueTest, WrapAroundCorrectness) { - // Push-pop cycles to advance head/tail past capacity boundary - for (int cycle = 0; cycle < 100; cycle++) { - ASSERT_TRUE(queue.push(&slots[cycle % 64])) << "push failed at cycle=" << cycle; - PTO2TaskSlotState *out[1]; - int count = queue.pop_batch(out, 1); - ASSERT_EQ(count, 1) << "pop_batch failed at cycle=" << cycle; - EXPECT_EQ(out[0], &slots[cycle % 64]); - } - EXPECT_EQ(queue.size(), 0u); -} - -TEST_F(SpscQueueTest, WrapAroundBatchCorrectness) { - // Multiple cycles of batch push/pop across wrap boundary - for (int cycle = 0; cycle < 20; cycle++) { - int batch = 5; - for (int i = 0; i < batch; i++) { - ASSERT_TRUE(queue.push(&slots[(cycle * batch + i) % 64])); - } - PTO2TaskSlotState *out[5]; - int count = queue.pop_batch(out, batch); - ASSERT_EQ(count, batch); - for (int i = 0; i < batch; i++) { - EXPECT_EQ(out[i], &slots[(cycle * batch + i) % 64]); - } - } -} - -// ============================================================================= -// size() accuracy -// ============================================================================= - -TEST_F(SpscQueueTest, SizeTracksOperations) { - EXPECT_EQ(queue.size(), 0u); - - queue.push(&slots[0]); - EXPECT_EQ(queue.size(), 1u); - - queue.push(&slots[1]); - queue.push(&slots[2]); - EXPECT_EQ(queue.size(), 3u); - - PTO2TaskSlotState *out[2]; - queue.pop_batch(out, 2); - EXPECT_EQ(queue.size(), 1u); - - queue.pop_batch(out, 1); - EXPECT_EQ(queue.size(), 0u); -} - -// ============================================================================= -// Producer-consumer (two threads) -// ============================================================================= - -TEST_F(SpscQueueTest, TwoThreadProducerConsumer) { - constexpr int TOTAL = 10000; - std::vector consumed; - consumed.reserve(TOTAL); - - // Use a large pool of slot states for unique pointers - std::vector big_pool(TOTAL); - - std::thread producer([&]() { - for (int i = 0; i < TOTAL; i++) { - while (!queue.push(&big_pool[i])) { - // spin - } - } - }); - - std::thread consumer([&]() { - int total = 0; - PTO2TaskSlotState *out[32]; - while (total < TOTAL) { - int count = queue.pop_batch(out, 32); - for (int i = 0; i < count; i++) { - consumed.push_back(out[i]); - } - total += count; - } - }); - - producer.join(); - consumer.join(); - - ASSERT_EQ(consumed.size(), static_cast(TOTAL)); - // Verify FIFO order - for (int i = 0; i < TOTAL; i++) { - EXPECT_EQ(consumed[i], &big_pool[i]) << "FIFO violated at i=" << i; - } -} - -// ============================================================================= -// Cached index behavior -// ============================================================================= - -TEST_F(SpscQueueTest, CachedIndexLazyRefresh) { - // Fill queue - for (uint64_t i = 0; i < CAPACITY - 1; i++) { - queue.push(&slots[i]); - } - - // Consumer pops all - PTO2TaskSlotState *out[16]; - int count = queue.pop_batch(out, CAPACITY); - EXPECT_EQ(count, static_cast(CAPACITY - 1)); - - // Producer's tail_cached_ is stale (still thinks queue is full) - // Next push should refresh tail_cached_ and succeed - EXPECT_TRUE(queue.push(&slots[0])); -} - -TEST_F(SpscQueueTest, CachedIndexConsumerRefresh) { - // Consumer calls pop_batch on empty queue (head_cached_ is 0) - PTO2TaskSlotState *out[1]; - EXPECT_EQ(queue.pop_batch(out, 1), 0); - - // Producer pushes - queue.push(&slots[0]); - - // Consumer's head_cached_ is stale, pop_batch must refresh - int count = queue.pop_batch(out, 1); - EXPECT_EQ(count, 1); - EXPECT_EQ(out[0], &slots[0]); -} diff --git a/tests/ut/cpp/a5/test_wiring.cpp b/tests/ut/cpp/a5/test_wiring.cpp index 8d4b18cc6..101176ea7 100644 --- a/tests/ut/cpp/a5/test_wiring.cpp +++ b/tests/ut/cpp/a5/test_wiring.cpp @@ -11,7 +11,7 @@ /** * Unit tests for scheduler wiring and completion paths: * - * 1. wire_task() — fanout wiring, early-finished detection, + * 1. Orch-side wiring — fanout wiring, early-finished detection, * fanin_count initialization, ready push * 2. on_task_complete() — COMPLETED transition, fanout traversal, * consumer fanin release @@ -77,13 +77,30 @@ class WiringTest : public ::testing::Test { slot.logical_block_num = 1; slot.dep_pool_mark = 0; } + + void publish_no_fanin(PTO2TaskSlotState &slot) { + auto &rss = sched.ring_sched_states[slot.ring_id]; + slot.fanin_count = 1; + slot.fanin_refcount.store(1, std::memory_order_release); + sched.mark_dep_pool_position(rss, slot); + sched.push_ready_routed(&slot); + } + + void wire_fanin(PTO2TaskSlotState &slot, int32_t wfanin) { + auto &rss = sched.ring_sched_states[slot.ring_id]; + bool ok = rss.dep_pool.ensure_space(*rss.ring, wfanin); + if (ok) { + sched.wire_fanin_task(rss, slot, wfanin); + } + ASSERT_TRUE(ok); + } }; // ============================================================================= -// wire_task: no fanin (independent task) +// Orch-side publish: no fanin (independent task) // ============================================================================= -TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { +TEST_F(WiringTest, NoFaninTaskBecomesReady) { // A task with 0 actual fanins should immediately be pushed to ready queue alignas(64) PTO2TaskSlotState task_slot; alignas(64) PTO2TaskPayload payload; @@ -95,8 +112,7 @@ TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 0); + publish_no_fanin(task_slot); // fanin_count set to 0 + 1 = 1 (the wiring "+1" sentinel) EXPECT_EQ(task_slot.fanin_count, 1); @@ -110,7 +126,7 @@ TEST_F(WiringTest, WireTaskNoFaninBecomesReady) { } // ============================================================================= -// wire_task: with fanin, all producers already completed (early-finished) +// Orch-side wiring: with fanin, all producers already completed (early-finished) // ============================================================================= TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { @@ -134,8 +150,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 2); + wire_fanin(task_slot, 2); // fanin_count = 2 + 1 = 3 EXPECT_EQ(task_slot.fanin_count, 3); @@ -149,7 +164,7 @@ TEST_F(WiringTest, WireTaskAllProducersEarlyFinished) { } // ============================================================================= -// wire_task: with fanin, producers still pending (task NOT ready) +// Orch-side wiring: with fanin, producers still pending (task NOT ready) // ============================================================================= TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { @@ -171,8 +186,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 2); + wire_fanin(task_slot, 2); // fanin_count = 3 (2 + 1) EXPECT_EQ(task_slot.fanin_count, 3); @@ -193,7 +207,7 @@ TEST_F(WiringTest, WireTaskProducersPendingTaskNotReady) { } // ============================================================================= -// wire_task: mixed early-finished and pending producers +// Orch-side wiring: mixed early-finished and pending producers // ============================================================================= TEST_F(WiringTest, WireTaskMixedProducerStates) { @@ -215,8 +229,7 @@ TEST_F(WiringTest, WireTaskMixedProducerStates) { task_slot.payload = &payload; task_slot.task = &desc; - auto &rss = sched.ring_sched_states[0]; - sched.wire_task(rss, &task_slot, 3); + wire_fanin(task_slot, 3); // fanin_count = 4 (3 + 1) EXPECT_EQ(task_slot.fanin_count, 4); @@ -385,11 +398,7 @@ TEST_F(WiringTest, AdvanceRingPointersResetsSlots) { EXPECT_EQ(slot.fanout_head, nullptr); } -// ============================================================================= -// drain_wiring_queue: pushes tasks through SPSC queue -// ============================================================================= - -TEST_F(WiringTest, DrainWiringQueueProcessesTasks) { +TEST_F(WiringTest, NoEdgePublishRecordsDepPoolMark) { alignas(64) PTO2TaskSlotState task_slot; alignas(64) PTO2TaskPayload payload; memset(&payload, 0, sizeof(payload)); @@ -400,54 +409,8 @@ TEST_F(WiringTest, DrainWiringQueueProcessesTasks) { task_slot.payload = &payload; task_slot.task = &desc; - // Push into wiring SPSC queue (orchestrator side) - ASSERT_TRUE(sched.wiring.queue.push(&task_slot)); - - // Drain (scheduler thread 0 side) - int wired = sched.drain_wiring_queue(true /* force_drain */); - EXPECT_EQ(wired, 1); - - // Task should be ready - PTO2ResourceShape shape = task_slot.active_mask.to_shape(); - auto *popped = sched.ready_queues[static_cast(shape)].pop(); - EXPECT_EQ(popped, &task_slot); -} - -TEST_F(WiringTest, DrainWiringQueueBackoffDefers) { - alignas(64) PTO2TaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; - memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; - - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); - payload.fanin_actual_count = 0; - task_slot.payload = &payload; - task_slot.task = &desc; - - sched.wiring.queue.push(&task_slot); - - // Without force_drain, single item < BATCH_SIZE → backoff - sched.wiring.backoff_counter = 0; - int wired = sched.drain_wiring_queue(false); - EXPECT_EQ(wired, 0) << "Backoff should defer when queue < BATCH_SIZE"; - EXPECT_EQ(sched.wiring.backoff_counter, 1); -} - -TEST_F(WiringTest, DrainWiringQueueBackoffLimitForcesProcess) { - alignas(64) PTO2TaskSlotState task_slot; - alignas(64) PTO2TaskPayload payload; - memset(&payload, 0, sizeof(payload)); - PTO2TaskDescriptor desc{}; - - init_slot(task_slot, PTO2_TASK_PENDING, 0, 1); - payload.fanin_actual_count = 0; - task_slot.payload = &payload; - task_slot.task = &desc; - - sched.wiring.queue.push(&task_slot); - - // Set backoff at limit → should process - sched.wiring.backoff_counter = PTO2SchedulerState::WiringState::BACKOFF_LIMIT; - int wired = sched.drain_wiring_queue(false); - EXPECT_EQ(wired, 1) << "Backoff limit reached should force processing"; + auto &rss = sched.ring_sched_states[0]; + int32_t before_top = rss.dep_pool.top; + publish_no_fanin(task_slot); + EXPECT_EQ(task_slot.dep_pool_mark, before_top); }