Skip to content

Commit ea65ecd

Browse files
ChaoWaoclaude
ChaoWao
and
claude
committed
fix: defer O-side dep_pool overflow to scheduler-side fatal codes
Move the live-fanin dep_pool wait out of PTO2DepListPool::ensure_space (fast 100k-spin fatal in <1ms) into the orchestrator's orch_wire_live_fanin_task, mirroring the fanin spill pool's ensure_space backstop: an absolute time budget plus a check for a fatal already latched elsewhere. With wiring now on the orchestrator, a workload that stalls the scheduler (e.g. the scheduler_timeout fatal-code test, which lowers PTO2_SCHEDULER_TIMEOUT_MS) used to see O fatal PTO2_ERROR_DEP_POOL_OVERFLOW (-4) at submit time within <1ms, masking the real scheduler-timeout code (-100). Now O bails without latching when sched_error_code / orch_error_code is already set, so the root-cause code surfaces. The residual dep_pool-deadlock backstop is sized to run strictly longer than the scheduler timeout (scheduler timeout + alloc-deadlock slack) so the scheduler's own fault is always observed first, instead of racing a fixed 500ms budget that ties with a 500ms scheduler timeout under load. Applied to a2a3 and a5. Test: tests/st/runtime_fatal_codes (sim, a2a3+a5) — scheduler_timeout surfaces -100, dep_pool_overflow still -4, all cases pass. Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 3b90ab6 commit ea65ecd

2 files changed

Lines changed: 112 additions & 14 deletions

File tree

‎src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp‎

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -349,8 +349,7 @@ static bool append_fanin_or_fail(
349349
static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) {
350350
if (fanin_builder.count == 0) return true;
351351
return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool {
352-
return producer != nullptr &&
353-
producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED;
352+
return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED;
354353
});
355354
}
356355

@@ -363,15 +362,65 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, PTO2TaskSlotS
363362
PTO2SchedulerState *sched = orch->scheduler;
364363
auto &rss = sched->ring_sched_states[slot_state.ring_id];
365364

366-
bool ok = rss.dep_pool.ensure_space(*rss.ring, wfanin);
367-
if (ok) {
365+
// dep_pool is orchestrator-exclusive (no lock). Block until space frees.
366+
// A stall here is usually a *symptom* of the scheduler having stalled
367+
// (e.g. a scheduler timeout latches sched_error_code); latching our own
368+
// dep_pool overflow would mask that root cause. So bail without latching
369+
// when any fatal is already set elsewhere, and only declare a genuine
370+
// dep_pool deadlock after a backstop that runs STRICTLY LONGER than the
371+
// scheduler timeout — so a scheduler-side fault (e.g. -100) surfaces first
372+
// — with no forward progress and no other fault.
373+
if (rss.dep_pool.available() >= wfanin) {
368374
sched->wire_fanin_task(rss, slot_state, wfanin);
375+
return true;
369376
}
370377

371-
if (!ok) {
372-
orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW);
378+
// Backstop = scheduler timeout + alloc-deadlock slack. Must exceed the
379+
// scheduler timeout so the scheduler's own fault (-100) is latched and
380+
// observed here before this dep_pool overflow (-4).
381+
const uint64_t backstop_cycles =
382+
static_cast<uint64_t>(PLATFORM_SCHEDULER_TIMEOUT_MS) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) +
383+
PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES;
384+
385+
int spin_count = 0;
386+
int32_t prev_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire);
387+
uint64_t block_cycle0 = 0;
388+
bool block_timing = false;
389+
while (rss.dep_pool.available() < wfanin) {
390+
rss.dep_pool.reclaim(*rss.ring, prev_last_alive);
391+
if (rss.dep_pool.available() >= wfanin) break;
392+
393+
spin_count++;
394+
int32_t cur_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire);
395+
if (cur_last_alive > prev_last_alive) {
396+
spin_count = 0;
397+
prev_last_alive = cur_last_alive;
398+
block_timing = false;
399+
} else if ((spin_count & 1023) == 0) {
400+
// A fatal latched elsewhere (scheduler timeout -> sched_error_code,
401+
// or an earlier orch_error_code) is the root cause of the stall —
402+
// unwind without latching dep_pool overflow so that code surfaces.
403+
if (orch->sm_header->sched_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE ||
404+
orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) {
405+
orch->fatal = true;
406+
return false;
407+
}
408+
// Absolute-time backstop, stable across contention. get_sys_cnt_aicpu
409+
// is an MMIO read, so sample it only once per 1024 spins.
410+
uint64_t now = get_sys_cnt_aicpu();
411+
if (!block_timing) {
412+
block_cycle0 = now;
413+
block_timing = true;
414+
} else if (now - block_cycle0 >= backstop_cycles) {
415+
orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW);
416+
return false;
417+
}
418+
}
419+
SPIN_WAIT_HINT();
373420
}
374-
return ok;
421+
422+
sched->wire_fanin_task(rss, slot_state, wfanin);
423+
return true;
375424
}
376425

377426
static void scope_tasks_push(PTO2OrchestratorState *orch, PTO2TaskSlotState *task_slot_state);

‎src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp‎

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -343,8 +343,7 @@ static bool append_fanin_or_fail(
343343
static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) {
344344
if (fanin_builder.count == 0) return true;
345345
return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool {
346-
return producer != nullptr &&
347-
producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED;
346+
return producer != nullptr && producer->task_state.load(std::memory_order_acquire) >= PTO2_TASK_COMPLETED;
348347
});
349348
}
350349

@@ -357,15 +356,65 @@ static bool orch_wire_live_fanin_task(PTO2OrchestratorState *orch, PTO2TaskSlotS
357356
PTO2SchedulerState *sched = orch->scheduler;
358357
auto &rss = sched->ring_sched_states[slot_state.ring_id];
359358

360-
bool ok = rss.dep_pool.ensure_space(*rss.ring, wfanin);
361-
if (ok) {
359+
// dep_pool is orchestrator-exclusive (no lock). Block until space frees.
360+
// A stall here is usually a *symptom* of the scheduler having stalled
361+
// (e.g. a scheduler timeout latches sched_error_code); latching our own
362+
// dep_pool overflow would mask that root cause. So bail without latching
363+
// when any fatal is already set elsewhere, and only declare a genuine
364+
// dep_pool deadlock after a backstop that runs STRICTLY LONGER than the
365+
// scheduler timeout — so a scheduler-side fault (e.g. -100) surfaces first
366+
// — with no forward progress and no other fault.
367+
if (rss.dep_pool.available() >= wfanin) {
362368
sched->wire_fanin_task(rss, slot_state, wfanin);
369+
return true;
363370
}
364371

365-
if (!ok) {
366-
orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW);
372+
// Backstop = scheduler timeout + alloc-deadlock slack. Must exceed the
373+
// scheduler timeout so the scheduler's own fault (-100) is latched and
374+
// observed here before this dep_pool overflow (-4).
375+
const uint64_t backstop_cycles =
376+
static_cast<uint64_t>(PLATFORM_SCHEDULER_TIMEOUT_MS) * (PLATFORM_PROF_SYS_CNT_FREQ / 1000) +
377+
PTO2_ALLOC_DEADLOCK_TIMEOUT_CYCLES;
378+
379+
int spin_count = 0;
380+
int32_t prev_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire);
381+
uint64_t block_cycle0 = 0;
382+
bool block_timing = false;
383+
while (rss.dep_pool.available() < wfanin) {
384+
rss.dep_pool.reclaim(*rss.ring, prev_last_alive);
385+
if (rss.dep_pool.available() >= wfanin) break;
386+
387+
spin_count++;
388+
int32_t cur_last_alive = rss.ring->fc.last_task_alive.load(std::memory_order_acquire);
389+
if (cur_last_alive > prev_last_alive) {
390+
spin_count = 0;
391+
prev_last_alive = cur_last_alive;
392+
block_timing = false;
393+
} else if ((spin_count & 1023) == 0) {
394+
// A fatal latched elsewhere (scheduler timeout -> sched_error_code,
395+
// or an earlier orch_error_code) is the root cause of the stall —
396+
// unwind without latching dep_pool overflow so that code surfaces.
397+
if (orch->sm_header->sched_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE ||
398+
orch->sm_header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) {
399+
orch->fatal = true;
400+
return false;
401+
}
402+
// Absolute-time backstop, stable across contention. get_sys_cnt_aicpu
403+
// is an MMIO read, so sample it only once per 1024 spins.
404+
uint64_t now = get_sys_cnt_aicpu();
405+
if (!block_timing) {
406+
block_cycle0 = now;
407+
block_timing = true;
408+
} else if (now - block_cycle0 >= backstop_cycles) {
409+
orch_mark_fatal(orch, PTO2_ERROR_DEP_POOL_OVERFLOW);
410+
return false;
411+
}
412+
}
413+
SPIN_WAIT_HINT();
367414
}
368-
return ok;
415+
416+
sched->wire_fanin_task(rss, slot_state, wfanin);
417+
return true;
369418
}
370419

371420
static void scope_tasks_push(PTO2OrchestratorState *orch, PTO2TaskSlotState *task_slot_state);

0 commit comments

Comments
 (0)