diff --git a/docs/models/qwen35/adaptive-scheduler-policy.md b/docs/models/qwen35/adaptive-scheduler-policy.md index 83f73e2b7..afa5322da 100644 --- a/docs/models/qwen35/adaptive-scheduler-policy.md +++ b/docs/models/qwen35/adaptive-scheduler-policy.md @@ -3,9 +3,12 @@ > **TL;DR:** Issue #727 now lands Qwen3.5 scheduler policy plumbing with > conservative defaults: `off` remains the default, `auto` is explicit opt-in, > `--max-prefill-tokens` remains a hard per-step cap, and TP rejects `auto` -> instead of silently downgrading to `off`. +> instead of silently downgrading to `off`. With `--decode-overlap stream`, +> `auto` keeps prefill running through the finishing window (the overlapped +> chunk no longer stalls decode), trading a redundant QPS16 TPOT win for 31% +> TTFT and 14% throughput at an unchanged tail. > -> **Last touched:** 2026-07 +> **Last touched:** 2026-09 ## Preparation @@ -37,6 +40,7 @@ - `Off` preserves the fixed base prefill budget. - No active decode or no in-flight prefill keeps the fixed budget. - Active requests with at most 4 tokens remaining get one decode-priority tick before the FIFO-front prefill continues. + - With `--decode-overlap stream` the finishing-window deferral is disabled: the overlapped chunk already runs off the decode step, so deferring only delays prefill. Measured on A100-40GB, single run (1024/128 QPS16): TTFT `1828 → 1264 ms`, output throughput `873 → 992 tok/s`, ITL p99 unchanged (`41.7 ms`), TPOT back to vLLM parity (`23.8 vs 23.6 ms`); see #727. - `Auto` never returns more than the configured base budget; `--max-prefill-tokens` stays a hard per-step cap. - Final chunks may shrink below the cap when fewer prompt tokens remain. diff --git a/docs/models/qwen35/unified-prefill-overlap.md b/docs/models/qwen35/unified-prefill-overlap.md index c581320ca..9f79152e1 100644 --- a/docs/models/qwen35/unified-prefill-overlap.md +++ b/docs/models/qwen35/unified-prefill-overlap.md @@ -13,8 +13,13 @@ > this change, vLLM 0.27.0 baseline), the combination dominates every > single-lever config: 1024/256 c8 ITL p99 `65.5 → 34.2 ms`, c16 p99 > `81.4 → 36.5 ms` (vLLM `83.3`), QPS16 TPOT `36.7 → 20.8 ms` (vLLM `23.6`) -> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`); the trade is open-loop TTFT -> (QPS16 `867 → 1828 ms`, vLLM `218`) and −15% QPS16 output throughput. +> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`). With the combination enabled, +> the `auto` finishing-window deferral is disabled (the overlapped chunk no +> longer stalls decode), which reclaims most of its open-loop cost. The +> deferral-reclaim run is measured separately on `fb16fc15` + this change +> (same A100-40GB host and bench): QPS16 TTFT `1828 → 1264 ms`, output +> throughput `873 → 992 tok/s`, ITL p99 unchanged; TPOT returns to vLLM +> parity (`23.8 vs 23.6 ms`). > > **Last touched:** 2026-09 diff --git a/pegainfer-qwen35/src/scheduler/mod.rs b/pegainfer-qwen35/src/scheduler/mod.rs index f8531998e..d81b64f9b 100644 --- a/pegainfer-qwen35/src/scheduler/mod.rs +++ b/pegainfer-qwen35/src/scheduler/mod.rs @@ -671,6 +671,10 @@ fn scheduler_loop( let mut prefilling: Vec = Vec::new(); let mut inflight_prefill: Option = None; let max_batch = backend.max_batch(); + let decode_overlap = matches!( + &backend, + SchedulerBackend::Single(single) if single.overlap_enabled() + ); info!("scheduler ready (max_batch={})", max_batch); @@ -946,6 +950,7 @@ fn scheduler_loop( prefill_budget, &active_decode, &prefill_queue, + decode_overlap, ); let scheduled = take_prefill_chunks(&mut prefilling, step_prefill_budget); // ITL diagnostics (#470): capture the *actual* prefill-chunk token count @@ -958,9 +963,7 @@ fn scheduler_loop( let plan = plan::build_next_plan(!active.is_empty(), scheduled); if let Some(plan) = plan { let itl_plan_kind = match &plan { - ExecutionPlan::Unified { .. } if matches!(&backend, SchedulerBackend::Single(single) if single.overlap_enabled()) => { - "overlap_launch" - } + ExecutionPlan::Unified { .. } if decode_overlap => "overlap_launch", ExecutionPlan::Unified { .. } => "unified", ExecutionPlan::Prefill { .. } => "prefill", ExecutionPlan::Decode => "decode", @@ -968,8 +971,7 @@ fn scheduler_loop( let itl_step_start = itl_debug.then(Instant::now); let step_result = match plan { ExecutionPlan::Unified { pending } => { - if matches!(&backend, SchedulerBackend::Single(single) if single.overlap_enabled()) - { + if decode_overlap { launch_overlap_step( &mut backend, &mut active, diff --git a/pegainfer-qwen35/src/scheduler/plan.rs b/pegainfer-qwen35/src/scheduler/plan.rs index 2abf3f3a4..86bc66545 100644 --- a/pegainfer-qwen35/src/scheduler/plan.rs +++ b/pegainfer-qwen35/src/scheduler/plan.rs @@ -67,6 +67,7 @@ pub(super) fn choose_prefill_budget( base_budget: usize, active: &[ActiveDecodeState], prefilling: &[PrefillQueueState], + decode_overlap: bool, ) -> usize { assert!( base_budget > 0, @@ -76,9 +77,13 @@ pub(super) fn choose_prefill_budget( return base_budget; } - if active - .iter() - .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS) + // With stream overlap the finishing request's last tokens keep ticking on + // the decode stream while the chunk runs aside, so the full prefill + // deferral would only delay prefill without buying decode latency. + if !decode_overlap + && active + .iter() + .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS) { return 0; } @@ -312,7 +317,13 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Off, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Off, + 1024, + &active, + &prefilling, + false + ), 1024, "off keeps the fixed chunk budget" ); @@ -329,7 +340,13 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + false + ), 1024, "auto preserves --max-prefill-tokens as a hard per-step cap" ); @@ -346,7 +363,13 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + false + ), 1024, "standard serving cells with long outputs keep the fixed chunk path" ); @@ -363,7 +386,13 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + false + ), 512, "auto may shrink the final chunk but never expands beyond the configured cap" ); @@ -386,16 +415,51 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + false + ), 0, "a near-finished active request gets a decode-priority tick before a long prefill" ); + } + + #[test] + fn adaptive_prefill_budget_keeps_prefill_running_under_stream_overlap() { + let active = [ + ActiveDecodeState { + generated_count: 252, + max_tokens: 256, + }, + ActiveDecodeState { + generated_count: 16, + max_tokens: 4096, + }, + ]; + let prefilling = [PrefillQueueState { + remaining_tokens: 4096, + }]; + + assert_eq!( + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + true + ), + 1024, + "with stream overlap the finishing window keeps prefill on the prefill stream instead of deferring it" + ); assert!( matches!( - build_next_plan::(true, vec![]), - Some(ExecutionPlan::Decode) + build_next_plan::(true, vec![pending(1, 4096)]), + Some(ExecutionPlan::Unified { .. }) ), - "zero prefill budget turns the scheduler tick into decode-only work" + "the kept budget keeps the tick a unified prefill+decode step instead of a decode-only tick" ); } @@ -410,7 +474,13 @@ mod tests { }]; assert_eq!( - choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling), + choose_prefill_budget( + Qwen35SchedulerPolicy::Auto, + 1024, + &active, + &prefilling, + false + ), 0, "decode-priority applies before even a final prefill chunk when an active request is finishing" );