feat(qwen35): keep the auto prefill budget running under stream overlap

docs/models/qwen35/adaptive-scheduler-policy.md

@@ -3,9 +3,12 @@
 > **TL;DR:** Issue #727 now lands Qwen3.5 scheduler policy plumbing with
 > conservative defaults: `off` remains the default, `auto` is explicit opt-in,
 > `--max-prefill-tokens` remains a hard per-step cap, and TP rejects `auto`
-> instead of silently downgrading to `off`.
+> instead of silently downgrading to `off`. With `--decode-overlap stream`,
+> `auto` keeps prefill running through the finishing window (the overlapped
+> chunk no longer stalls decode), trading a redundant QPS16 TPOT win for 31%
+> TTFT and 14% throughput at an unchanged tail.
 >
-> **Last touched:** 2026-07
+> **Last touched:** 2026-09
 
 ## Preparation
 
@@ -37,6 +40,7 @@
   - `Off` preserves the fixed base prefill budget.
   - No active decode or no in-flight prefill keeps the fixed budget.
   - Active requests with at most 4 tokens remaining get one decode-priority tick before the FIFO-front prefill continues.
+  - With `--decode-overlap stream` the finishing-window deferral is disabled: the overlapped chunk already runs off the decode step, so deferring only delays prefill. Measured on A100-40GB, single run (1024/128 QPS16): TTFT `1828 → 1264 ms`, output throughput `873 → 992 tok/s`, ITL p99 unchanged (`41.7 ms`), TPOT back to vLLM parity (`23.8 vs 23.6 ms`); see #727.
   - `Auto` never returns more than the configured base budget; `--max-prefill-tokens` stays a hard per-step cap.
   - Final chunks may shrink below the cap when fewer prompt tokens remain.
 

docs/models/qwen35/unified-prefill-overlap.md

@@ -13,8 +13,11 @@
 > this change, vLLM 0.27.0 baseline), the combination dominates every
 > single-lever config: 1024/256 c8 ITL p99 `65.5 → 34.2 ms`, c16 p99
 > `81.4 → 36.5 ms` (vLLM `83.3`), QPS16 TPOT `36.7 → 20.8 ms` (vLLM `23.6`)
-> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`); the trade is open-loop TTFT
-> (QPS16 `867 → 1828 ms`, vLLM `218`) and −15% QPS16 output throughput.
+> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`). With the combination enabled,
+> the `auto` finishing-window deferral is disabled (the overlapped chunk no
+> longer stalls decode), which reclaims most of its open-loop cost: QPS16 TTFT
+> `1828 → 1264 ms`, output throughput `873 → 992 tok/s`, ITL p99 unchanged;
+> TPOT returns to vLLM parity (`23.8 vs 23.6 ms`).
 >
 > **Last touched:** 2026-09
 

pegainfer-qwen35/src/scheduler/mod.rs

@@ -671,6 +671,10 @@ fn scheduler_loop(
     let mut prefilling: Vec<PrefillingRequest35> = Vec::new();
     let mut inflight_prefill: Option<InflightPrefill> = None;
     let max_batch = backend.max_batch();
+    let decode_overlap = matches!(
+        &backend,
+        SchedulerBackend::Single(single) if single.overlap_enabled()
+    );
 
     info!("scheduler ready (max_batch={})", max_batch);
 
@@ -946,6 +950,7 @@ fn scheduler_loop(
             prefill_budget,
             &active_decode,
             &prefill_queue,
+            decode_overlap,
         );
         let scheduled = take_prefill_chunks(&mut prefilling, step_prefill_budget);
         // ITL diagnostics (#470): capture the *actual* prefill-chunk token count
@@ -958,18 +963,15 @@ fn scheduler_loop(
         let plan = plan::build_next_plan(!active.is_empty(), scheduled);
         if let Some(plan) = plan {
             let itl_plan_kind = match &plan {
-                ExecutionPlan::Unified { .. } if matches!(&backend, SchedulerBackend::Single(single) if single.overlap_enabled()) => {
-                    "overlap_launch"
-                }
+                ExecutionPlan::Unified { .. } if decode_overlap => "overlap_launch",
                 ExecutionPlan::Unified { .. } => "unified",
                 ExecutionPlan::Prefill { .. } => "prefill",
                 ExecutionPlan::Decode => "decode",
             };
             let itl_step_start = itl_debug.then(Instant::now);
             let step_result = match plan {
                 ExecutionPlan::Unified { pending } => {
-                    if matches!(&backend, SchedulerBackend::Single(single) if single.overlap_enabled())
-                    {
+                    if decode_overlap {
                         launch_overlap_step(
                             &mut backend,
                             &mut active,

pegainfer-qwen35/src/scheduler/plan.rs

@@ -67,6 +67,7 @@ pub(super) fn choose_prefill_budget(
     base_budget: usize,
     active: &[ActiveDecodeState],
     prefilling: &[PrefillQueueState],
+    decode_overlap: bool,
 ) -> usize {
     assert!(
         base_budget > 0,
@@ -76,9 +77,13 @@ pub(super) fn choose_prefill_budget(
         return base_budget;
     }
 
-    if active
-        .iter()
-        .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS)
+    // With stream overlap the finishing request's last tokens keep ticking on
+    // the decode stream while the chunk runs aside, so the full prefill
+    // deferral would only delay prefill without buying decode latency.
+    if !decode_overlap
+        && active
+            .iter()
+            .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS)
     {
         return 0;
     }
@@ -312,7 +317,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Off, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Off,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "off keeps the fixed chunk budget"
         );
@@ -329,7 +340,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "auto preserves --max-prefill-tokens as a hard per-step cap"
         );
@@ -346,7 +363,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "standard serving cells with long outputs keep the fixed chunk path"
         );
@@ -363,7 +386,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             512,
             "auto may shrink the final chunk but never expands beyond the configured cap"
         );
@@ -386,10 +415,45 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             0,
             "a near-finished active request gets a decode-priority tick before a long prefill"
         );
+    }
+
+    #[test]
+    fn adaptive_prefill_budget_keeps_prefill_running_under_stream_overlap() {
+        let active = [
+            ActiveDecodeState {
+                generated_count: 252,
+                max_tokens: 256,
+            },
+            ActiveDecodeState {
+                generated_count: 16,
+                max_tokens: 4096,
+            },
+        ];
+        let prefilling = [PrefillQueueState {
+            remaining_tokens: 4096,
+        }];
+
+        assert_eq!(
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                true
+            ),
+            1024,
+            "with stream overlap the finishing window keeps prefill on the prefill stream instead of deferring it"
+        );
         assert!(
             matches!(
                 build_next_plan::<Pending>(true, vec![]),
@@ -410,7 +474,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             0,
             "decode-priority applies before even a final prefill chunk when an active request is finishing"
         );