feat(qwen35): keep the auto prefill budget running under stream overlap

docs/models/qwen35/adaptive-scheduler-policy.md

@@ -37,6 +37,7 @@
   - `Off` preserves the fixed base prefill budget.
   - No active decode or no in-flight prefill keeps the fixed budget.
   - Active requests with at most 4 tokens remaining get one decode-priority tick before the FIFO-front prefill continues.
+  - With `--decode-overlap stream` the finishing-window deferral is disabled: the overlapped chunk already runs off the decode step, so deferring only delays prefill. Measured on A100-40GB (1024/128 QPS16): TTFT `1828 → 1264 ms`, output throughput `873 → 992 tok/s`, ITL p99 unchanged (`41.7 ms`), TPOT back to vLLM parity (`23.8 vs 23.6 ms`); see #727.
   - `Auto` never returns more than the configured base budget; `--max-prefill-tokens` stays a hard per-step cap.
   - Final chunks may shrink below the cap when fewer prompt tokens remain.
 

docs/models/qwen35/unified-prefill-overlap.md

@@ -13,8 +13,11 @@
 > this change, vLLM 0.27.0 baseline), the combination dominates every
 > single-lever config: 1024/256 c8 ITL p99 `65.5 → 34.2 ms`, c16 p99
 > `81.4 → 36.5 ms` (vLLM `83.3`), QPS16 TPOT `36.7 → 20.8 ms` (vLLM `23.6`)
-> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`); the trade is open-loop TTFT
-> (QPS16 `867 → 1828 ms`, vLLM `218`) and −15% QPS16 output throughput.
+> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`). With the combination enabled,
+> the `auto` finishing-window deferral is disabled (the overlapped chunk no
+> longer stalls decode), which reclaims most of its open-loop cost: QPS16 TTFT
+> `1828 → 1264 ms`, output throughput `873 → 992 tok/s`, ITL p99 unchanged;
+> TPOT returns to vLLM parity (`23.8 vs 23.6 ms`).
 >
 > **Last touched:** 2026-09
 

pegainfer-qwen35/src/scheduler/mod.rs

@@ -941,11 +941,16 @@ fn scheduler_loop(
                 remaining_tokens: req.req.prompt_tokens.len().saturating_sub(req.cursor),
             })
             .collect();
+        let decode_overlap = matches!(
+            &backend,
+            SchedulerBackend::Single(single) if single.overlap_enabled()
+        );
         let step_prefill_budget = choose_prefill_budget(
             scheduler_policy,
             prefill_budget,
             &active_decode,
             &prefill_queue,
+            decode_overlap,
         );
         let scheduled = take_prefill_chunks(&mut prefilling, step_prefill_budget);
         // ITL diagnostics (#470): capture the *actual* prefill-chunk token count

pegainfer-qwen35/src/scheduler/plan.rs

@@ -67,6 +67,7 @@ pub(super) fn choose_prefill_budget(
     base_budget: usize,
     active: &[ActiveDecodeState],
     prefilling: &[PrefillQueueState],
+    decode_overlap: bool,
 ) -> usize {
     assert!(
         base_budget > 0,
@@ -76,9 +77,13 @@ pub(super) fn choose_prefill_budget(
         return base_budget;
     }
 
-    if active
-        .iter()
-        .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS)
+    // With stream overlap the finishing request's last tokens keep ticking on
+    // the decode stream while the chunk runs aside, so the full prefill
+    // deferral would only delay prefill without buying decode latency.
+    if !decode_overlap
+        && active
+            .iter()
+            .any(|req| req.remaining_tokens() <= DECODE_FINISH_WINDOW_TOKENS)
     {
         return 0;
     }
@@ -312,7 +317,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Off, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Off,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "off keeps the fixed chunk budget"
         );
@@ -329,7 +340,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "auto preserves --max-prefill-tokens as a hard per-step cap"
         );
@@ -346,7 +363,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             1024,
             "standard serving cells with long outputs keep the fixed chunk path"
         );
@@ -363,7 +386,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             512,
             "auto may shrink the final chunk but never expands beyond the configured cap"
         );
@@ -386,10 +415,45 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             0,
             "a near-finished active request gets a decode-priority tick before a long prefill"
         );
+    }
+
+    #[test]
+    fn adaptive_prefill_budget_keeps_prefill_running_under_stream_overlap() {
+        let active = [
+            ActiveDecodeState {
+                generated_count: 252,
+                max_tokens: 256,
+            },
+            ActiveDecodeState {
+                generated_count: 16,
+                max_tokens: 4096,
+            },
+        ];
+        let prefilling = [PrefillQueueState {
+            remaining_tokens: 4096,
+        }];
+
+        assert_eq!(
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                true
+            ),
+            1024,
+            "with stream overlap the finishing window keeps prefill on the prefill stream instead of deferring it"
+        );
         assert!(
             matches!(
                 build_next_plan::<Pending>(true, vec![]),
@@ -410,7 +474,13 @@ mod tests {
         }];
 
         assert_eq!(
-            choose_prefill_budget(Qwen35SchedulerPolicy::Auto, 1024, &active, &prefilling),
+            choose_prefill_budget(
+                Qwen35SchedulerPolicy::Auto,
+                1024,
+                &active,
+                &prefilling,
+                false
+            ),
             0,
             "decode-priority applies before even a final prefill chunk when an active request is finishing"
         );