diff --git a/docs/models/qwen35/unified-prefill-overlap.md b/docs/models/qwen35/unified-prefill-overlap.md index 845e80bbf..c581320ca 100644 --- a/docs/models/qwen35/unified-prefill-overlap.md +++ b/docs/models/qwen35/unified-prefill-overlap.md @@ -7,14 +7,22 @@ > per-stream cuBLAS route. The original HTTP table below predates that cap and > used the server's implicit Qwen3.5 max-batch default, so treat it as pre-cap > evidence until the HTTP cells are re-run with an explicit safe max batch. +> The `auto` scheduler policy may now be combined with `stream` (the #715 +> startup rejection is retired): `auto` shapes the per-step prefill budget, +> `stream` moves the chunk off the decode step. On A100-40GB (`70a600b7` + +> this change, vLLM 0.27.0 baseline), the combination dominates every +> single-lever config: 1024/256 c8 ITL p99 `65.5 → 34.2 ms`, c16 p99 +> `81.4 → 36.5 ms` (vLLM `83.3`), QPS16 TPOT `36.7 → 20.8 ms` (vLLM `23.6`) +> and QPS16 ITL p99 `101 → 42 ms` (vLLM `93.4`); the trade is open-loop TTFT +> (QPS16 `867 → 1828 ms`, vLLM `218`) and −15% QPS16 output throughput. > -> **Last touched:** 2026-08 +> **Last touched:** 2026-09 ## Preparation - **Read**: - `docs/index.md` - routes Qwen3.5 scheduler, accuracy, mixed-load, and profiling evidence. - - `docs/models/qwen35/adaptive-scheduler-policy.md` - `off` stays the default; `auto` is a separate opt-in policy and cannot be combined with overlap yet. + - `docs/models/qwen35/adaptive-scheduler-policy.md` - `off` stays the default; `auto` is a separate opt-in policy and may now be combined with `stream`. - `docs/models/qwen35/mixed-load-itl-470.md` - valid mixed load needs spare admission capacity and an observed prefill/decode intersection. - `docs/models/qwen35/accuracy.md` - `hf_golden_gate` is the numerical oracle; generated-text hashes are sanity evidence. - `docs/playbooks/bench-vs-vllm.md` and `docs/playbooks/profiling-guide.md` - bind A/B numbers and profiler claims to a fixed environment and raw artifacts. @@ -52,7 +60,6 @@ Unsupported combinations fail before model loading: - Qwen3.5 TP plus overlap; -- `--qwen35-scheduler-policy auto --decode-overlap stream`; - Qwen3.5 `--decode-overlap green-ctx`; - Qwen3.5 `--decode-overlap stream` with `--max-batch > 32`. diff --git a/pegainfer-qwen35/src/lib.rs b/pegainfer-qwen35/src/lib.rs index f46db580c..fdac53670 100644 --- a/pegainfer-qwen35/src/lib.rs +++ b/pegainfer-qwen35/src/lib.rs @@ -210,11 +210,6 @@ pub fn start_engine_with_capacity_policy_and_overlap( seed, .. } = options; - anyhow::ensure!( - scheduler_policy == Qwen35SchedulerPolicy::Off - || decode_overlap == Qwen35DecodeOverlap::Off, - "Qwen3.5 --decode-overlap=stream requires --qwen35-scheduler-policy=off" - ); if decode_overlap == Qwen35DecodeOverlap::SharedSm { anyhow::ensure!( max_batch <= MAX_SHARED_SM_DECODE_BATCH, @@ -355,29 +350,6 @@ mod tests { assert!(err.contains("single GPU")); } - #[test] - fn auto_policy_rejects_decode_overlap_before_loading_model() { - let err = start_engine_with_capacity_policy_and_overlap( - Path::new("unused-model-path"), - EngineLoadOptions { - enable_cuda_graph: true, - device_ordinals: vec![0], - parallel_config: None, - ep_backend: EpBackend::Nccl, - seed: 42, - }, - 1, - 1, - Qwen35SchedulerPolicy::Auto, - Qwen35DecodeOverlap::SharedSm, - ) - .err() - .expect("decode-overlap validation should reject auto policy") - .to_string(); - - assert!(err.contains("scheduler-policy=off")); - } - #[test] fn shared_sm_rejects_unsafe_decode_bucket_before_loading_model() { let err = start_engine_with_capacity_policy_and_overlap( diff --git a/pegainfer-qwen35/src/model_line.rs b/pegainfer-qwen35/src/model_line.rs index e6a47194f..0788a0dfd 100644 --- a/pegainfer-qwen35/src/model_line.rs +++ b/pegainfer-qwen35/src/model_line.rs @@ -122,13 +122,6 @@ impl ModelLine for Qwen35Line { ))); } } - if decode_overlap != Qwen35DecodeOverlap::Off - && matches!(cli.qwen35_scheduler_policy, CliQwen35SchedulerPolicy::Auto) - { - return Err(CliError::rule( - "Qwen3.5 --decode-overlap=stream requires --qwen35-scheduler-policy=off", - )); - } if ctx.shared.tp_size > 1 && decode_overlap != Qwen35DecodeOverlap::Off { return Err(CliError::rule( "--decode-overlap is single-GPU only; tp_size>1 has no prefill/decode overlap", @@ -242,8 +235,8 @@ mod tests { } #[test] - fn rejects_auto_policy_with_overlap() { - let error = validate_argv(&[ + fn accepts_auto_policy_with_overlap() { + validate_argv(&[ "pegainfer", "--decode-overlap", "stream", @@ -252,8 +245,7 @@ mod tests { "--qwen35-scheduler-policy", "auto", ]) - .expect_err("Qwen3.5 should reject auto policy with overlap"); - assert!(error.to_string().contains("scheduler-policy=off")); + .expect("Qwen3.5 should accept the auto policy together with stream overlap"); } #[test] diff --git a/pegainfer-qwen35/tests/e2e_scheduler.rs b/pegainfer-qwen35/tests/e2e_scheduler.rs index 9886399bb..6db911f4c 100644 --- a/pegainfer-qwen35/tests/e2e_scheduler.rs +++ b/pegainfer-qwen35/tests/e2e_scheduler.rs @@ -741,6 +741,64 @@ fn test_e2e_qwen35_shared_sm_last_decoder() { off.tokens }; + // The auto policy may now combine with Shared-SM overlap: it only reshapes + // per-step prefill chunk budgets, so chunk boundaries change while greedy + // tokens must not. + { + let auto_handle = pegainfer_qwen35::start_engine_with_capacity_policy_and_overlap( + Path::new(&model_path), + EngineLoadOptions { + enable_cuda_graph: true, + device_ordinals: vec![0], + seed: 42, + ..EngineLoadOptions::default() + }, + 4, + 8192, + pegainfer_qwen35::Qwen35SchedulerPolicy::Auto, + pegainfer_qwen35::Qwen35DecodeOverlap::SharedSm, + ) + .expect("Failed to start Qwen3.5 auto + shared-SM scheduler"); + let mut auto_load = auto_handle + .metrics_watch() + .expect("scheduler must expose metrics"); + + let mut auto_active_rx = submit_repeated_token_request( + &auto_handle, + "overlap-auto-last-decoder", + seed_token, + 512, + 128, + ); + wait_for_first_token(&mut auto_active_rx, "overlap-auto-last-decoder"); + let _ = drain_tokens(&mut auto_active_rx, "overlap-auto-last-decoder"); + let mut auto_prefill_rx = submit_repeated_token_request( + &auto_handle, + "overlap-auto-inflight-prefill", + seed_token, + 8192, + 2, + ); + wait_for_running_requests(&mut auto_load, 2, std::time::Duration::from_secs(10)); + assert_no_generated_event(&mut auto_prefill_rx, "overlap-auto-inflight-prefill"); + drop(auto_active_rx); + let auto_prefill = collect_generation_with_timeout( + &mut auto_prefill_rx, + "overlap-auto-inflight-prefill", + 0, + std::time::Duration::from_secs(30), + ); + assert_eq!( + auto_prefill.tokens.len(), + 2, + "auto + shared-SM in-flight prefill must finish after the last decoder is cancelled" + ); + assert_eq!( + auto_prefill.tokens, off_reference_tokens, + "auto + shared-SM overlapped prefill must match the greedy default-Off reference" + ); + } + let handle = pegainfer_qwen35::start_engine_with_capacity_policy_and_overlap( Path::new(&model_path), EngineLoadOptions {