From b387352c92f9f86e8ebb4d4e3f185409b150198d Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 13:25:21 +0800
Subject: [PATCH 01/31] feat(qwen3): add n-gram (prompt-lookup) speculative
 proposer

Stateless prompt-lookup proposer: finds the most recent earlier occurrence
of the current token suffix in the running context and proposes the tokens
that followed it, longest-suffix-first with shorter-suffix fallback. No draft
model. This is the proposer half of n-gram speculative decoding; verification
and KV rollback in the decode path are a follow-up.

Pure-CPU and unit-tested (recent-match continuation, longest-suffix
preference, fallback, no-match, K cap, short context, config clamping).

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/lib.rs   |   1 +
 openinfer-qwen3-4b/src/ngram.rs | 186 ++++++++++++++++++++++++++++++++
 2 files changed, 187 insertions(+)
 create mode 100644 openinfer-qwen3-4b/src/ngram.rs

diff --git a/openinfer-qwen3-4b/src/lib.rs b/openinfer-qwen3-4b/src/lib.rs
index 23c8a43c..fe7a782c 100644
--- a/openinfer-qwen3-4b/src/lib.rs
+++ b/openinfer-qwen3-4b/src/lib.rs
@@ -8,6 +8,7 @@ mod config;
 mod executor;
 pub mod kernel_bench;
 mod lora;
+pub mod ngram;
 mod prefill;
 mod scheduler;
 mod unified_forward;
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
new file mode 100644
index 00000000..80fc69c1
--- /dev/null
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -0,0 +1,186 @@
+//! N-gram (prompt-lookup) speculative proposer for Qwen3.
+//!
+//! This proposes candidate continuation tokens **without a draft model**: it
+//! looks for the most recent earlier occurrence of the current token suffix in
+//! the running context (prompt + tokens generated so far) and proposes the
+//! tokens that followed that occurrence. The target model then verifies these
+//! candidates in a single forward pass and accepts the longest matching prefix.
+//!
+//! This is effective for repetitive / structured text (code, quoting, JSON,
+//! long-form copying) at zero extra model cost. When no n-gram match is found
+//! it proposes nothing and decoding falls back to the normal one-token step.
+//!
+//! Only the proposer is implemented here; verification and KV rollback are
+//! handled by the decode path and are intentionally out of scope for this
+//! module so the lookup logic can be unit-tested in isolation.
+
+/// Configuration for [`NgramProposer`].
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct NgramConfig {
+    /// Largest suffix length to match. Longer suffixes are tried first because
+    /// they are more specific (a longer match is a stronger predictor).
+    pub max_ngram: usize,
+    /// Smallest suffix length to match before giving up. Must be `>= 1`.
+    pub min_ngram: usize,
+    /// Maximum number of speculative tokens proposed per step.
+    pub num_speculative: usize,
+}
+
+impl Default for NgramConfig {
+    fn default() -> Self {
+        Self {
+            max_ngram: 3,
+            min_ngram: 1,
+            num_speculative: 4,
+        }
+    }
+}
+
+/// Stateless n-gram / prompt-lookup proposer.
+///
+/// The proposer keeps no history of its own; each call scans the supplied
+/// context, so the caller is free to reuse a single instance across requests.
+#[derive(Clone, Copy, Debug)]
+pub struct NgramProposer {
+    config: NgramConfig,
+}
+
+impl NgramProposer {
+    /// Create a proposer. `min_ngram` is clamped to at least 1 and to at most
+    /// `max_ngram`, and `max_ngram` to at least 1, so the config is always
+    /// usable regardless of caller input.
+    #[must_use]
+    pub fn new(config: NgramConfig) -> Self {
+        let max_ngram = config.max_ngram.max(1);
+        let min_ngram = config.min_ngram.clamp(1, max_ngram);
+        Self {
+            config: NgramConfig {
+                max_ngram,
+                min_ngram,
+                num_speculative: config.num_speculative,
+            },
+        }
+    }
+
+    /// Propose up to `num_speculative` continuation tokens for `context`
+    /// (the full token sequence so far: prompt followed by generated tokens).
+    ///
+    /// Returns an empty `Vec` when no usable n-gram match exists or when
+    /// `num_speculative == 0`. Tries the longest configured suffix first and
+    /// falls back to shorter suffixes.
+    #[must_use]
+    pub fn propose(&self, context: &[u32]) -> Vec<u32> {
+        if self.config.num_speculative == 0 {
+            return Vec::new();
+        }
+        let len = context.len();
+        // A match plus at least one following token needs `n + 1` tokens.
+        let max_n = self.config.max_ngram.min(len.saturating_sub(1));
+        for n in (self.config.min_ngram..=max_n).rev() {
+            let suffix = &context[len - n..];
+            if let Some(start) = latest_earlier_match(context, suffix) {
+                let pred_start = start + n;
+                let end = (pred_start + self.config.num_speculative).min(len);
+                debug_assert!(pred_start < end, "match must leave a following token");
+                return context[pred_start..end].to_vec();
+            }
+        }
+        Vec::new()
+    }
+}
+
+/// Find the latest start index `i < len - n` such that `context[i..i + n]`
+/// equals `suffix` (the trailing `n` tokens). "Latest" prefers the most recent
+/// context. Guaranteed to leave at least one following token at `i + n`.
+fn latest_earlier_match(context: &[u32], suffix: &[u32]) -> Option<usize> {
+    let n = suffix.len();
+    let len = context.len();
+    if n == 0 || len < n + 1 {
+        return None;
+    }
+    // Candidate starts run from `len - n - 1` (immediately before the trailing
+    // suffix) down to 0; the trailing occurrence itself (start `len - n`) is
+    // excluded so we never "predict" from the suffix we are matching.
+    (0..=len - n - 1)
+        .rev()
+        .find(|&i| &context[i..i + n] == suffix)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn proposer(max_ngram: usize, min_ngram: usize, k: usize) -> NgramProposer {
+        NgramProposer::new(NgramConfig {
+            max_ngram,
+            min_ngram,
+            num_speculative: k,
+        })
+    }
+
+    #[test]
+    fn proposes_continuation_after_recent_match() {
+        // ...1 2 3 1 2 3 1 2 ; suffix [1,2] last matched at index 3 -> follow [3,1,2]
+        let ctx = [1u32, 2, 3, 1, 2, 3, 1, 2];
+        let got = proposer(2, 1, 4).propose(&ctx);
+        assert_eq!(got, vec![3, 1, 2]);
+    }
+
+    #[test]
+    fn prefers_longest_suffix_match() {
+        // Both [9] (len1) and [4,9] (len2) recur, but the len-2 match is more
+        // specific and must win.
+        let ctx = [4u32, 9, 7, 1, 9, 0, 4, 9];
+        // suffix len2 = [4,9] earlier at index 0 -> follow [7,1,...]
+        let got = proposer(2, 1, 2).propose(&ctx);
+        assert_eq!(got, vec![7, 1]);
+    }
+
+    #[test]
+    fn falls_back_to_shorter_suffix() {
+        // No repeated 2-gram suffix, but the last token 5 occurred earlier.
+        let ctx = [5u32, 8, 2, 7, 5];
+        // suffix len2 = [7,5] has no earlier match; len1 = [5] matches at 0 -> [8]
+        let got = proposer(2, 1, 3).propose(&ctx);
+        assert_eq!(got, vec![8, 2, 7]);
+    }
+
+    #[test]
+    fn returns_empty_when_no_match() {
+        let ctx = [1u32, 2, 3, 4, 5];
+        assert!(proposer(3, 1, 4).propose(&ctx).is_empty());
+    }
+
+    #[test]
+    fn caps_proposal_at_num_speculative() {
+        let ctx = [1u32, 2, 1, 2, 1, 2, 1, 2];
+        let got = proposer(2, 1, 1).propose(&ctx);
+        assert_eq!(got.len(), 1);
+    }
+
+    #[test]
+    fn handles_short_context() {
+        assert!(proposer(3, 1, 4).propose(&[]).is_empty());
+        assert!(proposer(3, 1, 4).propose(&[7]).is_empty());
+    }
+
+    #[test]
+    fn zero_speculative_proposes_nothing() {
+        let ctx = [1u32, 2, 1, 2];
+        assert!(proposer(2, 1, 0).propose(&ctx).is_empty());
+    }
+
+    #[test]
+    fn new_clamps_invalid_config() {
+        // min_ngram > max_ngram and zero max are clamped, not panicked.
+        let p = NgramProposer::new(NgramConfig {
+            max_ngram: 0,
+            min_ngram: 5,
+            num_speculative: 2,
+        });
+        // With max_ngram clamped to 1, a single-token recurrence still works:
+        // suffix [3] recurs at index 0, so the follower [1, 3] is proposed.
+        let ctx = [3u32, 1, 3];
+        assert_eq!(p.propose(&ctx), vec![1, 3]);
+    }
+}

From 2a855695c9265e98c68db53e2329423864036c15 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 13:29:02 +0800
Subject: [PATCH 02/31] feat(qwen3): add greedy speculative acceptance + spec
 config

Adds the verification-side brain of n-gram speculative decoding:
`accept_greedy` commits the longest proposed prefix the target model agrees
with plus one model token (correction or bonus), so a verify step always
makes >=1 token of progress; `num_accepted` reports how many speculative KV
positions to keep on rollback. Also adds `SpeculativeConfig` (disabled by
default). Pure-CPU and unit-tested.

The GPU verify forward (multi-token decode returning per-position logits) and
the executor / kvbm KV-rollback wiring are a follow-up.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/lib.rs         |   1 +
 openinfer-qwen3-4b/src/speculative.rs | 131 ++++++++++++++++++++++++++
 2 files changed, 132 insertions(+)
 create mode 100644 openinfer-qwen3-4b/src/speculative.rs

diff --git a/openinfer-qwen3-4b/src/lib.rs b/openinfer-qwen3-4b/src/lib.rs
index fe7a782c..e4e74848 100644
--- a/openinfer-qwen3-4b/src/lib.rs
+++ b/openinfer-qwen3-4b/src/lib.rs
@@ -11,6 +11,7 @@ mod lora;
 pub mod ngram;
 mod prefill;
 mod scheduler;
+pub mod speculative;
 mod unified_forward;
 mod weights;
 
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
new file mode 100644
index 00000000..c70fb54b
--- /dev/null
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -0,0 +1,131 @@
+//! Speculative-decoding glue for Qwen3: config and greedy acceptance.
+//!
+//! The proposer ([`crate::ngram`]) suggests `K` candidate tokens; the target
+//! model is then run once on `[last_confirmed, c_0, .., c_{K-1}]` to produce a
+//! greedy token at each of the `K + 1` positions. [`accept_greedy`] turns those
+//! into the tokens to commit. The GPU verify forward and KV rollback that feed
+//! this live in the decode path and are intentionally not part of this module,
+//! so the acceptance rule stays pure and unit-tested.
+
+use crate::ngram::NgramConfig;
+
+/// Speculative-decoding configuration.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct SpeculativeConfig {
+    /// Master switch. When `false` the decode path runs the normal one-token
+    /// step and never calls the proposer.
+    pub enabled: bool,
+    /// N-gram proposer settings (ignored when `enabled` is `false`).
+    pub ngram: NgramConfig,
+}
+
+impl Default for SpeculativeConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            ngram: NgramConfig::default(),
+        }
+    }
+}
+
+/// Greedy speculative acceptance.
+///
+/// * `proposed` — the `K` candidate tokens from the proposer.
+/// * `target_argmax` — the target model's greedy token at each of the `K + 1`
+///   verify positions. `target_argmax[i]` is the model's prediction *after*
+///   consuming verify input `i`; `target_argmax[0]` is the token that follows
+///   the last confirmed token, and `target_argmax[K]` is the model's own
+///   continuation after the whole candidate run.
+///
+/// Returns the tokens to commit: the longest prefix of `proposed` that the
+/// model agrees with, followed by exactly one model token — the correction at
+/// the first divergence, or the bonus continuation when every candidate is
+/// accepted. The result is therefore always between `1` and `K + 1` tokens, so
+/// a verify step always makes at least one token of progress.
+///
+/// # Panics
+/// Panics (debug builds) if `target_argmax.len() != proposed.len() + 1`.
+#[must_use]
+pub fn accept_greedy(proposed: &[u32], target_argmax: &[u32]) -> Vec<u32> {
+    debug_assert_eq!(
+        target_argmax.len(),
+        proposed.len() + 1,
+        "verify must produce one greedy token per candidate plus a bonus"
+    );
+    let mut committed = Vec::with_capacity(proposed.len() + 1);
+    let mut i = 0;
+    while i < proposed.len() && proposed[i] == target_argmax[i] {
+        committed.push(proposed[i]);
+        i += 1;
+    }
+    // The model's own token at the first divergence (or the bonus continuation
+    // when the whole run was accepted). `i <= proposed.len() < target_argmax.len()`
+    // so this index is always valid.
+    committed.push(target_argmax[i]);
+    committed
+}
+
+/// Number of candidate tokens accepted before the committed bonus token, i.e.
+/// `accept_greedy(..).len() - 1`. Useful for KV rollback bookkeeping (how many
+/// of the speculatively written candidate positions to keep).
+#[must_use]
+pub fn num_accepted(proposed: &[u32], target_argmax: &[u32]) -> usize {
+    let mut i = 0;
+    while i < proposed.len() && proposed[i] == target_argmax[i] {
+        i += 1;
+    }
+    i
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn accepts_full_run_plus_bonus() {
+        // Model agrees with all 3 candidates, then emits its own bonus token.
+        let proposed = [10u32, 11, 12];
+        let argmax = [10u32, 11, 12, 13];
+        assert_eq!(accept_greedy(&proposed, &argmax), vec![10, 11, 12, 13]);
+        assert_eq!(num_accepted(&proposed, &argmax), 3);
+    }
+
+    #[test]
+    fn accepts_prefix_then_correction() {
+        // Candidates 10,11 match; 99 diverges (model wanted 22) -> commit 10,11,22.
+        let proposed = [10u32, 11, 99];
+        let argmax = [10u32, 11, 22, 33];
+        assert_eq!(accept_greedy(&proposed, &argmax), vec![10, 11, 22]);
+        assert_eq!(num_accepted(&proposed, &argmax), 2);
+    }
+
+    #[test]
+    fn rejects_first_candidate_commits_one() {
+        // First candidate already wrong -> only the model's own token commits.
+        let proposed = [10u32, 11, 12];
+        let argmax = [7u32, 8, 9, 10];
+        assert_eq!(accept_greedy(&proposed, &argmax), vec![7]);
+        assert_eq!(num_accepted(&proposed, &argmax), 0);
+    }
+
+    #[test]
+    fn empty_proposal_commits_model_token() {
+        // No candidates (proposer returned nothing): plain one-token decode.
+        let proposed: [u32; 0] = [];
+        let argmax = [42u32];
+        assert_eq!(accept_greedy(&proposed, &argmax), vec![42]);
+        assert_eq!(num_accepted(&proposed, &argmax), 0);
+    }
+
+    #[test]
+    fn always_commits_at_least_one_token() {
+        let proposed = [1u32, 2];
+        let argmax = [9u32, 9, 9];
+        assert!(!accept_greedy(&proposed, &argmax).is_empty());
+    }
+
+    #[test]
+    fn config_default_is_disabled() {
+        assert!(!SpeculativeConfig::default().enabled);
+    }
+}

From 1e8ff27e8e9275a61bda3d0f58d088695d999815 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 13:39:07 +0800
Subject: [PATCH 03/31] feat(kv-cache): expose speculative schedule/apply on
 RequestKv

kvbm's SchedulableSequence already implements speculative decode
(schedule_speculative + apply_speculative with automatic LIFO release of
rejected drafts' blocks), but RequestKv only wrapped the single-token decode
path. Add thin pass-throughs so the Qwen3 decode path can schedule a
speculative step and commit the accepted prefix; KV rollback for rejected
drafts is handled by kvbm, not the caller.

The verify forward reuses prefill_view(1 + num_draft_tokens). Unit-tested:
accepted-prefix commit advances kv_position by accepted.len(); over-commit
past the scheduled budget is rejected.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-kv-cache/src/pool.rs | 65 ++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/openinfer-kv-cache/src/pool.rs b/openinfer-kv-cache/src/pool.rs
index dc66fdf6..9e92abc3 100644
--- a/openinfer-kv-cache/src/pool.rs
+++ b/openinfer-kv-cache/src/pool.rs
@@ -284,6 +284,21 @@ impl RequestKv {
         self.seq.schedule_decode(&pool.block_manager)
     }
 
+    /// Schedule a speculative decode that may commit up to `max_commit_tokens`
+    /// tokens this step — the accepted prefix of the proposed continuation plus
+    /// the model's bonus/correction token. Pre-allocates for the worst case;
+    /// [`Self::apply_speculative`] releases any blocks the accepted count does
+    /// not use. The verify forward's `KvView` is built with
+    /// `prefill_view(1 + num_draft_tokens)` (the dangling token plus drafts).
+    pub fn schedule_speculative(
+        &mut self,
+        max_commit_tokens: usize,
+        pool: &BlockPool,
+    ) -> Result<(), ScheduleError> {
+        self.seq
+            .schedule_speculative(max_commit_tokens, &pool.block_manager)
+    }
+
     // ── Views (for forward pass) ───────────────────────────────────────
 
     /// Build an immutable `KvView` for prefill.
@@ -316,6 +331,22 @@ impl RequestKv {
             .map_err(|e| anyhow::anyhow!("apply_decode: {e}"))
     }
 
+    /// Commit the accepted tokens of a scheduled speculative step. `accepted`
+    /// is the verified continuation (accepted draft prefix plus the model's
+    /// bonus token); its length must be in `1..=max_commit_tokens`. Excess
+    /// blocks pre-allocated for rejected drafts are released (LIFO) by kvbm.
+    /// `kv_position` advances by `accepted.len()` and the last token becomes
+    /// the new dangling token for the next step.
+    pub fn apply_speculative(
+        &mut self,
+        accepted: &[u32],
+        pool: &BlockPool,
+    ) -> anyhow::Result<DecodeOutcome> {
+        self.seq
+            .apply_speculative(accepted, &pool.block_manager)
+            .map_err(|e| anyhow::anyhow!("apply_speculative: {e}"))
+    }
+
     pub fn release(&mut self) -> anyhow::Result<()> {
         self.seq
             .release()
@@ -508,4 +539,38 @@ mod tests {
              step_page_indices and this test can be retired"
         );
     }
+
+    /// A speculative step commits the accepted prefix and advances
+    /// `kv_position` by exactly `accepted.len()`; the rejected drafts'
+    /// pre-allocated blocks are released by kvbm without manual rollback.
+    #[test]
+    fn speculative_commits_accepted_prefix() {
+        let pool = BlockPool::new(16, 256).unwrap();
+        let mut kv = pool.new_request((0..8u32).collect(), 32, None);
+        kv.schedule_prefill(8, &pool).unwrap();
+        kv.apply_prefill(1000, &pool).unwrap();
+        assert_eq!(kv.kv_position(), 8);
+
+        // Schedule room for 4, accept only 2 (prefix + bonus).
+        kv.schedule_speculative(4, &pool).unwrap();
+        kv.apply_speculative(&[100, 101], &pool).unwrap();
+        assert_eq!(kv.kv_position(), 10, "advances by accepted.len()");
+        assert_eq!(kv.generated_tokens(), 3, "1 from prefill + 2 committed");
+    }
+
+    /// Committing more tokens than were scheduled is rejected (the verify
+    /// forward must never accept past the pre-allocated draft budget).
+    #[test]
+    fn speculative_rejects_overcommit() {
+        let pool = BlockPool::new(16, 256).unwrap();
+        let mut kv = pool.new_request((0..8u32).collect(), 32, None);
+        kv.schedule_prefill(8, &pool).unwrap();
+        kv.apply_prefill(1000, &pool).unwrap();
+
+        kv.schedule_speculative(2, &pool).unwrap();
+        let err = kv
+            .apply_speculative(&[100, 101, 102], &pool)
+            .expect_err("committing 3 of scheduled 2 must fail");
+        assert!(err.to_string().contains("apply_speculative"));
+    }
 }

From 28f9295acb96acbd1052a579de63d16b2d803f38 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 13:56:26 +0800
Subject: [PATCH 04/31] docs(qwen3): design note for n-gram speculative
 decoding

Captures the n-gram (prompt-lookup) speculative-decode design for Qwen3-4B:
per-step data flow, the lossless greedy-acceptance correctness oracle, the
token/KV accounting against kvbm's verified speculative contract, the layering
mirrored from vLLM V1 (proposer in the scheduler layer, GPU-side argmax,
scheduler reserves KV), and the remaining GPU verify-forward + executor wiring.
Records the building blocks already landed (proposer, acceptance, RequestKv
speculative pass-throughs) and the open interface questions.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 107 +++++++++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 docs/models/qwen3/ngram-speculative.md

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
new file mode 100644
index 00000000..f18803a6
--- /dev/null
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -0,0 +1,107 @@
+# Qwen3-4B n-gram speculative decoding (design)
+
+**TL;DR**: Draft-model-free speculative decoding for Qwen3-4B. An n-gram
+(prompt-lookup) proposer suggests `K` continuation tokens from the running
+context; the target model verifies them in one forward pass and commits the
+longest greedy-agreed prefix plus one model token. Greedy verification is
+**lossless** — output is bit-for-bit identical to plain greedy decode, just
+fewer forward passes on repetitive / structured text (code, quoting, JSON).
+The KV layer (kvbm) already supports speculative scheduling natively, so
+rejected drafts need no manual rollback.
+
+Last touched: 2026-06 · Status: **design + building blocks landed; GPU verify
+forward + executor/scheduler wiring pending.**
+
+## Why this is cheap to add here
+
+Three of the four pieces already exist or are trivial:
+
+- **Proposer** — `openinfer-qwen3-4b/src/ngram.rs` (`NgramProposer`). Done, unit-tested.
+- **Acceptance** — `openinfer-qwen3-4b/src/speculative.rs` (`accept_greedy`,
+  `num_accepted`, `SpeculativeConfig`). Done, unit-tested.
+- **KV scheduling / rollback** — kvbm's `SchedulableSequence` already implements
+  `schedule_speculative` / `apply_speculative` with **automatic LIFO release of
+  the blocks pre-allocated for rejected drafts**. Exposed on `RequestKv`
+  (`openinfer-kv-cache/src/pool.rs`). Done, unit-tested.
+- **GPU verify forward + orchestration** — the only remaining work (below).
+
+## Per-step data flow (single request, greedy)
+
+Layering mirrors vLLM V1 (proposer in the runner/scheduler layer that owns
+token history; the executor reserves KV, runs the target forward, and accepts):
+
+```
+scheduler owns the request's token history (prompt + generated)
+  1. drafts = NgramProposer.propose(history)        # K candidates; empty -> plain decode
+  2. verify_inputs = [d0, c0, .., c_{K-1}]           # d0 = last committed token (dangling)
+  3. RequestKv.schedule_speculative(K + 1)           # room for drafts + bonus token
+  4. argmax[0..K] = verify_forward(verify_inputs, prefill_view(1 + K))
+  5. committed = accept_greedy(drafts, argmax)        # m accepted drafts + 1 model token
+  6. RequestKv.apply_speculative(committed)           # kvbm releases rejected blocks
+  7. scheduler appends committed to history; applies stop / max_tokens;
+     committed.last() becomes the next step's d0
+```
+
+### Token / KV accounting (matches kvbm's verified contract)
+
+- The verify forward computes KV for `1 + K` positions (`d0` + `K` drafts) at
+  `base_pos = kv_position`. Structurally this is a prefill of `1 + K` tokens at
+  the current position, so the `KvView` is `RequestKv::prefill_view(1 + K)` and
+  the forward reuses the existing paged-prefill attention path.
+- `argmax[i]` is the model's greedy token *after* consuming `verify_inputs[i]`:
+  `argmax[0]` follows `d0` (the true next token), `argmax[i]` follows `c_{i-1}`,
+  `argmax[K]` is the bonus continuation. This index convention is exactly what
+  `accept_greedy(proposed = drafts, target_argmax = argmax)` expects.
+- `apply_speculative(committed)` advances `kv_position` by `committed.len()`
+  (`m + 1`); kvbm LIFO-drops the over-allocated blocks. `committed.last()` (the
+  model token) becomes the new dangling token. `schedule_speculative(K + 1)`
+  guarantees `m + 1 <= K + 1`.
+
+### Why it is lossless
+
+`accept_greedy` only keeps the prefix where `draft[i] == argmax[i]` and then
+appends one of the model's own tokens, so the committed sequence is identical
+to what plain greedy decode would have produced one token at a time. This gives
+a free correctness oracle: **speculative-on must equal speculative-off** for the
+same prompt under greedy params.
+
+## Remaining work (the GPU verify forward + wiring)
+
+1. **`verify_forward(verify_tokens, kv_view) -> Vec<u32>`** on `Qwen3Model` — a
+   standalone method (kept separate so it does not pollute `batch_prefill` /
+   `batch_decode`). Internally reuses the paged-prefill attention + the existing
+   all-position-logits path, then takes a per-position argmax on the GPU via the
+   existing batch sampler (`select_batch_tokens_into` with greedy params over
+   the `K + 1` positions). Only the `K + 1` token ids are copied to host — never
+   the `[vocab, K + 1]` logits — mirroring vLLM's GPU-side rejection sampler.
+2. **`StepCommand::SpeculativeVerify` + `execute_speculative`** in the executor:
+   `schedule_speculative` → `prefill_view(1 + K)` → run verify step → argmax →
+   `accept_greedy` → `apply_speculative` → return committed tokens.
+3. **Scheduler wiring**: invoke the proposer over per-request history when
+   `SpeculativeConfig.enabled`; commit loop applies stop / max-token handling to
+   each committed token. (vLLM proposes the *next* step's drafts right after a
+   forward and stores them on the request for pipelining; the first cut may
+   propose at step start for simplicity.)
+
+## Scope / deferred
+
+First cut is **single-request, greedy, non-CUDA-graph**. Deferred: batched
+speculation (ragged verify across requests), sampling (non-greedy) acceptance,
+CUDA-graph-captured verify, interaction with the unified prefill+decode step,
+and pipelined ahead-of-time proposal.
+
+## Open questions for review
+
+1. `verify_forward` as a standalone model method (preferred) vs. reusing
+   `batch_prefill(echo = true)`.
+2. Confirm the GPU-side argmax via `select_batch_tokens_into` over the `K + 1`
+   verify positions is acceptable (vs. a dedicated argmax kernel later).
+3. Proposer placement in the scheduler layer (owns token history), matching
+   vLLM's runner / `request.spec_token_ids` split.
+
+## Prior art
+
+- vLLM V1 n-gram / prompt-lookup spec decode (`NgramProposer` in the runner,
+  GPU rejection sampler, scheduler reserves KV for `k` draft tokens).
+- kvbm `SchedulableSequence` speculative lifecycle (`schedule_speculative` /
+  `apply_speculative`, LIFO block release).

From 05d90b73b84cdddba7e74b4f3f1991dadbcabf40 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 14:30:24 +0800
Subject: [PATCH 05/31] feat(qwen3): wire speculative verify forward into the
 executor

Add the GPU verify step for n-gram speculative decode, as an additive path
that leaves prefill/decode/unified untouched:

- LocalQwen3Lane::execute_verify runs the model on [dangling, drafts..] over
  the request's existing paged KV by reusing batch_prefill(echo=true) for the
  per-position logits, then takes a GPU argmax (argmax_batch_bf16_into) and
  copies back only the per-position token ids.
- New StepCommand::SpeculativeVerify / WorkerStepOutcome::Speculative carry it
  through the rank-worker channel (TP-safe: workers run the forward, primary
  returns the argmax).
- Qwen3Executor::execute_speculative orchestrates one request's step:
  schedule_speculative(K+1) -> verify -> accept_greedy -> apply_speculative
  (kvbm releases rejected drafts' blocks). Returns the committed tokens.
- SpeculativeStepItem carries the dangling token + drafts; exported via runtime.

Scheduler wiring (proposer invocation over token history) and GPU validation
of the lossless spec-on == spec-off property are the remaining steps.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/executor.rs    | 141 ++++++++++++++++++++++++++
 openinfer-qwen3-4b/src/lib.rs         |   2 +-
 openinfer-qwen3-4b/src/ngram.rs       |   4 +-
 openinfer-qwen3-4b/src/speculative.rs |  14 +--
 4 files changed, 147 insertions(+), 14 deletions(-)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 0d2572e6..4e2d1281 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -109,6 +109,29 @@ impl DecodeStepItem {
     }
 }
 
+/// One request's speculative-decode step input: the last committed token
+/// (`token_id`, the dangling `d0`) plus the proposer's draft continuation.
+/// The verify forward runs the model on `[token_id, drafts..]` and the
+/// executor commits the greedy-accepted prefix plus one model token.
+#[derive(Clone)]
+pub struct SpeculativeStepItem {
+    pub request_id: RequestId,
+    pub token_id: u32,
+    pub drafts: Vec<u32>,
+    pub lora_adapter: Option<String>,
+}
+
+impl SpeculativeStepItem {
+    pub fn new(request_id: RequestId, token_id: u32, drafts: Vec<u32>) -> Self {
+        Self {
+            request_id,
+            token_id,
+            drafts,
+            lora_adapter: None,
+        }
+    }
+}
+
 fn build_prefill_request_results(
     lane: &mut LocalQwen3Lane,
     requests: &[PrefillStepItem],
@@ -316,6 +339,18 @@ fn execute_step_on_lane(
                 Ok(WorkerStepOutcome::Ack)
             }
         }
+        StepCommand::SpeculativeVerify {
+            verify_tokens,
+            kv_view,
+            lora_adapter,
+        } => {
+            let argmax = lane.execute_verify(verify_tokens, kv_view, lora_adapter.as_deref())?;
+            if collect_result {
+                Ok(WorkerStepOutcome::Speculative(argmax))
+            } else {
+                Ok(WorkerStepOutcome::Ack)
+            }
+        }
     }
 }
 
@@ -796,6 +831,63 @@ impl Qwen3Executor {
         <Self as ModelExecutor>::execute_unified(self, plan)
     }
 
+    /// Run one n-gram speculative decode step for a single request and return
+    /// the committed tokens (the greedy-accepted draft prefix plus one model
+    /// token; always >= 1). The verify forward runs the model on
+    /// `[dangling, drafts..]` over the request's existing KV; kvbm releases the
+    /// blocks pre-allocated for rejected drafts. Greedy verification is
+    /// lossless: the committed tokens equal plain greedy decode.
+    ///
+    /// The proposer that produces `item.drafts` lives in the scheduler layer
+    /// (it owns the request's token history); this method only verifies and
+    /// commits. Speculative decode is disabled by default
+    /// ([`crate::speculative::SpeculativeConfig`]); the scheduler calls this
+    /// only when it is enabled and the proposer returned a non-empty draft.
+    pub fn execute_speculative(&mut self, item: &SpeculativeStepItem) -> Result<Vec<u32>> {
+        anyhow::ensure!(!item.drafts.is_empty(), "speculative step needs >= 1 draft");
+        let k = item.drafts.len();
+
+        let rkv = self
+            .request_kvs
+            .get_mut(&item.request_id)
+            .ok_or_else(|| anyhow::anyhow!("missing RequestKv for {:?}", item.request_id))?;
+        // Reserve room for every draft plus the model's bonus/correction token.
+        rkv.schedule_speculative(k + 1, self.kv_mgr.pool())
+            .map_err(|e| {
+                anyhow::anyhow!("schedule_speculative failed for {:?}: {e}", item.request_id)
+            })?;
+
+        let mut verify_tokens = Vec::with_capacity(k + 1);
+        verify_tokens.push(item.token_id);
+        verify_tokens.extend_from_slice(&item.drafts);
+        let kv_view = rkv.prefill_view(verify_tokens.len());
+
+        let step = StepCommand::SpeculativeVerify {
+            verify_tokens,
+            kv_view,
+            lora_adapter: item.lora_adapter.clone(),
+        };
+        let outcome = self.run_step(&step)?;
+        let argmax = match outcome {
+            WorkerStepOutcome::Speculative(argmax) => argmax,
+            other => {
+                return Err(anyhow::anyhow!(
+                    "speculative returned unexpected: {}",
+                    other.kind()
+                ));
+            }
+        };
+
+        let committed = crate::speculative::accept_greedy(&item.drafts, &argmax);
+        let rkv = self
+            .request_kvs
+            .get_mut(&item.request_id)
+            .expect("request must exist after speculative verify");
+        rkv.apply_speculative(&committed, self.kv_mgr.pool())?;
+        self.save_sealed_blocks(item.request_id);
+        Ok(committed)
+    }
+
     /// Prefix caching is on by default; tests that assert bit-identical
     /// replay disable it (a cache hit changes prefill GEMM shapes, which
     /// drifts logits by bf16 ULPs).
@@ -1755,6 +1847,45 @@ impl LocalQwen3Lane {
         )
     }
 
+    /// Speculative verify forward: run the model on `verify_tokens`
+    /// (`[dangling, drafts..]`) over the request's existing paged KV and return
+    /// the greedy token at each of the `verify_tokens.len()` positions.
+    ///
+    /// Structurally a prefill of `verify_tokens` at the current position
+    /// (`kv_view = prefill_view(verify_tokens.len())`), so it reuses
+    /// `batch_prefill(echo = true)` for the per-position logits and takes the
+    /// argmax on the GPU — only the position token ids are copied to host,
+    /// never the `[vocab, n]` logits.
+    fn execute_verify(
+        &mut self,
+        verify_tokens: &[u32],
+        kv_view: &KvView,
+        lora_adapter: Option<&str>,
+    ) -> Result<Vec<u32>> {
+        let (_last_logits, all_logits) = self.model.batch_prefill(
+            &[verify_tokens],
+            std::slice::from_ref(kv_view),
+            &[lora_adapter],
+            self.kv_buffer.buffer(),
+            &self.layout,
+            true,
+        )?;
+        let all_logits = all_logits
+            .ok_or_else(|| anyhow::anyhow!("verify forward must return all-position logits"))?;
+
+        let rows = verify_tokens.len();
+        let ctx = self.model.device_ctx();
+        let mut values: cudarc::driver::CudaSlice<half::bf16> = ctx.stream.alloc_zeros(rows)?;
+        let mut indices: cudarc::driver::CudaSlice<i32> = ctx.stream.alloc_zeros(rows)?;
+        openinfer_core::ops::argmax_batch_bf16_into(ctx, &all_logits, &mut values, &mut indices)?;
+        let host = ctx
+            .stream
+            .clone_dtoh(&indices)
+            .map_err(|e| anyhow::anyhow!("verify argmax D2H failed: {e}"))?;
+        ctx.sync()?;
+        Ok(host.into_iter().map(|i| i as u32).collect())
+    }
+
     fn execute_unified(
         &mut self,
         prefill_prompts: &[&[u32]],
@@ -1814,6 +1945,11 @@ enum StepCommand {
         decode_requests: Vec<DecodeStepItem>,
         decode_kv_views: Vec<KvView>,
     },
+    SpeculativeVerify {
+        verify_tokens: Vec<u32>,
+        kv_view: KvView,
+        lora_adapter: Option<String>,
+    },
 }
 
 impl StepCommand {
@@ -1822,6 +1958,7 @@ impl StepCommand {
             Self::Prefill { .. } => "prefill",
             Self::Decode { .. } => "decode",
             Self::Unified { .. } => "unified",
+            Self::SpeculativeVerify { .. } => "speculative_verify",
         }
     }
 }
@@ -1854,6 +1991,9 @@ enum WorkerStepOutcome {
     Prefill(PrefillResult),
     Decode(DecodeResult),
     Unified(UnifiedResult),
+    /// Per-position greedy argmax (one token per verify input) from a
+    /// speculative verify forward.
+    Speculative(Vec<u32>),
 }
 
 impl WorkerStepOutcome {
@@ -1863,6 +2003,7 @@ impl WorkerStepOutcome {
             Self::Prefill(_) => "prefill",
             Self::Decode(_) => "decode",
             Self::Unified(_) => "unified",
+            Self::Speculative(_) => "speculative",
         }
     }
 }
diff --git a/openinfer-qwen3-4b/src/lib.rs b/openinfer-qwen3-4b/src/lib.rs
index e4e74848..008efe02 100644
--- a/openinfer-qwen3-4b/src/lib.rs
+++ b/openinfer-qwen3-4b/src/lib.rs
@@ -112,7 +112,7 @@ pub mod runtime {
     pub use crate::executor::{
         DecodePlan, DecodeRequestResult, DecodeResult, DecodeStepItem, PrefillPlan,
         PrefillRequestResult, PrefillResult, PrefillStepItem, Qwen3Executor, RequestId,
-        UnifiedPlan, UnifiedResult,
+        SpeculativeStepItem, UnifiedPlan, UnifiedResult,
     };
 }
 
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index 80fc69c1..ecf8af95 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -101,9 +101,7 @@ fn latest_earlier_match(context: &[u32], suffix: &[u32]) -> Option<usize> {
     // Candidate starts run from `len - n - 1` (immediately before the trailing
     // suffix) down to 0; the trailing occurrence itself (start `len - n`) is
     // excluded so we never "predict" from the suffix we are matching.
-    (0..=len - n - 1)
-        .rev()
-        .find(|&i| &context[i..i + n] == suffix)
+    (0..len - n).rev().find(|&i| &context[i..i + n] == suffix)
 }
 
 #[cfg(test)]
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index c70fb54b..74b1034b 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -10,7 +10,10 @@
 use crate::ngram::NgramConfig;
 
 /// Speculative-decoding configuration.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+///
+/// `Default` is disabled with the default n-gram settings (`enabled: false`),
+/// so the decode path runs the normal one-token step until explicitly turned on.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Default)]
 pub struct SpeculativeConfig {
     /// Master switch. When `false` the decode path runs the normal one-token
     /// step and never calls the proposer.
@@ -19,15 +22,6 @@ pub struct SpeculativeConfig {
     pub ngram: NgramConfig,
 }
 
-impl Default for SpeculativeConfig {
-    fn default() -> Self {
-        Self {
-            enabled: false,
-            ngram: NgramConfig::default(),
-        }
-    }
-}
-
 /// Greedy speculative acceptance.
 ///
 /// * `proposed` — the `K` candidate tokens from the proposer.

From 64645788fa785b417854bcf804627780c222e9a0 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 14:32:26 +0800
Subject: [PATCH 06/31] docs(qwen3): mark speculative verify forward landed in
 design note

Move the verify forward + executor step from "remaining" to "landed"; the
remaining work is scheduler wiring and GPU validation of the lossless property.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 45 +++++++++++++++-----------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index f18803a6..c30ab5eb 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -9,8 +9,10 @@ fewer forward passes on repetitive / structured text (code, quoting, JSON).
 The KV layer (kvbm) already supports speculative scheduling natively, so
 rejected drafts need no manual rollback.
 
-Last touched: 2026-06 · Status: **design + building blocks landed; GPU verify
-forward + executor/scheduler wiring pending.**
+Last touched: 2026-06 · Status: **proposer, acceptance, KV speculative
+pass-throughs, and the executor verify forward (`execute_speculative`) landed
+and compile; scheduler wiring (proposer over token history) and GPU validation
+of the lossless `spec-on == spec-off` property pending.**
 
 ## Why this is cheap to add here
 
@@ -65,23 +67,28 @@ to what plain greedy decode would have produced one token at a time. This gives
 a free correctness oracle: **speculative-on must equal speculative-off** for the
 same prompt under greedy params.
 
-## Remaining work (the GPU verify forward + wiring)
-
-1. **`verify_forward(verify_tokens, kv_view) -> Vec<u32>`** on `Qwen3Model` — a
-   standalone method (kept separate so it does not pollute `batch_prefill` /
-   `batch_decode`). Internally reuses the paged-prefill attention + the existing
-   all-position-logits path, then takes a per-position argmax on the GPU via the
-   existing batch sampler (`select_batch_tokens_into` with greedy params over
-   the `K + 1` positions). Only the `K + 1` token ids are copied to host — never
-   the `[vocab, K + 1]` logits — mirroring vLLM's GPU-side rejection sampler.
-2. **`StepCommand::SpeculativeVerify` + `execute_speculative`** in the executor:
-   `schedule_speculative` → `prefill_view(1 + K)` → run verify step → argmax →
-   `accept_greedy` → `apply_speculative` → return committed tokens.
-3. **Scheduler wiring**: invoke the proposer over per-request history when
-   `SpeculativeConfig.enabled`; commit loop applies stop / max-token handling to
-   each committed token. (vLLM proposes the *next* step's drafts right after a
-   forward and stores them on the request for pipelining; the first cut may
-   propose at step start for simplicity.)
+## Landed: GPU verify forward + executor step
+
+- **`LocalQwen3Lane::execute_verify(verify_tokens, kv_view, lora)`** — reuses
+  `batch_prefill(echo = true)` for per-position logits over the existing paged
+  KV, then a GPU argmax (`argmax_batch_bf16_into`) returns one token per verify
+  position. Only the position ids cross to host, never the `[vocab, n]` logits
+  (vLLM-style). Kept additive: `batch_prefill` / `batch_decode` are untouched.
+- **`StepCommand::SpeculativeVerify` + `Qwen3Executor::execute_speculative`** —
+  `schedule_speculative(K + 1)` → `prefill_view(1 + K)` → verify step → argmax →
+  `accept_greedy` → `apply_speculative`, returning the committed tokens. The
+  rank-worker channel carries it (TP-safe).
+
+## Remaining work
+
+1. **Scheduler wiring**: invoke the proposer over per-request token history when
+   `SpeculativeConfig.enabled`; the commit loop applies stop / max-token
+   handling to each committed token. (vLLM proposes the *next* step's drafts
+   right after a forward and stores them on the request for pipelining; the
+   first cut may propose at step start for simplicity.)
+2. **GPU validation**: assert the lossless `spec-on == spec-off` property on a
+   real Qwen3-4B run, plus an acceptance-rate / speedup measurement on
+   repetitive prompts.
 
 ## Scope / deferred
 

From 218164614c37df8c99dc6da86471b6c2fca0719a Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 14:40:30 +0800
Subject: [PATCH 07/31] test(qwen3): GPU validation that n-gram speculative
 decode is lossless

End-to-end test on real Qwen3-4B weights: greedy n-gram speculative decode
(proposer -> schedule_speculative -> verify forward -> accept_greedy ->
apply_speculative) produces a token-identical sequence to plain greedy decode.
Prefix cache disabled for cold, identically-shaped runs; repetitive prompt
keeps the greedy argmax well-separated despite the verify (prefill kernel) vs
reference (decode kernel) path difference. Skips cleanly without GPU/weights.

Passes on RTX (sm_89), Qwen3-4B: 24-token greedy continuation matches exactly.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/tests/ngram_speculative.rs | 159 ++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 openinfer-qwen3-4b/tests/ngram_speculative.rs

diff --git a/openinfer-qwen3-4b/tests/ngram_speculative.rs b/openinfer-qwen3-4b/tests/ngram_speculative.rs
new file mode 100644
index 00000000..398ea0f2
--- /dev/null
+++ b/openinfer-qwen3-4b/tests/ngram_speculative.rs
@@ -0,0 +1,159 @@
+//! GPU validation for n-gram speculative decoding: greedy speculation is
+//! lossless, so the token sequence produced with the proposer must match plain
+//! greedy decode token-for-token.
+//!
+//! Methodology note: the verify forward runs the prefill attention kernel while
+//! the reference decode runs the decode attention kernel. The math is identical
+//! but bf16 reduction order can differ, so a near-tie position could in
+//! principle diverge; the repetitive prompt keeps the greedy argmax
+//! well-separated. Prefix caching is disabled so both runs are cold and
+//! identically shaped.
+//!
+//! Requires a CUDA GPU and Qwen3-4B weights; skips cleanly when absent
+//! (point `OPENINFER_TEST_MODEL_PATH` at the weights to run it).
+
+use std::path::Path;
+
+use openinfer_core::sampler::SamplingParams;
+use openinfer_qwen3_4b::ngram::{NgramConfig, NgramProposer};
+use openinfer_qwen3_4b::runtime::{
+    DecodePlan, DecodeStepItem, PrefillPlan, PrefillStepItem, Qwen3Executor, RequestId,
+    SpeculativeStepItem,
+};
+
+const MODEL_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../models/Qwen3-4B");
+const N_GENERATE: usize = 24;
+const MAX_OUTPUT: usize = N_GENERATE + 8;
+
+fn model_path_or_skip() -> Option<String> {
+    match std::env::var("OPENINFER_TEST_MODEL_PATH") {
+        Ok(path) => Some(path),
+        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => Some(MODEL_PATH.to_string()),
+        Err(_) => {
+            eprintln!(
+                "skipping qwen3 ngram_speculative: {MODEL_PATH}/config.json missing; set OPENINFER_TEST_MODEL_PATH to run it"
+            );
+            None
+        }
+    }
+}
+
+/// Repetitive prompt so the n-gram proposer fires (its suffix recurs earlier).
+fn repetitive_prompt() -> Vec<u32> {
+    let unit: [u32; 8] = [1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007];
+    let mut p = Vec::new();
+    for _ in 0..8 {
+        p.extend_from_slice(&unit);
+    }
+    p
+}
+
+fn prefill_item(id: u64, prompt: &[u32]) -> PrefillStepItem {
+    PrefillStepItem::new(
+        RequestId::new(id),
+        prompt.to_vec(),
+        MAX_OUTPUT,
+        SamplingParams::default(),
+        0,
+        false,
+        0.0,
+    )
+}
+
+/// Plain greedy decode (spec-off): one model token per step.
+fn greedy_decode(ex: &mut Qwen3Executor, id: u64, prompt: &[u32]) -> Vec<u32> {
+    let pr = ex
+        .execute_prefill(PrefillPlan {
+            requests: &[prefill_item(id, prompt)],
+            echo: false,
+        })
+        .expect("prefill");
+    let mut last = pr.requests[0].first_token;
+    let mut out = vec![last];
+    while out.len() < N_GENERATE {
+        let dr = ex
+            .execute_decode(DecodePlan {
+                requests: &[DecodeStepItem::new(
+                    RequestId::new(id),
+                    last,
+                    SamplingParams::default(),
+                    0,
+                    0.0,
+                )],
+            })
+            .expect("decode");
+        last = dr.requests[0].token;
+        out.push(last);
+    }
+    out
+}
+
+/// Greedy n-gram speculative decode (spec-on).
+fn speculative_decode(ex: &mut Qwen3Executor, id: u64, prompt: &[u32]) -> Vec<u32> {
+    let proposer = NgramProposer::new(NgramConfig {
+        max_ngram: 3,
+        min_ngram: 1,
+        num_speculative: 4,
+    });
+    let pr = ex
+        .execute_prefill(PrefillPlan {
+            requests: &[prefill_item(id, prompt)],
+            echo: false,
+        })
+        .expect("prefill");
+    let first = pr.requests[0].first_token;
+    let mut history = prompt.to_vec();
+    history.push(first);
+    let mut out = vec![first];
+
+    while out.len() < N_GENERATE {
+        let last = *out.last().unwrap();
+        let drafts = proposer.propose(&history);
+        if drafts.is_empty() {
+            let dr = ex
+                .execute_decode(DecodePlan {
+                    requests: &[DecodeStepItem::new(
+                        RequestId::new(id),
+                        last,
+                        SamplingParams::default(),
+                        0,
+                        0.0,
+                    )],
+                })
+                .expect("decode");
+            let t = dr.requests[0].token;
+            history.push(t);
+            out.push(t);
+        } else {
+            let committed = ex
+                .execute_speculative(&SpeculativeStepItem::new(RequestId::new(id), last, drafts))
+                .expect("speculative");
+            for t in committed {
+                history.push(t);
+                out.push(t);
+            }
+        }
+    }
+    out.truncate(N_GENERATE);
+    out
+}
+
+#[test]
+fn ngram_speculative_is_lossless_vs_greedy() {
+    let Some(model_path) = model_path_or_skip() else {
+        return;
+    };
+    let mut ex =
+        Qwen3Executor::from_runtime(&model_path, false, &[0]).expect("build eager executor");
+    // Cold, identically-shaped runs so the comparison is exact.
+    ex.set_prefix_cache_enabled(false);
+
+    let prompt = repetitive_prompt();
+    let reference = greedy_decode(&mut ex, 1, &prompt);
+    let speculative = speculative_decode(&mut ex, 2, &prompt);
+
+    assert_eq!(
+        reference, speculative,
+        "greedy speculative decode must be token-identical to plain greedy decode"
+    );
+}

From 4fd4c7a04f3f636fc04a0c05e11c5051e85c9666 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 14:54:58 +0800
Subject: [PATCH 08/31] docs(qwen3): record GPU-validated lossless property for
 ngram spec decode

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index c30ab5eb..e43e59c9 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -10,9 +10,10 @@ The KV layer (kvbm) already supports speculative scheduling natively, so
 rejected drafts need no manual rollback.
 
 Last touched: 2026-06 · Status: **proposer, acceptance, KV speculative
-pass-throughs, and the executor verify forward (`execute_speculative`) landed
-and compile; scheduler wiring (proposer over token history) and GPU validation
-of the lossless `spec-on == spec-off` property pending.**
+pass-throughs, and the executor verify forward (`execute_speculative`) landed,
+and GPU-validated lossless on real Qwen3-4B (`tests/ngram_speculative.rs`:
+greedy spec-on == spec-off, token-identical). Only the scheduler wiring
+(proposer over token history in the serving loop) remains.**
 
 ## Why this is cheap to add here
 
@@ -79,6 +80,14 @@ same prompt under greedy params.
   `accept_greedy` → `apply_speculative`, returning the committed tokens. The
   rank-worker channel carries it (TP-safe).
 
+## Validated
+
+- **Lossless on GPU** — `tests/ngram_speculative.rs` runs real Qwen3-4B and
+  asserts greedy n-gram speculative decode is token-identical to plain greedy
+  decode (prefix cache off; repetitive prompt). Confirms the full pipeline
+  (proposer → schedule_speculative → verify forward → accept_greedy →
+  apply_speculative) end-to-end.
+
 ## Remaining work
 
 1. **Scheduler wiring**: invoke the proposer over per-request token history when
@@ -86,9 +95,8 @@ same prompt under greedy params.
    handling to each committed token. (vLLM proposes the *next* step's drafts
    right after a forward and stores them on the request for pipelining; the
    first cut may propose at step start for simplicity.)
-2. **GPU validation**: assert the lossless `spec-on == spec-off` property on a
-   real Qwen3-4B run, plus an acceptance-rate / speedup measurement on
-   repetitive prompts.
+2. **Speedup measurement**: acceptance-rate / TPOT on repetitive prompts (the
+   feature's actual payoff, distinct from the correctness oracle above).
 
 ## Scope / deferred
 

From 75bfd3532a240e6ab9e2779a4611ab76cd472195 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 16:08:47 +0800
Subject: [PATCH 09/31] feat(qwen3): expose execute_speculative on the
 ModelExecutor trait
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add execute_speculative to ModelExecutor (default: unsupported) so a generic
scheduler can drive speculative decode; Qwen3Executor's trait method delegates
to the existing inherent body. No behaviour change — nothing calls it yet.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/executor.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 4e2d1281..fc8b2319 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -505,6 +505,14 @@ pub(crate) trait ModelExecutor: Send {
     fn execute_decode(&mut self, plan: DecodePlan<'_>) -> Result<DecodeResult>;
     fn execute_unified(&mut self, plan: UnifiedPlan<'_>) -> Result<UnifiedResult>;
 
+    /// Run one n-gram speculative decode step for a single request, returning
+    /// the committed tokens (accepted draft prefix + one model token; >= 1).
+    /// Default: unsupported. Implemented by `Qwen3Executor`; the scheduler only
+    /// calls it when speculative decode is enabled.
+    fn execute_speculative(&mut self, _item: &SpeculativeStepItem) -> Result<Vec<u32>> {
+        anyhow::bail!("speculative decode is not supported by this executor")
+    }
+
     fn load_lora_adapter(&mut self, request: &LoadLoraAdapterRequest) -> Result<()> {
         anyhow::bail!(
             "Qwen3 LoRA adapter loading is not implemented yet: name={}, path={}",
@@ -1399,6 +1407,12 @@ impl ModelExecutor for Qwen3Executor {
         Ok(result)
     }
 
+    fn execute_speculative(&mut self, item: &SpeculativeStepItem) -> Result<Vec<u32>> {
+        // Inherent method holds the body (and is the public surface used by
+        // tests); the trait method delegates so generic schedulers can call it.
+        Qwen3Executor::execute_speculative(self, item)
+    }
+
     fn execute_unified(&mut self, plan: UnifiedPlan<'_>) -> Result<UnifiedResult> {
         // 1. Create RequestKvs for prefill requests, reuse cached prefix
         // blocks, and schedule the rest

From b8114676ecdae72c65deb376bc67c84f089687c8 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 16:29:07 +0800
Subject: [PATCH 10/31] feat(qwen3): wire n-gram speculative decode into the
 scheduler
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the serving-loop integration behind a (default-disabled) SpeculativeConfig:

- ActiveRequestState gains token_history (prompt + generated), maintained on
  promote (resolve) and each committed decode token (effects).
- New scheduler/speculative.rs: speculative_decode_step proposes drafts per
  active request, runs execute_speculative (or falls back to a single decode
  when there is no draft), then streams the committed tokens applying stop /
  max-token handling — kept isolated from the one-token-per-step plan/resolve/
  effects pipeline.
- scheduler_loop routes pure-decode ticks through it when enabled.

Mock-tested (FakeExecutor.execute_speculative): a step streams every committed
token and advances state; commits past max_tokens truncate, finish, and retire.
Enabling it from the public API (a config knob) is a follow-up.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/scheduler.rs           | 121 ++++++++++++++++
 openinfer-qwen3-4b/src/scheduler/effects.rs   |   1 +
 openinfer-qwen3-4b/src/scheduler/resolve.rs   |   3 +
 .../src/scheduler/speculative.rs              | 131 ++++++++++++++++++
 openinfer-qwen3-4b/tests/ngram_speculative.rs |   4 +-
 5 files changed, 259 insertions(+), 1 deletion(-)
 create mode 100644 openinfer-qwen3-4b/src/scheduler/speculative.rs

diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index 711fc6e3..d54aaae1 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -8,6 +8,7 @@
 mod effects;
 mod plan;
 mod resolve;
+mod speculative;
 
 use std::collections::{HashSet, VecDeque};
 use std::thread;
@@ -43,6 +44,9 @@ pub(super) struct ActiveRequestState {
     pub(super) params: SamplingParams,
     /// Number of top logprobs to return (0 = disabled).
     pub(super) logprobs: usize,
+    /// Full token context (prompt + generated so far). Maintained for the
+    /// n-gram speculative proposer; empty/unused when speculation is off.
+    pub(super) token_history: Vec<u32>,
 }
 
 pub(super) struct PendingRequest {
@@ -268,6 +272,13 @@ fn scheduler_loop<E>(
     // Requests parked while their async CPU-tier KV prefetch loads.
     let mut loading: Vec<PendingRequest> = Vec::new();
 
+    // N-gram speculative decode. Disabled by default; enabling it is a follow-up
+    // (a server/config knob is not wired yet). When on, pure-decode ticks route
+    // through `speculative::speculative_decode_step` instead of the batched
+    // single-token decode.
+    let spec_config = crate::speculative::SpeculativeConfig::default();
+    let ngram_proposer = crate::ngram::NgramProposer::new(spec_config.ngram);
+
     info!("Scheduler ready");
 
     loop {
@@ -338,6 +349,12 @@ fn scheduler_loop<E>(
         let Some(plan) = build_next_plan(!active.is_empty(), pending) else {
             continue;
         };
+        // Speculative decode replaces the batched single-token decode on
+        // pure-decode ticks (no new prefill this step). Lossless under greedy.
+        if spec_config.enabled && matches!(plan, self::plan::ExecutionPlan::Decode) {
+            self::speculative::speculative_decode_step(&mut executor, &mut active, &ngram_proposer);
+            continue;
+        }
         let failure_targets = failure_targets_for(&active, &plan);
         let artifacts = match execute_plan(&mut executor, &mut active, plan, &mut rng) {
             Ok(v) => v,
@@ -1048,6 +1065,22 @@ mod tests {
             })
         }
 
+        fn execute_speculative(
+            &mut self,
+            item: &crate::executor::SpeculativeStepItem,
+        ) -> Result<Vec<u32>> {
+            // Deterministic: accept every draft, then append one bonus token.
+            let mut committed = item.drafts.clone();
+            committed.push(300 + item.request_id.get() as u32);
+            let current = self
+                .held_tokens
+                .get(&item.request_id)
+                .copied()
+                .ok_or_else(|| anyhow::anyhow!("missing fake request state"))?;
+            self.ensure_request_tokens(item.request_id, current + committed.len())?;
+            Ok(committed)
+        }
+
         fn execute_unified(&mut self, plan: UnifiedPlan<'_>) -> Result<UnifiedResult> {
             self.prefill_batches.lock().unwrap().push(
                 plan.prefill_requests
@@ -1128,6 +1161,7 @@ mod tests {
             prompt_len: 16,
             params: SamplingParams::default(),
             logprobs: 0,
+            token_history: Vec::new(),
         };
         assert_eq!(current_active_tokens(&after_prefill), 16);
         assert_eq!(max_active_tokens(&after_prefill), 18);
@@ -1143,6 +1177,7 @@ mod tests {
             prompt_len: 16,
             params: SamplingParams::default(),
             logprobs: 0,
+            token_history: Vec::new(),
         };
         assert_eq!(current_active_tokens(&after_one_decode), 17);
         assert_eq!(max_active_tokens(&after_one_decode), 18);
@@ -1164,6 +1199,7 @@ mod tests {
             prompt_len: 16,
             params: SamplingParams::default(),
             logprobs: 0,
+            token_history: Vec::new(),
         }];
 
         let mk = |id: u64, prompt_len, max_tokens| {
@@ -1258,6 +1294,7 @@ mod tests {
                 prompt_len: 16,
                 params: SamplingParams::default(),
                 logprobs: 0,
+                token_history: Vec::new(),
             });
         }
         let pending = PendingRequest::from_scheduler_request(RequestId(64), request(16, 1).0);
@@ -1698,6 +1735,7 @@ mod tests {
                 prompt_len: 16,
                 params: SamplingParams::default(),
                 logprobs: 0,
+                token_history: Vec::new(),
             });
             executor
                 .ensure_request_tokens(request_id, 16)
@@ -1841,4 +1879,87 @@ mod tests {
             .expect_err("adapter load should be a stub error");
         assert!(matches!(error, EngineControlError::OperationFailed(_)));
     }
+
+    fn spec_active(
+        request_id: RequestId,
+        max_tokens: usize,
+        history: Vec<u32>,
+    ) -> (ActiveRequestState, mpsc::UnboundedReceiver<TokenEvent>) {
+        let (token_tx, token_rx) = mpsc::unbounded_channel();
+        let last_token = *history.last().unwrap();
+        (
+            ActiveRequestState {
+                request_id,
+                lora_adapter: None,
+                token_tx,
+                last_token,
+                generated_count: 1,
+                max_tokens,
+                prompt_len: history.len(),
+                params: SamplingParams::default(),
+                logprobs: 0,
+                token_history: history,
+            },
+            token_rx,
+        )
+    }
+
+    fn ngram2() -> crate::ngram::NgramProposer {
+        crate::ngram::NgramProposer::new(crate::ngram::NgramConfig {
+            max_ngram: 1,
+            min_ngram: 1,
+            num_speculative: 2,
+        })
+    }
+
+    fn drain_tokens(rx: &mut mpsc::UnboundedReceiver<TokenEvent>) -> Vec<u32> {
+        let mut out = Vec::new();
+        while let Ok(ev) = rx.try_recv() {
+            if let TokenEvent::Token { id, .. } = ev {
+                out.push(id);
+            }
+        }
+        out
+    }
+
+    // A speculative step streams every committed token, advances the request's
+    // history/count, and keeps it active when no stop condition is hit.
+    #[test]
+    fn speculative_step_streams_committed_tokens() {
+        let dropped = Arc::new(Mutex::new(Vec::new()));
+        let mut executor = FakeExecutor::new(64, Arc::clone(&dropped));
+        executor.ensure_request_tokens(RequestId(0), 3).unwrap();
+        // history [5,6,5]: suffix [5] recurs at idx 0 -> drafts [6,5];
+        // FakeExecutor commits drafts + bonus 300 -> [6,5,300].
+        let (state, mut rx) = spec_active(RequestId(0), 100, vec![5, 6, 5]);
+        let mut active = vec![state];
+
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2());
+
+        assert_eq!(drain_tokens(&mut rx), vec![6, 5, 300]);
+        assert_eq!(active.len(), 1, "request continues");
+        assert_eq!(active[0].generated_count, 4, "1 + 3 committed");
+        assert_eq!(active[0].last_token, 300);
+        assert_eq!(active[0].token_history, vec![5, 6, 5, 6, 5, 300]);
+        assert!(dropped.lock().unwrap().is_empty());
+    }
+
+    // Committed tokens past max_tokens are truncated: the step stops emitting at
+    // the limit, finishes the request, and retires it.
+    #[test]
+    fn speculative_step_truncates_at_max_tokens() {
+        let dropped = Arc::new(Mutex::new(Vec::new()));
+        let mut executor = FakeExecutor::new(64, Arc::clone(&dropped));
+        executor.ensure_request_tokens(RequestId(0), 3).unwrap();
+        // max_tokens 3, generated 1: commit [6,5,300] -> emit 6 (gen 2), 5 (gen 3
+        // == limit -> Length), 300 never emitted.
+        let (state, mut rx) = spec_active(RequestId(0), 3, vec![5, 6, 5]);
+        let mut active = vec![state];
+
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2());
+
+        assert_eq!(drain_tokens(&mut rx), vec![6, 5]);
+        assert!(active.is_empty(), "request retired at length limit");
+        assert_eq!(dropped.lock().unwrap().clone(), vec![0]);
+    }
 }
diff --git a/openinfer-qwen3-4b/src/scheduler/effects.rs b/openinfer-qwen3-4b/src/scheduler/effects.rs
index 9c1848e5..8e385628 100644
--- a/openinfer-qwen3-4b/src/scheduler/effects.rs
+++ b/openinfer-qwen3-4b/src/scheduler/effects.rs
@@ -149,6 +149,7 @@ pub(super) fn apply_effects(
                 } else {
                     req.last_token = token;
                     req.generated_count = completion_tokens;
+                    req.token_history.push(token);
                 }
             }
         }
diff --git a/openinfer-qwen3-4b/src/scheduler/resolve.rs b/openinfer-qwen3-4b/src/scheduler/resolve.rs
index 9e9cf2b8..e2fd6b0a 100644
--- a/openinfer-qwen3-4b/src/scheduler/resolve.rs
+++ b/openinfer-qwen3-4b/src/scheduler/resolve.rs
@@ -71,6 +71,8 @@ fn resolve_prefill_outputs(
             continue;
         }
 
+        let mut token_history = req.prompt_tokens.clone();
+        token_history.push(result.first_token);
         effects.pending.push(PendingEffect::Promote {
             state: ActiveRequestState {
                 request_id: req.request_id,
@@ -82,6 +84,7 @@ fn resolve_prefill_outputs(
                 prompt_len,
                 params: req.params,
                 logprobs: req.logprobs,
+                token_history,
             },
             first_token: result.first_token,
             logprob: result.first_token_logprob,
diff --git a/openinfer-qwen3-4b/src/scheduler/speculative.rs b/openinfer-qwen3-4b/src/scheduler/speculative.rs
new file mode 100644
index 00000000..87c5df93
--- /dev/null
+++ b/openinfer-qwen3-4b/src/scheduler/speculative.rs
@@ -0,0 +1,131 @@
+//! Self-contained n-gram speculative decode step for the scheduler.
+//!
+//! When speculation is enabled this replaces the batched single-token decode
+//! for the active set. Per request: propose drafts from the request's token
+//! history, run one speculative verify (or fall back to a single decode when
+//! there is no draft), then stream the committed tokens — applying stop /
+//! max-token handling to each.
+//!
+//! Greedy speculation is lossless, so this emits the same tokens as the normal
+//! decode path, just (often) several per step. It is kept isolated from the
+//! generic plan/resolve/effects pipeline, which assumes exactly one token per
+//! request per step.
+
+use log::warn;
+use openinfer_core::engine::FinishReason;
+
+use crate::executor::{DecodePlan, DecodeStepItem, ModelExecutor, SpeculativeStepItem};
+use crate::ngram::NgramProposer;
+
+use super::{ActiveRequestState, TokenEvent};
+
+/// Advance every active request by one speculative step. Finished requests are
+/// dropped from `active`.
+pub(super) fn speculative_decode_step(
+    executor: &mut impl ModelExecutor,
+    active: &mut Vec<ActiveRequestState>,
+    proposer: &NgramProposer,
+) {
+    let mut to_retire = Vec::new();
+    for idx in 0..active.len() {
+        let request_id = active[idx].request_id;
+        let drafts = proposer.propose(&active[idx].token_history);
+
+        let committed = match commit_tokens(executor, &active[idx], drafts) {
+            Ok(tokens) => tokens,
+            Err(e) => {
+                warn!("speculative decode failed for {request_id:?}: {e}");
+                let req = &active[idx];
+                let _ = req.token_tx.send(TokenEvent::Finished {
+                    finish_reason: FinishReason::Stop,
+                    prompt_tokens: req.prompt_len,
+                    completion_tokens: req.generated_count,
+                });
+                let _ = executor.drop_request(request_id);
+                to_retire.push(idx);
+                continue;
+            }
+        };
+
+        if apply_committed(executor, &mut active[idx], &committed) {
+            to_retire.push(idx);
+        }
+    }
+    to_retire.sort_unstable();
+    to_retire.dedup();
+    for &i in to_retire.iter().rev() {
+        active.swap_remove(i);
+    }
+}
+
+/// Verify the drafts (or fall back to a single decode) and return the committed
+/// tokens. Always at least one token.
+fn commit_tokens(
+    executor: &mut impl ModelExecutor,
+    req: &ActiveRequestState,
+    drafts: Vec<u32>,
+) -> anyhow::Result<Vec<u32>> {
+    if drafts.is_empty() {
+        let result = executor.execute_decode(DecodePlan {
+            requests: &[DecodeStepItem {
+                request_id: req.request_id,
+                token_id: req.last_token,
+                params: req.params,
+                logprobs: 0,
+                lora_adapter: req.lora_adapter.clone(),
+                random_val: 0.0,
+            }],
+        })?;
+        Ok(result.requests.into_iter().map(|r| r.token).collect())
+    } else {
+        let mut item = SpeculativeStepItem::new(req.request_id, req.last_token, drafts);
+        item.lora_adapter = req.lora_adapter.clone();
+        executor.execute_speculative(&item)
+    }
+}
+
+/// Stream `committed`, updating request state and applying stop / max-token
+/// handling. Returns `true` when the request finished (caller retires it).
+fn apply_committed(
+    executor: &mut impl ModelExecutor,
+    req: &mut ActiveRequestState,
+    committed: &[u32],
+) -> bool {
+    for &token in committed {
+        let completion = req.generated_count + 1;
+        let is_eos = !req.params.ignore_eos && executor.is_stop_token(token);
+        if req
+            .token_tx
+            .send(TokenEvent::Token {
+                id: token,
+                logprob: None,
+            })
+            .is_err()
+        {
+            // Client hung up: drop without a Finished event.
+            let _ = executor.drop_request(req.request_id);
+            return true;
+        }
+        req.generated_count = completion;
+        req.last_token = token;
+        req.token_history.push(token);
+
+        let finish = if is_eos {
+            Some(FinishReason::Stop)
+        } else if completion >= req.max_tokens {
+            Some(FinishReason::Length)
+        } else {
+            None
+        };
+        if let Some(finish_reason) = finish {
+            let _ = req.token_tx.send(TokenEvent::Finished {
+                finish_reason,
+                prompt_tokens: req.prompt_len,
+                completion_tokens: req.generated_count,
+            });
+            let _ = executor.drop_request(req.request_id);
+            return true;
+        }
+    }
+    false
+}
diff --git a/openinfer-qwen3-4b/tests/ngram_speculative.rs b/openinfer-qwen3-4b/tests/ngram_speculative.rs
index 398ea0f2..0d58212b 100644
--- a/openinfer-qwen3-4b/tests/ngram_speculative.rs
+++ b/openinfer-qwen3-4b/tests/ngram_speculative.rs
@@ -28,7 +28,9 @@ const MAX_OUTPUT: usize = N_GENERATE + 8;
 fn model_path_or_skip() -> Option<String> {
     match std::env::var("OPENINFER_TEST_MODEL_PATH") {
         Ok(path) => Some(path),
-        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => Some(MODEL_PATH.to_string()),
+        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => {
+            Some(MODEL_PATH.to_string())
+        }
         Err(_) => {
             eprintln!(
                 "skipping qwen3 ngram_speculative: {MODEL_PATH}/config.json missing; set OPENINFER_TEST_MODEL_PATH to run it"

From 2a6e2aabc2f0f180809e1ca23140a52e7adf8876 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 16:30:06 +0800
Subject: [PATCH 11/31] docs(qwen3): record scheduler wiring landed; remaining
 = config knob + perf

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 37 ++++++++++++++++++--------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index e43e59c9..6d729be6 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -9,11 +9,12 @@ fewer forward passes on repetitive / structured text (code, quoting, JSON).
 The KV layer (kvbm) already supports speculative scheduling natively, so
 rejected drafts need no manual rollback.
 
-Last touched: 2026-06 · Status: **proposer, acceptance, KV speculative
-pass-throughs, and the executor verify forward (`execute_speculative`) landed,
-and GPU-validated lossless on real Qwen3-4B (`tests/ngram_speculative.rs`:
-greedy spec-on == spec-off, token-identical). Only the scheduler wiring
-(proposer over token history in the serving loop) remains.**
+Last touched: 2026-06 · Status: **end-to-end implemented and gated behind a
+(default-off) `SpeculativeConfig`: proposer, greedy acceptance, KV speculative
+pass-throughs, the executor verify forward (`execute_speculative`,
+GPU-validated lossless), and the scheduler serving-loop wiring
+(`speculative_decode_step`, mock-tested). Remaining: a public config knob to
+turn it on, and a speedup measurement.**
 
 ## Why this is cheap to add here
 
@@ -88,15 +89,29 @@ same prompt under greedy params.
   (proposer → schedule_speculative → verify forward → accept_greedy →
   apply_speculative) end-to-end.
 
+## Landed: scheduler serving-loop wiring
+
+- `ActiveRequestState` carries `token_history` (prompt + generated), maintained
+  on promote and each committed decode token.
+- `scheduler/speculative.rs::speculative_decode_step` proposes per active
+  request, runs `execute_speculative` (or a single decode when no draft), and
+  streams the committed tokens with stop / max-token handling — isolated from
+  the one-token-per-step plan/resolve/effects pipeline. `scheduler_loop` routes
+  pure-decode ticks through it when `SpeculativeConfig.enabled`.
+- Mock-tested (`FakeExecutor::execute_speculative`): streams every committed
+  token + advances state; commits past `max_tokens` truncate, finish, retire.
+
 ## Remaining work
 
-1. **Scheduler wiring**: invoke the proposer over per-request token history when
-   `SpeculativeConfig.enabled`; the commit loop applies stop / max-token
-   handling to each committed token. (vLLM proposes the *next* step's drafts
-   right after a forward and stores them on the request for pipelining; the
-   first cut may propose at step start for simplicity.)
+1. **Public config knob**: `SpeculativeConfig` is currently constructed
+   default-off inside `scheduler_loop`; thread a knob through `start_qwen3*` /
+   `start_engine*` (and a server flag) to turn it on. Single-request greedy
+   first; the unified prefill+decode tick still uses plain decode.
 2. **Speedup measurement**: acceptance-rate / TPOT on repetitive prompts (the
-   feature's actual payoff, distinct from the correctness oracle above).
+   payoff, distinct from the lossless correctness oracle).
+3. **Batched / vLLM-style verify** (perf): fold the verify tokens into the
+   unified batched forward (FlashInfer varlen) with a GPU rejection step,
+   instead of the current per-request `batch_prefill`-based verify.
 
 ## Scope / deferred
 

From 9cb33092951aeb9c83a6d808827eb12b81c981c1 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 16:38:48 +0800
Subject: [PATCH 12/31] feat(qwen3): env switch for n-gram speculative decode

Add SpeculativeConfig::from_env (OPENINFER_QWEN3_NGRAM_SPEC[_TOKENS|_MAX_NGRAM])
and have scheduler_loop build the config from it instead of hardcoding off, so
the speculative decode path can be turned on end-to-end. Log once on enable.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 19 +++++++++++----
 openinfer-qwen3-4b/src/scheduler.rs    | 12 +++++++---
 openinfer-qwen3-4b/src/speculative.rs  | 32 ++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 6d729be6..ea4f5729 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -101,12 +101,23 @@ same prompt under greedy params.
 - Mock-tested (`FakeExecutor::execute_speculative`): streams every committed
   token + advances state; commits past `max_tokens` truncate, finish, retire.
 
+## Enabling it
+
+`scheduler_loop` builds the config via `SpeculativeConfig::from_env()`
+(default-off). Operational switch:
+
+- `OPENINFER_QWEN3_NGRAM_SPEC=1` — turn it on.
+- `OPENINFER_QWEN3_NGRAM_SPEC_TOKENS=K` — draft count (default 4).
+- `OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM=N` — longest matched suffix (default 3).
+
+Only the non-LoRA `scheduler_loop` reads it; the unified prefill+decode tick
+still uses plain decode.
+
 ## Remaining work
 
-1. **Public config knob**: `SpeculativeConfig` is currently constructed
-   default-off inside `scheduler_loop`; thread a knob through `start_qwen3*` /
-   `start_engine*` (and a server flag) to turn it on. Single-request greedy
-   first; the unified prefill+decode tick still uses plain decode.
+1. **First-class config knob**: env var is the current switch; thread a typed
+   knob through `start_qwen3*` / `start_engine*` (and a server flag) so it shows
+   up in the engine config rather than the environment.
 2. **Speedup measurement**: acceptance-rate / TPOT on repetitive prompts (the
    payoff, distinct from the lossless correctness oracle).
 3. **Batched / vLLM-style verify** (perf): fold the verify tokens into the
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index d54aaae1..067ac4e6 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -272,11 +272,17 @@ fn scheduler_loop<E>(
     // Requests parked while their async CPU-tier KV prefetch loads.
     let mut loading: Vec<PendingRequest> = Vec::new();
 
-    // N-gram speculative decode. Disabled by default; enabling it is a follow-up
-    // (a server/config knob is not wired yet). When on, pure-decode ticks route
+    // N-gram speculative decode. Off unless OPENINFER_QWEN3_NGRAM_SPEC is set
+    // (see SpeculativeConfig::from_env). When on, pure-decode ticks route
     // through `speculative::speculative_decode_step` instead of the batched
     // single-token decode.
-    let spec_config = crate::speculative::SpeculativeConfig::default();
+    let spec_config = crate::speculative::SpeculativeConfig::from_env();
+    if spec_config.enabled {
+        info!(
+            "n-gram speculative decode enabled (K={}, max_ngram={})",
+            spec_config.ngram.num_speculative, spec_config.ngram.max_ngram
+        );
+    }
     let ngram_proposer = crate::ngram::NgramProposer::new(spec_config.ngram);
 
     info!("Scheduler ready");
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index 74b1034b..64a14154 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -22,6 +22,38 @@ pub struct SpeculativeConfig {
     pub ngram: NgramConfig,
 }
 
+impl SpeculativeConfig {
+    /// Read the config from environment variables (the operational switch until
+    /// a first-class server/config knob is wired):
+    ///
+    /// * `OPENINFER_QWEN3_NGRAM_SPEC` = `1`/`true` enables it (default off).
+    /// * `OPENINFER_QWEN3_NGRAM_SPEC_TOKENS` = draft count `K` (default 4).
+    /// * `OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM` = longest suffix to match (default 3).
+    #[must_use]
+    pub fn from_env() -> Self {
+        let defaults = NgramConfig::default();
+        let enabled = std::env::var("OPENINFER_QWEN3_NGRAM_SPEC")
+            .ok()
+            .is_some_and(|v| v == "1" || v.eq_ignore_ascii_case("true"));
+        let num_speculative =
+            env_usize("OPENINFER_QWEN3_NGRAM_SPEC_TOKENS").unwrap_or(defaults.num_speculative);
+        let max_ngram =
+            env_usize("OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM").unwrap_or(defaults.max_ngram);
+        Self {
+            enabled,
+            ngram: NgramConfig {
+                max_ngram,
+                min_ngram: defaults.min_ngram,
+                num_speculative,
+            },
+        }
+    }
+}
+
+fn env_usize(name: &str) -> Option<usize> {
+    std::env::var(name).ok().and_then(|v| v.parse().ok())
+}
+
 /// Greedy speculative acceptance.
 ///
 /// * `proposed` — the `K` candidate tokens from the proposer.

From 83f7e14e8daffebd3041aaa49026f05bd61b2a7c Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 16:44:58 +0800
Subject: [PATCH 13/31] test(qwen3): n-gram speculative decode speedup
 benchmark

Add an ignored GPU benchmark that times greedy vs. speculative decode and
reports forward-pass count, accepted-per-verify, and wall-clock speedup. On the
periodic synthetic prompt (best case) it shows 3.96x / 5.00 tokens per verify;
document the numbers and the best-case caveat.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md        |  24 ++-
 openinfer-qwen3-4b/tests/ngram_speculative.rs | 180 +++++++++++++++++-
 2 files changed, 201 insertions(+), 3 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index ea4f5729..59586ae9 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -118,12 +118,32 @@ still uses plain decode.
 1. **First-class config knob**: env var is the current switch; thread a typed
    knob through `start_qwen3*` / `start_engine*` (and a server flag) so it shows
    up in the engine config rather than the environment.
-2. **Speedup measurement**: acceptance-rate / TPOT on repetitive prompts (the
-   payoff, distinct from the lossless correctness oracle).
+2. **Speedup measurement** (initial numbers below — generalize to realistic
+   prompts and the scheduler loop).
 3. **Batched / vLLM-style verify** (perf): fold the verify tokens into the
    unified batched forward (FlashInfer varlen) with a GPU rejection step,
    instead of the current per-request `batch_prefill`-based verify.
 
+## Measured speedup (best case)
+
+`tests/ngram_speculative.rs::ngram_speculative_speedup` (ignored; needs GPU +
+weights) times greedy vs. speculative on Qwen3-4B (eager, single request, 192
+tokens). On the perfectly periodic synthetic prompt:
+
+| metric            | greedy   | speculative |
+| ----------------- | -------- | ----------- |
+| forward passes    | 191      | 39          |
+| ms / token        | 9.99     | 2.52        |
+| accepted / verify | —        | 5.00 (max with K=4) |
+| wall-clock        | 1908 ms  | 481 ms (**3.96x**) |
+
+This is the ceiling: the prompt is exactly periodic so every draft is accepted.
+Real prompts accept a fraction of drafts, so expect smaller wins; the benchmark
+exists to track acceptance-rate / TPOT as the proposer and verify path evolve.
+Run: `cargo test -p openinfer-qwen3-4b --release --test ngram_speculative \
+ngram_speculative_speedup -- --ignored --nocapture` (`OPENINFER_BENCH_TOKENS`
+overrides the 192-token default).
+
 ## Scope / deferred
 
 First cut is **single-request, greedy, non-CUDA-graph**. Deferred: batched
diff --git a/openinfer-qwen3-4b/tests/ngram_speculative.rs b/openinfer-qwen3-4b/tests/ngram_speculative.rs
index 0d58212b..143e949e 100644
--- a/openinfer-qwen3-4b/tests/ngram_speculative.rs
+++ b/openinfer-qwen3-4b/tests/ngram_speculative.rs
@@ -51,10 +51,14 @@ fn repetitive_prompt() -> Vec<u32> {
 }
 
 fn prefill_item(id: u64, prompt: &[u32]) -> PrefillStepItem {
+    prefill_item_max(id, prompt, MAX_OUTPUT)
+}
+
+fn prefill_item_max(id: u64, prompt: &[u32], max_output: usize) -> PrefillStepItem {
     PrefillStepItem::new(
         RequestId::new(id),
         prompt.to_vec(),
-        MAX_OUTPUT,
+        max_output,
         SamplingParams::default(),
         0,
         false,
@@ -159,3 +163,177 @@ fn ngram_speculative_is_lossless_vs_greedy() {
         "greedy speculative decode must be token-identical to plain greedy decode"
     );
 }
+
+/// Counters for the speculative run: how many forward passes (verify + fallback
+/// single decodes) it took and how many tokens those passes committed.
+struct SpecStats {
+    tokens: usize,
+    forwards: usize,
+    drafted_forwards: usize,
+    drafted_tokens: usize,
+}
+
+/// Greedy speculative decode that also tallies forward passes / acceptance.
+fn speculative_decode_timed(
+    ex: &mut Qwen3Executor,
+    id: u64,
+    prompt: &[u32],
+    n_generate: usize,
+) -> (Vec<u32>, SpecStats) {
+    let proposer = NgramProposer::new(NgramConfig {
+        max_ngram: 3,
+        min_ngram: 1,
+        num_speculative: 4,
+    });
+    let pr = ex
+        .execute_prefill(PrefillPlan {
+            requests: &[prefill_item_max(id, prompt, n_generate + 8)],
+            echo: false,
+        })
+        .expect("prefill");
+    let first = pr.requests[0].first_token;
+    let mut history = prompt.to_vec();
+    history.push(first);
+    let mut out = vec![first];
+    let mut stats = SpecStats {
+        tokens: 0,
+        forwards: 0,
+        drafted_forwards: 0,
+        drafted_tokens: 0,
+    };
+
+    while out.len() < n_generate {
+        let last = *out.last().unwrap();
+        let drafts = proposer.propose(&history);
+        if drafts.is_empty() {
+            let dr = ex
+                .execute_decode(DecodePlan {
+                    requests: &[DecodeStepItem::new(
+                        RequestId::new(id),
+                        last,
+                        SamplingParams::default(),
+                        0,
+                        0.0,
+                    )],
+                })
+                .expect("decode");
+            let t = dr.requests[0].token;
+            history.push(t);
+            out.push(t);
+            stats.forwards += 1;
+        } else {
+            let committed = ex
+                .execute_speculative(&SpeculativeStepItem::new(RequestId::new(id), last, drafts))
+                .expect("speculative");
+            stats.forwards += 1;
+            stats.drafted_forwards += 1;
+            stats.drafted_tokens += committed.len();
+            for t in committed {
+                history.push(t);
+                out.push(t);
+            }
+        }
+    }
+    out.truncate(n_generate);
+    stats.tokens = n_generate;
+    (out, stats)
+}
+
+/// Wall-clock + acceptance benchmark; ignored by default (needs a GPU + weights
+/// and is timing-sensitive). Run with:
+/// `cargo test -p openinfer-qwen3-4b --release --test ngram_speculative \
+///   ngram_speculative_speedup -- --ignored --nocapture`
+#[test]
+#[ignore]
+fn ngram_speculative_speedup() {
+    use std::time::Instant;
+
+    let Some(model_path) = model_path_or_skip() else {
+        return;
+    };
+    let n_generate = std::env::var("OPENINFER_BENCH_TOKENS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(192usize);
+
+    let mut ex =
+        Qwen3Executor::from_runtime(&model_path, false, &[0]).expect("build eager executor");
+    ex.set_prefix_cache_enabled(false);
+    let prompt = repetitive_prompt();
+
+    // Warm up CUDA/cuBLAS so the first run's lazy init doesn't skew timings.
+    let _ = greedy_decode(&mut ex, 100, &prompt);
+
+    let t0 = Instant::now();
+    let baseline = {
+        // Inline greedy decode for `n_generate` tokens (greedy_decode is fixed at
+        // N_GENERATE), counting decode forwards.
+        let pr = ex
+            .execute_prefill(PrefillPlan {
+                requests: &[prefill_item_max(1, &prompt, n_generate + 8)],
+                echo: false,
+            })
+            .expect("prefill");
+        let mut last = pr.requests[0].first_token;
+        let mut out = vec![last];
+        while out.len() < n_generate {
+            let dr = ex
+                .execute_decode(DecodePlan {
+                    requests: &[DecodeStepItem::new(
+                        RequestId::new(1),
+                        last,
+                        SamplingParams::default(),
+                        0,
+                        0.0,
+                    )],
+                })
+                .expect("decode");
+            last = dr.requests[0].token;
+            out.push(last);
+        }
+        out
+    };
+    let baseline_dt = t0.elapsed();
+    let baseline_forwards = baseline.len() - 1; // one decode forward per token after prefill
+
+    let t1 = Instant::now();
+    let (spec_out, stats) = speculative_decode_timed(&mut ex, 2, &prompt, n_generate);
+    let spec_dt = t1.elapsed();
+
+    assert_eq!(
+        baseline, spec_out,
+        "speculative output must stay token-identical to greedy"
+    );
+
+    let baseline_tpot = baseline_dt.as_secs_f64() / baseline_forwards as f64 * 1e3;
+    let spec_tpot = spec_dt.as_secs_f64() / (stats.tokens - 1) as f64 * 1e3;
+    let accept_per_verify = stats.drafted_tokens as f64 / stats.drafted_forwards.max(1) as f64;
+
+    eprintln!("=== n-gram speculative decode benchmark ===");
+    eprintln!("tokens generated   : {}", stats.tokens);
+    eprintln!(
+        "baseline (greedy)  : {:>7.1} ms, {} forwards, {:.3} ms/token",
+        baseline_dt.as_secs_f64() * 1e3,
+        baseline_forwards,
+        baseline_tpot
+    );
+    eprintln!(
+        "speculative        : {:>7.1} ms, {} forwards ({} verify), {:.3} ms/token",
+        spec_dt.as_secs_f64() * 1e3,
+        stats.forwards,
+        stats.drafted_forwards,
+        spec_tpot
+    );
+    eprintln!(
+        "accepted/verify    : {:.2} tokens (K=4, so up to 5 per verify)",
+        accept_per_verify
+    );
+    eprintln!(
+        "forward-pass saving: {:.1}% fewer model calls",
+        (1.0 - stats.forwards as f64 / baseline_forwards as f64) * 100.0
+    );
+    eprintln!(
+        "wall-clock speedup : {:.2}x",
+        baseline_dt.as_secs_f64() / spec_dt.as_secs_f64()
+    );
+}

From ca5324a2dbe78e1f3c43401035596464e7e9ca32 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 17:00:35 +0800
Subject: [PATCH 14/31] refactor(qwen3): abstract speculative decode behind a
 proposer trait

Introduce SpeculativeProposer (propose(&[u32]) -> Vec<u32>) as the single axis
of variation between speculative methods, with NgramProposer implementing it.
SpeculativeConfig now carries a SpeculativeMethod enum + build_proposer factory;
scheduler_loop builds one boxed proposer and speculative_decode_step takes
&dyn SpeculativeProposer. Verify forward, KV rollback, greedy acceptance, and
the streaming step are reused unchanged. Acceptance (greedy-only) and the
token-only proposer context are left as documented future seams.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md        | 21 ++++++
 openinfer-qwen3-4b/src/ngram.rs               |  6 ++
 openinfer-qwen3-4b/src/scheduler.rs           | 13 ++--
 .../src/scheduler/speculative.rs              |  8 ++-
 openinfer-qwen3-4b/src/speculative.rs         | 68 ++++++++++++++++---
 5 files changed, 99 insertions(+), 17 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 59586ae9..9a4ec1b5 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -101,6 +101,27 @@ same prompt under greedy params.
 - Mock-tested (`FakeExecutor::execute_speculative`): streams every committed
   token + advances state; commits past `max_tokens` truncate, finish, retire.
 
+## Pluggable proposer
+
+The proposer is the one axis that varies between speculative methods; everything
+below it is method-agnostic and reused unchanged:
+
+- `speculative::SpeculativeProposer` — `fn propose(&self, context: &[u32]) -> Vec<u32>`.
+  `NgramProposer` implements it; a draft-model proposer slots in the same way.
+- `SpeculativeConfig.method: SpeculativeMethod` (enum, one variant per method) +
+  `build_proposer()` factory. `scheduler_loop` builds one boxed
+  `dyn SpeculativeProposer` at startup and `speculative_decode_step` takes
+  `&dyn SpeculativeProposer`.
+- Reused regardless of method: verify forward (`SpeculativeVerify`), KV
+  reserve/rollback (`schedule/apply_speculative`), `accept_greedy`, and the
+  scheduler step that streams committed tokens.
+
+Two seams are intentionally **not** abstracted yet (no second implementation to
+test against): **acceptance** (`accept_greedy` is greedy-only; sampling/rejection
+needs verify to return distributions, not just argmax) and the proposer
+**context** (token-only `&[u32]` fits n-gram and draft-model; EAGLE/Medusa will
+need request id + hidden states).
+
 ## Enabling it
 
 `scheduler_loop` builds the config via `SpeculativeConfig::from_env()`
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index ecf8af95..f3c9df53 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -89,6 +89,12 @@ impl NgramProposer {
     }
 }
 
+impl crate::speculative::SpeculativeProposer for NgramProposer {
+    fn propose(&self, context: &[u32]) -> Vec<u32> {
+        NgramProposer::propose(self, context)
+    }
+}
+
 /// Find the latest start index `i < len - n` such that `context[i..i + n]`
 /// equals `suffix` (the trailing `n` tokens). "Latest" prefers the most recent
 /// context. Guaranteed to leave at least one following token at `i + n`.
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index 067ac4e6..d4f6d359 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -278,12 +278,9 @@ fn scheduler_loop<E>(
     // single-token decode.
     let spec_config = crate::speculative::SpeculativeConfig::from_env();
     if spec_config.enabled {
-        info!(
-            "n-gram speculative decode enabled (K={}, max_ngram={})",
-            spec_config.ngram.num_speculative, spec_config.ngram.max_ngram
-        );
+        info!("speculative decode enabled: {}", spec_config.describe());
     }
-    let ngram_proposer = crate::ngram::NgramProposer::new(spec_config.ngram);
+    let spec_proposer = spec_config.build_proposer();
 
     info!("Scheduler ready");
 
@@ -358,7 +355,11 @@ fn scheduler_loop<E>(
         // Speculative decode replaces the batched single-token decode on
         // pure-decode ticks (no new prefill this step). Lossless under greedy.
         if spec_config.enabled && matches!(plan, self::plan::ExecutionPlan::Decode) {
-            self::speculative::speculative_decode_step(&mut executor, &mut active, &ngram_proposer);
+            self::speculative::speculative_decode_step(
+                &mut executor,
+                &mut active,
+                spec_proposer.as_ref(),
+            );
             continue;
         }
         let failure_targets = failure_targets_for(&active, &plan);
diff --git a/openinfer-qwen3-4b/src/scheduler/speculative.rs b/openinfer-qwen3-4b/src/scheduler/speculative.rs
index 87c5df93..664cc358 100644
--- a/openinfer-qwen3-4b/src/scheduler/speculative.rs
+++ b/openinfer-qwen3-4b/src/scheduler/speculative.rs
@@ -15,16 +15,18 @@ use log::warn;
 use openinfer_core::engine::FinishReason;
 
 use crate::executor::{DecodePlan, DecodeStepItem, ModelExecutor, SpeculativeStepItem};
-use crate::ngram::NgramProposer;
+use crate::speculative::SpeculativeProposer;
 
 use super::{ActiveRequestState, TokenEvent};
 
 /// Advance every active request by one speculative step. Finished requests are
-/// dropped from `active`.
+/// dropped from `active`. The proposer is method-agnostic ([`SpeculativeProposer`]);
+/// everything below it (verify, KV rollback, acceptance, streaming) is reused
+/// unchanged regardless of which proposer is plugged in.
 pub(super) fn speculative_decode_step(
     executor: &mut impl ModelExecutor,
     active: &mut Vec<ActiveRequestState>,
-    proposer: &NgramProposer,
+    proposer: &dyn SpeculativeProposer,
 ) {
     let mut to_retire = Vec::new();
     for idx in 0..active.len() {
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index 64a14154..15f55033 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -7,19 +7,51 @@
 //! this live in the decode path and are intentionally not part of this module,
 //! so the acceptance rule stays pure and unit-tested.
 
-use crate::ngram::NgramConfig;
+use crate::ngram::{NgramConfig, NgramProposer};
+
+/// A draft-token source for speculative decoding.
+///
+/// This is the single axis of variation between speculative methods: n-gram /
+/// prompt-lookup today, draft-model or EAGLE/Medusa later. Everything around it
+/// — the verify forward, KV reserve/rollback, greedy acceptance, and the
+/// scheduler step that streams committed tokens — is method-agnostic and is
+/// reused unchanged.
+///
+/// The context is the request's full token sequence so far (prompt + generated
+/// tokens). Returning an empty `Vec` means "no draft this step", and the decode
+/// path falls back to a single-token decode. A token-only context fits n-gram
+/// and draft-model proposers; hidden-state proposers (EAGLE/Medusa) will need a
+/// richer context (request id + hidden states), which is a deliberate future
+/// extension rather than something modelled speculatively here.
+pub trait SpeculativeProposer: Send + Sync {
+    /// Propose up to `K` continuation tokens for `context`.
+    fn propose(&self, context: &[u32]) -> Vec<u32>;
+}
+
+/// Which proposer to build. Add a variant per new speculative method.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum SpeculativeMethod {
+    /// N-gram / prompt-lookup (no draft model).
+    Ngram(NgramConfig),
+}
+
+impl Default for SpeculativeMethod {
+    fn default() -> Self {
+        Self::Ngram(NgramConfig::default())
+    }
+}
 
 /// Speculative-decoding configuration.
 ///
-/// `Default` is disabled with the default n-gram settings (`enabled: false`),
-/// so the decode path runs the normal one-token step until explicitly turned on.
+/// `Default` is disabled with the default method (`enabled: false`), so the
+/// decode path runs the normal one-token step until explicitly turned on.
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Default)]
 pub struct SpeculativeConfig {
     /// Master switch. When `false` the decode path runs the normal one-token
-    /// step and never calls the proposer.
+    /// step and never builds a proposer.
     pub enabled: bool,
-    /// N-gram proposer settings (ignored when `enabled` is `false`).
-    pub ngram: NgramConfig,
+    /// Which proposer to use (ignored when `enabled` is `false`).
+    pub method: SpeculativeMethod,
 }
 
 impl SpeculativeConfig {
@@ -41,11 +73,31 @@ impl SpeculativeConfig {
             env_usize("OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM").unwrap_or(defaults.max_ngram);
         Self {
             enabled,
-            ngram: NgramConfig {
+            method: SpeculativeMethod::Ngram(NgramConfig {
                 max_ngram,
                 min_ngram: defaults.min_ngram,
                 num_speculative,
-            },
+            }),
+        }
+    }
+
+    /// Construct the proposer described by [`Self::method`]. The scheduler calls
+    /// this once at startup and reuses the boxed proposer across all requests.
+    #[must_use]
+    pub fn build_proposer(&self) -> Box<dyn SpeculativeProposer> {
+        match self.method {
+            SpeculativeMethod::Ngram(cfg) => Box::new(NgramProposer::new(cfg)),
+        }
+    }
+
+    /// Human-readable one-liner for startup logging.
+    #[must_use]
+    pub fn describe(&self) -> String {
+        match self.method {
+            SpeculativeMethod::Ngram(cfg) => format!(
+                "n-gram (K={}, max_ngram={})",
+                cfg.num_speculative, cfg.max_ngram
+            ),
         }
     }
 }

From 1adef439e6a9cb99f2a37bd9b337a338c142a215 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 17:07:33 +0800
Subject: [PATCH 15/31] docs(qwen3): refresh speculative module header for the
 proposer abstraction

The module-level doc still described the file as ngram-specific glue, which
contradicts the SpeculativeProposer trait it now defines. Rewrite it to list the
three method-agnostic pieces (proposer trait, config/factory, greedy acceptance).

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 openinfer-qwen3-4b/src/speculative.rs | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index 15f55033..aea3412a 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -1,11 +1,17 @@
-//! Speculative-decoding glue for Qwen3: config and greedy acceptance.
+//! Method-agnostic core of speculative decoding for Qwen3.
 //!
-//! The proposer ([`crate::ngram`]) suggests `K` candidate tokens; the target
-//! model is then run once on `[last_confirmed, c_0, .., c_{K-1}]` to produce a
-//! greedy token at each of the `K + 1` positions. [`accept_greedy`] turns those
-//! into the tokens to commit. The GPU verify forward and KV rollback that feed
-//! this live in the decode path and are intentionally not part of this module,
-//! so the acceptance rule stays pure and unit-tested.
+//! Three pieces live here, none tied to a specific proposer:
+//! * [`SpeculativeProposer`] — the draft-token source (the one axis that varies
+//!   between methods; [`crate::ngram::NgramProposer`] is the first impl).
+//! * [`SpeculativeConfig`] / [`SpeculativeMethod`] — what to run and how to
+//!   build the proposer ([`SpeculativeConfig::build_proposer`]).
+//! * [`accept_greedy`] — greedy acceptance: given the `K` drafts and the target
+//!   model's greedy token at each of the `K + 1` verify positions, return the
+//!   tokens to commit.
+//!
+//! The GPU verify forward and KV reserve/rollback that feed `accept_greedy`
+//! live in the decode path (executor) and are intentionally not part of this
+//! module, so the acceptance rule stays pure and unit-tested.
 
 use crate::ngram::{NgramConfig, NgramProposer};
 

From 31e0aec5e0cbd22f017414e217e170f337514878 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 17:13:39 +0800
Subject: [PATCH 16/31] refactor(qwen3): push n-gram env parsing down to
 NgramConfig::from_env

SpeculativeConfig::from_env was hardcoding the n-gram-specific OPENINFER_QWEN3_
NGRAM_SPEC_* variables, so the generic config knew the proposer's knobs. Move
those to NgramConfig::from_env (the proposer owns its own parameters) and leave
only the generic switch on SpeculativeConfig. Rename the switch to the generic
OPENINFER_QWEN3_SPEC and the n-gram knobs to OPENINFER_QWEN3_NGRAM_{TOKENS,
MAX_NGRAM}; a OPENINFER_QWEN3_SPEC_METHOD dispatch slots in when a second
proposer lands.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md |  9 ++++----
 openinfer-qwen3-4b/src/ngram.rs        | 24 +++++++++++++++++++++
 openinfer-qwen3-4b/src/speculative.rs  | 29 ++++++++------------------
 3 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 9a4ec1b5..bb689bb1 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -125,11 +125,12 @@ need request id + hidden states).
 ## Enabling it
 
 `scheduler_loop` builds the config via `SpeculativeConfig::from_env()`
-(default-off). Operational switch:
+(default-off). The generic switch lives on `SpeculativeConfig`; each method
+parses its own knobs (`NgramConfig::from_env`):
 
-- `OPENINFER_QWEN3_NGRAM_SPEC=1` — turn it on.
-- `OPENINFER_QWEN3_NGRAM_SPEC_TOKENS=K` — draft count (default 4).
-- `OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM=N` — longest matched suffix (default 3).
+- `OPENINFER_QWEN3_SPEC=1` — turn speculation on (generic).
+- `OPENINFER_QWEN3_NGRAM_TOKENS=K` — draft count (n-gram, default 4).
+- `OPENINFER_QWEN3_NGRAM_MAX_NGRAM=N` — longest matched suffix (n-gram, default 3).
 
 Only the non-LoRA `scheduler_loop` reads it; the unified prefill+decode tick
 still uses plain decode.
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index f3c9df53..004702da 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -36,6 +36,30 @@ impl Default for NgramConfig {
     }
 }
 
+impl NgramConfig {
+    /// Read the n-gram-specific knobs from the environment, falling back to
+    /// [`Default`] for anything unset. The generic on/off switch lives in
+    /// [`crate::speculative::SpeculativeConfig::from_env`]; this only owns the
+    /// proposer's own parameters:
+    ///
+    /// * `OPENINFER_QWEN3_NGRAM_TOKENS` = draft count `K`.
+    /// * `OPENINFER_QWEN3_NGRAM_MAX_NGRAM` = longest suffix to match.
+    #[must_use]
+    pub fn from_env() -> Self {
+        let defaults = Self::default();
+        Self {
+            max_ngram: env_usize("OPENINFER_QWEN3_NGRAM_MAX_NGRAM").unwrap_or(defaults.max_ngram),
+            min_ngram: defaults.min_ngram,
+            num_speculative: env_usize("OPENINFER_QWEN3_NGRAM_TOKENS")
+                .unwrap_or(defaults.num_speculative),
+        }
+    }
+}
+
+fn env_usize(name: &str) -> Option<usize> {
+    std::env::var(name).ok().and_then(|v| v.parse().ok())
+}
+
 /// Stateless n-gram / prompt-lookup proposer.
 ///
 /// The proposer keeps no history of its own; each call scans the supplied
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index aea3412a..e3eb7b00 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -61,29 +61,22 @@ pub struct SpeculativeConfig {
 }
 
 impl SpeculativeConfig {
-    /// Read the config from environment variables (the operational switch until
-    /// a first-class server/config knob is wired):
+    /// Read the config from the environment (the operational switch until a
+    /// first-class server/config knob is wired). This owns only the generic
+    /// switch; each method parses its own knobs (see [`NgramConfig::from_env`]):
     ///
-    /// * `OPENINFER_QWEN3_NGRAM_SPEC` = `1`/`true` enables it (default off).
-    /// * `OPENINFER_QWEN3_NGRAM_SPEC_TOKENS` = draft count `K` (default 4).
-    /// * `OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM` = longest suffix to match (default 3).
+    /// * `OPENINFER_QWEN3_SPEC` = `1`/`true` enables speculation (default off).
+    ///
+    /// The method defaults to n-gram; add a `OPENINFER_QWEN3_SPEC_METHOD`
+    /// dispatch here when a second proposer lands.
     #[must_use]
     pub fn from_env() -> Self {
-        let defaults = NgramConfig::default();
-        let enabled = std::env::var("OPENINFER_QWEN3_NGRAM_SPEC")
+        let enabled = std::env::var("OPENINFER_QWEN3_SPEC")
             .ok()
             .is_some_and(|v| v == "1" || v.eq_ignore_ascii_case("true"));
-        let num_speculative =
-            env_usize("OPENINFER_QWEN3_NGRAM_SPEC_TOKENS").unwrap_or(defaults.num_speculative);
-        let max_ngram =
-            env_usize("OPENINFER_QWEN3_NGRAM_SPEC_MAX_NGRAM").unwrap_or(defaults.max_ngram);
         Self {
             enabled,
-            method: SpeculativeMethod::Ngram(NgramConfig {
-                max_ngram,
-                min_ngram: defaults.min_ngram,
-                num_speculative,
-            }),
+            method: SpeculativeMethod::Ngram(NgramConfig::from_env()),
         }
     }
 
@@ -108,10 +101,6 @@ impl SpeculativeConfig {
     }
 }
 
-fn env_usize(name: &str) -> Option<usize> {
-    std::env::var(name).ok().and_then(|v| v.parse().ok())
-}
-
 /// Greedy speculative acceptance.
 ///
 /// * `proposed` — the `K` candidate tokens from the proposer.

From 85581b9971871a72a08382e39b3716d16daec819 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 17:30:07 +0800
Subject: [PATCH 17/31] refactor(qwen3): honest speculative docs + tighten
 cohesion

Address review of the speculative abstraction:
- Stop over-promising. The module/trait/docs claimed a method-agnostic core
  reused unchanged; in reality the verify forward returns argmax (greedy-
  specific), accept_greedy is greedy-only, and the scheduler step assumes a
  stateless proposer. Rewrite to say this is a greedy, n-gram-sized seam and
  spell out what a draft-model/EAGLE proposer would additionally need.
- Down-level describe(): drop SpeculativeConfig::describe and add Display on
  SpeculativeMethod + NgramConfig, so a method owns formatting of its own
  fields. scheduler logs spec_config.method.
- num_accepted: now an internal helper for accept_greedy (dedup the accept
  loop), demoted from pub since the executor uses the committed length.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md | 40 ++++++-----
 openinfer-qwen3-4b/src/ngram.rs        | 10 +++
 openinfer-qwen3-4b/src/scheduler.rs    |  2 +-
 openinfer-qwen3-4b/src/speculative.rs  | 99 ++++++++++++++------------
 4 files changed, 86 insertions(+), 65 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index bb689bb1..d4c04be1 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -101,26 +101,32 @@ same prompt under greedy params.
 - Mock-tested (`FakeExecutor::execute_speculative`): streams every committed
   token + advances state; commits past `max_tokens` truncate, finish, retire.
 
-## Pluggable proposer
+## Proposer seam (closed-set, n-gram-sized)
 
-The proposer is the one axis that varies between speculative methods; everything
-below it is method-agnostic and reused unchanged:
+The proposer is factored out as the one piece meant to vary between methods:
 
 - `speculative::SpeculativeProposer` — `fn propose(&self, context: &[u32]) -> Vec<u32>`.
-  `NgramProposer` implements it; a draft-model proposer slots in the same way.
-- `SpeculativeConfig.method: SpeculativeMethod` (enum, one variant per method) +
-  `build_proposer()` factory. `scheduler_loop` builds one boxed
-  `dyn SpeculativeProposer` at startup and `speculative_decode_step` takes
-  `&dyn SpeculativeProposer`.
-- Reused regardless of method: verify forward (`SpeculativeVerify`), KV
-  reserve/rollback (`schedule/apply_speculative`), `accept_greedy`, and the
-  scheduler step that streams committed tokens.
-
-Two seams are intentionally **not** abstracted yet (no second implementation to
-test against): **acceptance** (`accept_greedy` is greedy-only; sampling/rejection
-needs verify to return distributions, not just argmax) and the proposer
-**context** (token-only `&[u32]` fits n-gram and draft-model; EAGLE/Medusa will
-need request id + hidden states).
+- `SpeculativeConfig.method: SpeculativeMethod` (a *closed* enum, one variant per
+  method) + `build_proposer()` factory. `scheduler_loop` builds one boxed
+  `dyn SpeculativeProposer` at startup; `speculative_decode_step` takes
+  `&dyn SpeculativeProposer`. This is closed-set enum dispatch, not an open
+  plugin system — the idiomatic Rust choice for a small known set.
+
+This is a good **n-gram** seam, not yet a general proposer abstraction. The
+trait fits stateless, token-emitting proposers; a draft-model / EAGLE proposer
+would need a wider trait (`&mut self` + per-request create/drop lifecycle, the
+request id, and returning draft probabilities for rejection sampling) **and**
+changes to the scheduler step and verify path. Concretely, the parts below the
+proposer are **greedy-specific**, not method-agnostic:
+
+- the verify forward returns argmax (part of the greedy acceptance rule; sampling
+  acceptance needs distributions),
+- `accept_greedy` is greedy-only,
+- `speculative_decode_step` assumes a stateless proposer (no per-request
+  create/drop).
+
+Widening these is deferred until a second proposer actually lands, so the shapes
+are validated against a real implementation rather than guessed at now.
 
 ## Enabling it
 
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index 004702da..1aacfaea 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -36,6 +36,16 @@ impl Default for NgramConfig {
     }
 }
 
+impl std::fmt::Display for NgramConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "K={}, max_ngram={}",
+            self.num_speculative, self.max_ngram
+        )
+    }
+}
+
 impl NgramConfig {
     /// Read the n-gram-specific knobs from the environment, falling back to
     /// [`Default`] for anything unset. The generic on/off switch lives in
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index d4f6d359..1884032b 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -278,7 +278,7 @@ fn scheduler_loop<E>(
     // single-token decode.
     let spec_config = crate::speculative::SpeculativeConfig::from_env();
     if spec_config.enabled {
-        info!("speculative decode enabled: {}", spec_config.describe());
+        info!("speculative decode enabled: {}", spec_config.method);
     }
     let spec_proposer = spec_config.build_proposer();
 
diff --git a/openinfer-qwen3-4b/src/speculative.rs b/openinfer-qwen3-4b/src/speculative.rs
index e3eb7b00..389305d8 100644
--- a/openinfer-qwen3-4b/src/speculative.rs
+++ b/openinfer-qwen3-4b/src/speculative.rs
@@ -1,34 +1,43 @@
-//! Method-agnostic core of speculative decoding for Qwen3.
+//! Core of **greedy** speculative decoding for Qwen3.
 //!
-//! Three pieces live here, none tied to a specific proposer:
-//! * [`SpeculativeProposer`] — the draft-token source (the one axis that varies
-//!   between methods; [`crate::ngram::NgramProposer`] is the first impl).
-//! * [`SpeculativeConfig`] / [`SpeculativeMethod`] — what to run and how to
-//!   build the proposer ([`SpeculativeConfig::build_proposer`]).
-//! * [`accept_greedy`] — greedy acceptance: given the `K` drafts and the target
-//!   model's greedy token at each of the `K + 1` verify positions, return the
-//!   tokens to commit.
+//! Three pieces live here:
+//! * [`SpeculativeProposer`] — the draft-token source. This is the one piece
+//!   meant to vary between methods; [`crate::ngram::NgramProposer`] is the only
+//!   impl today and the trait is sized for it (see its docs).
+//! * [`SpeculativeConfig`] / [`SpeculativeMethod`] — a *closed set* of methods
+//!   selected by enum dispatch, plus how to build the proposer.
+//! * [`accept_greedy`] — greedy acceptance over the target model's per-position
+//!   argmax.
 //!
-//! The GPU verify forward and KV reserve/rollback that feed `accept_greedy`
-//! live in the decode path (executor) and are intentionally not part of this
-//! module, so the acceptance rule stays pure and unit-tested.
+//! What is **not** generic yet (don't mistake this for a method-agnostic core):
+//! the verify forward in the executor returns argmax, which is part of the
+//! *greedy* acceptance rule — sampling acceptance would need distributions; and
+//! the scheduler step assumes a *stateless* proposer (no per-request
+//! create/drop). Adding a stateful, model-based proposer therefore touches the
+//! trait, the scheduler step, and the verify path, not just this module.
+
+use std::fmt;
 
 use crate::ngram::{NgramConfig, NgramProposer};
 
 /// A draft-token source for speculative decoding.
 ///
-/// This is the single axis of variation between speculative methods: n-gram /
-/// prompt-lookup today, draft-model or EAGLE/Medusa later. Everything around it
-/// — the verify forward, KV reserve/rollback, greedy acceptance, and the
-/// scheduler step that streams committed tokens — is method-agnostic and is
-/// reused unchanged.
-///
 /// The context is the request's full token sequence so far (prompt + generated
-/// tokens). Returning an empty `Vec` means "no draft this step", and the decode
-/// path falls back to a single-token decode. A token-only context fits n-gram
-/// and draft-model proposers; hidden-state proposers (EAGLE/Medusa) will need a
-/// richer context (request id + hidden states), which is a deliberate future
-/// extension rather than something modelled speculatively here.
+/// tokens); returning an empty `Vec` means "no draft this step" and the decode
+/// path falls back to a single-token decode.
+///
+/// Scope: this signature fits n-gram / prompt-lookup proposers, which are
+/// stateless (they rescan the context each step) and emit plain tokens. It is
+/// deliberately **not** a general proposer abstraction — a draft-model or
+/// EAGLE/Medusa proposer needs more than this trait can express:
+/// * `&mut self` / interior mutability plus a per-request create/drop lifecycle
+///   (a draft model keeps its own KV cache per request),
+/// * the request id, to key that per-request state,
+/// * returning draft *probabilities* alongside tokens, for rejection sampling.
+///
+/// Growing the trait to cover those is left until such a proposer actually
+/// lands, so the shape is validated against a real second implementation rather
+/// than guessed at now.
 pub trait SpeculativeProposer: Send + Sync {
     /// Propose up to `K` continuation tokens for `context`.
     fn propose(&self, context: &[u32]) -> Vec<u32>;
@@ -47,6 +56,16 @@ impl Default for SpeculativeMethod {
     }
 }
 
+impl fmt::Display for SpeculativeMethod {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            // Each method formats its own fields (e.g. `NgramConfig: Display`),
+            // so adding a method does not touch this arm beyond the name.
+            Self::Ngram(cfg) => write!(f, "n-gram ({cfg})"),
+        }
+    }
+}
+
 /// Speculative-decoding configuration.
 ///
 /// `Default` is disabled with the default method (`enabled: false`), so the
@@ -88,17 +107,6 @@ impl SpeculativeConfig {
             SpeculativeMethod::Ngram(cfg) => Box::new(NgramProposer::new(cfg)),
         }
     }
-
-    /// Human-readable one-liner for startup logging.
-    #[must_use]
-    pub fn describe(&self) -> String {
-        match self.method {
-            SpeculativeMethod::Ngram(cfg) => format!(
-                "n-gram (K={}, max_ngram={})",
-                cfg.num_speculative, cfg.max_ngram
-            ),
-        }
-    }
 }
 
 /// Greedy speculative acceptance.
@@ -125,24 +133,21 @@ pub fn accept_greedy(proposed: &[u32], target_argmax: &[u32]) -> Vec<u32> {
         proposed.len() + 1,
         "verify must produce one greedy token per candidate plus a bonus"
     );
-    let mut committed = Vec::with_capacity(proposed.len() + 1);
-    let mut i = 0;
-    while i < proposed.len() && proposed[i] == target_argmax[i] {
-        committed.push(proposed[i]);
-        i += 1;
-    }
+    let n = num_accepted(proposed, target_argmax);
+    let mut committed = Vec::with_capacity(n + 1);
+    committed.extend_from_slice(&proposed[..n]);
     // The model's own token at the first divergence (or the bonus continuation
-    // when the whole run was accepted). `i <= proposed.len() < target_argmax.len()`
+    // when the whole run was accepted). `n <= proposed.len() < target_argmax.len()`
     // so this index is always valid.
-    committed.push(target_argmax[i]);
+    committed.push(target_argmax[n]);
     committed
 }
 
-/// Number of candidate tokens accepted before the committed bonus token, i.e.
-/// `accept_greedy(..).len() - 1`. Useful for KV rollback bookkeeping (how many
-/// of the speculatively written candidate positions to keep).
-#[must_use]
-pub fn num_accepted(proposed: &[u32], target_argmax: &[u32]) -> usize {
+/// Number of leading drafts whose token matches the model's argmax — the length
+/// of the accepted prefix. Internal helper for [`accept_greedy`] (which appends
+/// one model token on top); KV rollback in the executor uses the committed
+/// length directly, so this is not part of the public surface.
+fn num_accepted(proposed: &[u32], target_argmax: &[u32]) -> usize {
     let mut i = 0;
     while i < proposed.len() && proposed[i] == target_argmax[i] {
         i += 1;

From 2f6d3e45b619f4f850a2fb14702a25fced1cf1f2 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 18:44:48 +0800
Subject: [PATCH 18/31] fix(qwen3): make speculative step transactional and
 budget-capped

Two robustness fixes for the speculative decode step:
- Transactional schedule: execute_speculative now reverts the
  schedule_speculative KV reservation (new RequestKv::revert_schedule) if the
  verify/commit body fails, so a mid-step error leaves the sequence idle for the
  next step instead of stranded half-scheduled.
- Budget cap: the scheduler caps the draft count to the request's remaining
  token budget, so a verify commits at most up to max_tokens. Otherwise
  schedule_speculative clamps to the budget and the larger accepted run is
  rejected. Test renamed/updated to assert the cap (drafts capped, commit lands
  exactly on the limit) and the docs drop the now-private num_accepted.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md        |  2 +-
 openinfer-kv-cache/src/pool.rs                | 11 +++++++
 openinfer-qwen3-4b/src/executor.rs            | 30 +++++++++++++++++++
 openinfer-qwen3-4b/src/scheduler.rs           | 15 +++++-----
 .../src/scheduler/speculative.rs              | 15 +++++++++-
 5 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index d4c04be1..767e12a9 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -22,7 +22,7 @@ Three of the four pieces already exist or are trivial:
 
 - **Proposer** — `openinfer-qwen3-4b/src/ngram.rs` (`NgramProposer`). Done, unit-tested.
 - **Acceptance** — `openinfer-qwen3-4b/src/speculative.rs` (`accept_greedy`,
-  `num_accepted`, `SpeculativeConfig`). Done, unit-tested.
+  `SpeculativeConfig`). Done, unit-tested.
 - **KV scheduling / rollback** — kvbm's `SchedulableSequence` already implements
   `schedule_speculative` / `apply_speculative` with **automatic LIFO release of
   the blocks pre-allocated for rejected drafts**. Exposed on `RequestKv`
diff --git a/openinfer-kv-cache/src/pool.rs b/openinfer-kv-cache/src/pool.rs
index 9e92abc3..8a030fbe 100644
--- a/openinfer-kv-cache/src/pool.rs
+++ b/openinfer-kv-cache/src/pool.rs
@@ -347,6 +347,17 @@ impl RequestKv {
             .map_err(|e| anyhow::anyhow!("apply_speculative: {e}"))
     }
 
+    /// Revert a scheduled-but-not-applied step, returning the sequence to its
+    /// pre-schedule (idle) state and LIFO-releasing the pre-allocated blocks.
+    /// Used to keep [`Self::schedule_speculative`] transactional: if the verify
+    /// forward fails before [`Self::apply_speculative`], the reservation is
+    /// undone instead of stranding the sequence in a half-scheduled state.
+    pub fn revert_schedule(&mut self) -> anyhow::Result<()> {
+        self.seq
+            .revert_schedule()
+            .map_err(|e| anyhow::anyhow!("revert_schedule: {e}"))
+    }
+
     pub fn release(&mut self) -> anyhow::Result<()> {
         self.seq
             .release()
diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index fc8b2319..b3153f0c 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -865,6 +865,36 @@ impl Qwen3Executor {
                 anyhow::anyhow!("schedule_speculative failed for {:?}: {e}", item.request_id)
             })?;
 
+        // schedule_speculative pre-allocated KV blocks and moved the sequence
+        // into the SpeculativeScheduled state. The verify + apply below must
+        // either complete (apply_speculative) or revert that reservation, so a
+        // mid-step failure leaves the sequence idle for the next step rather
+        // than stranded half-scheduled.
+        let result = self.verify_and_commit(item);
+        if result.is_err() {
+            if let Some(rkv) = self.request_kvs.get_mut(&item.request_id) {
+                if let Err(revert_err) = rkv.revert_schedule() {
+                    log::warn!(
+                        "failed to revert speculative schedule for {:?}: {revert_err}",
+                        item.request_id
+                    );
+                }
+            }
+        }
+        result
+    }
+
+    /// The fallible body of [`Self::execute_speculative`]: build the verify
+    /// input, run the verify forward, greedily accept, and commit. Factored out
+    /// so `execute_speculative` can revert the schedule reservation on any
+    /// error here. Assumes `schedule_speculative` already succeeded.
+    fn verify_and_commit(&mut self, item: &SpeculativeStepItem) -> Result<Vec<u32>> {
+        let k = item.drafts.len();
+        let rkv = self
+            .request_kvs
+            .get_mut(&item.request_id)
+            .expect("request must exist; schedule_speculative just succeeded");
+
         let mut verify_tokens = Vec::with_capacity(k + 1);
         verify_tokens.push(item.token_id);
         verify_tokens.extend_from_slice(&item.drafts);
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index 1884032b..5620abcd 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -272,7 +272,7 @@ fn scheduler_loop<E>(
     // Requests parked while their async CPU-tier KV prefetch loads.
     let mut loading: Vec<PendingRequest> = Vec::new();
 
-    // N-gram speculative decode. Off unless OPENINFER_QWEN3_NGRAM_SPEC is set
+    // N-gram speculative decode. Off unless OPENINFER_QWEN3_SPEC is set
     // (see SpeculativeConfig::from_env). When on, pure-decode ticks route
     // through `speculative::speculative_decode_step` instead of the batched
     // single-token decode.
@@ -1951,21 +1951,22 @@ mod tests {
         assert!(dropped.lock().unwrap().is_empty());
     }
 
-    // Committed tokens past max_tokens are truncated: the step stops emitting at
-    // the limit, finishes the request, and retires it.
+    // A request near its max_tokens budget caps the draft count so the commit
+    // lands exactly on the limit instead of over-committing. max_tokens 3,
+    // generated 1 -> only 2 tokens of budget, so drafts [6,5] are capped to [6]
+    // (1 draft); commit [6,300] -> emit 6 (gen 2), 300 (gen 3 == limit ->
+    // Length), then finish + retire.
     #[test]
-    fn speculative_step_truncates_at_max_tokens() {
+    fn speculative_step_caps_drafts_at_max_tokens() {
         let dropped = Arc::new(Mutex::new(Vec::new()));
         let mut executor = FakeExecutor::new(64, Arc::clone(&dropped));
         executor.ensure_request_tokens(RequestId(0), 3).unwrap();
-        // max_tokens 3, generated 1: commit [6,5,300] -> emit 6 (gen 2), 5 (gen 3
-        // == limit -> Length), 300 never emitted.
         let (state, mut rx) = spec_active(RequestId(0), 3, vec![5, 6, 5]);
         let mut active = vec![state];
 
         speculative::speculative_decode_step(&mut executor, &mut active, &ngram2());
 
-        assert_eq!(drain_tokens(&mut rx), vec![6, 5]);
+        assert_eq!(drain_tokens(&mut rx), vec![6, 300]);
         assert!(active.is_empty(), "request retired at length limit");
         assert_eq!(dropped.lock().unwrap().clone(), vec![0]);
     }
diff --git a/openinfer-qwen3-4b/src/scheduler/speculative.rs b/openinfer-qwen3-4b/src/scheduler/speculative.rs
index 664cc358..04aa7f58 100644
--- a/openinfer-qwen3-4b/src/scheduler/speculative.rs
+++ b/openinfer-qwen3-4b/src/scheduler/speculative.rs
@@ -31,7 +31,20 @@ pub(super) fn speculative_decode_step(
     let mut to_retire = Vec::new();
     for idx in 0..active.len() {
         let request_id = active[idx].request_id;
-        let drafts = proposer.propose(&active[idx].token_history);
+        let mut drafts = proposer.propose(&active[idx].token_history);
+
+        // A verify step commits at most `drafts.len() + 1` tokens. Cap the
+        // draft count to the request's remaining token budget so the commit can
+        // never exceed it — otherwise the executor's `schedule_speculative`
+        // clamps to the budget and the larger accepted run is rejected. Tokens
+        // past the budget would be truncated by `apply_committed` anyway.
+        let remaining = active[idx]
+            .max_tokens
+            .saturating_sub(active[idx].generated_count);
+        let max_drafts = remaining.saturating_sub(1);
+        if drafts.len() > max_drafts {
+            drafts.truncate(max_drafts);
+        }
 
         let committed = match commit_tokens(executor, &active[idx], drafts) {
             Ok(tokens) => tokens,

From 8969a2f78047a4996d2a224a6d48c3243d652976 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 11 Jun 2026 19:10:33 +0800
Subject: [PATCH 19/31] fix(qwen3): gate speculation to greedy, no-logprobs
 requests

Addresses Codex review on #349. With OPENINFER_QWEN3_SPEC on, the scheduler
routed every pure-decode tick through the speculative path regardless of each
request's SamplingParams: a temperature>0 request was forced to argmax and a
logprobs request silently lost its decode logprobs.

speculative_decode_step now decides per request: only greedy
(SamplingParams::is_greedy()) requests that ask for no decode logprobs take the
speculative step; everyone else takes a normal sampled single-token decode with
their own params, logprobs, and a fresh random_val. Factored emit_one out of
apply_committed so both paths share the stop/max-token handling. Adds two tests
(sampled and logprobs requests bypass speculation) and documents the gate.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/models/qwen3/ngram-speculative.md        |   9 +
 openinfer-qwen3-4b/src/scheduler.rs           |  51 ++++-
 .../src/scheduler/speculative.rs              | 202 ++++++++++++------
 3 files changed, 191 insertions(+), 71 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 767e12a9..36966787 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -141,6 +141,15 @@ parses its own knobs (`NgramConfig::from_env`):
 Only the non-LoRA `scheduler_loop` reads it; the unified prefill+decode tick
 still uses plain decode.
 
+**Per-request eligibility.** Even with the switch on, only requests that are
+greedy (`SamplingParams::is_greedy()`) **and** ask for no decode logprobs
+(`logprobs == 0`) take the speculative path. Speculation verifies with argmax
+and emits no per-token logprobs, so a sampled request would otherwise be forced
+to argmax and a logprobs request would silently lose them. Any ineligible
+request takes a normal sampled single-token decode (its own params, logprobs,
+and a fresh `random_val`) on that tick, so enabling speculation never changes a
+sampled request's output or strips requested logprobs.
+
 ## Remaining work
 
 1. **First-class config knob**: env var is the current switch; thread a typed
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index 5620abcd..6f791c9e 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -359,6 +359,7 @@ fn scheduler_loop<E>(
                 &mut executor,
                 &mut active,
                 spec_proposer.as_ref(),
+                &mut rng,
             );
             continue;
         }
@@ -1940,8 +1941,9 @@ mod tests {
         // FakeExecutor commits drafts + bonus 300 -> [6,5,300].
         let (state, mut rx) = spec_active(RequestId(0), 100, vec![5, 6, 5]);
         let mut active = vec![state];
+        let mut rng = StdRng::seed_from_u64(0);
 
-        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2());
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2(), &mut rng);
 
         assert_eq!(drain_tokens(&mut rx), vec![6, 5, 300]);
         assert_eq!(active.len(), 1, "request continues");
@@ -1963,11 +1965,56 @@ mod tests {
         executor.ensure_request_tokens(RequestId(0), 3).unwrap();
         let (state, mut rx) = spec_active(RequestId(0), 3, vec![5, 6, 5]);
         let mut active = vec![state];
+        let mut rng = StdRng::seed_from_u64(0);
 
-        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2());
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2(), &mut rng);
 
         assert_eq!(drain_tokens(&mut rx), vec![6, 300]);
         assert!(active.is_empty(), "request retired at length limit");
         assert_eq!(dropped.lock().unwrap().clone(), vec![0]);
     }
+
+    // A non-greedy (sampled) request must NOT use the speculative path even when
+    // speculation is enabled: it takes a normal single-token decode (FakeExecutor
+    // -> 200 + id), so its sampling is preserved instead of being forced to argmax.
+    #[test]
+    fn sampled_request_bypasses_speculation() {
+        let dropped = Arc::new(Mutex::new(Vec::new()));
+        let mut executor = FakeExecutor::new(64, Arc::clone(&dropped));
+        executor.ensure_request_tokens(RequestId(0), 3).unwrap();
+        let (mut state, mut rx) = spec_active(RequestId(0), 100, vec![5, 6, 5]);
+        state.params.temperature = 1.0; // non-greedy -> not spec-eligible
+        let mut active = vec![state];
+        let mut rng = StdRng::seed_from_u64(0);
+
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2(), &mut rng);
+
+        // One token from the normal decode path, not the multi-token spec commit.
+        assert_eq!(drain_tokens(&mut rx), vec![200]);
+        assert_eq!(active.len(), 1, "request continues");
+        assert_eq!(active[0].generated_count, 2, "1 + 1 decoded");
+        assert_eq!(active[0].last_token, 200);
+        assert_eq!(active[0].token_history, vec![5, 6, 5, 200]);
+        assert!(dropped.lock().unwrap().is_empty());
+    }
+
+    // A request that asked for decode logprobs is likewise not spec-eligible
+    // (speculation drops per-token logprobs), so it takes the normal decode path.
+    #[test]
+    fn logprobs_request_bypasses_speculation() {
+        let dropped = Arc::new(Mutex::new(Vec::new()));
+        let mut executor = FakeExecutor::new(64, Arc::clone(&dropped));
+        executor.ensure_request_tokens(RequestId(0), 3).unwrap();
+        let (mut state, mut rx) = spec_active(RequestId(0), 100, vec![5, 6, 5]);
+        state.logprobs = 1; // requested decode logprobs -> not spec-eligible
+        let mut active = vec![state];
+        let mut rng = StdRng::seed_from_u64(0);
+
+        speculative::speculative_decode_step(&mut executor, &mut active, &ngram2(), &mut rng);
+
+        assert_eq!(drain_tokens(&mut rx), vec![200]);
+        assert_eq!(active.len(), 1, "request continues");
+        assert_eq!(active[0].last_token, 200);
+        assert!(dropped.lock().unwrap().is_empty());
+    }
 }
diff --git a/openinfer-qwen3-4b/src/scheduler/speculative.rs b/openinfer-qwen3-4b/src/scheduler/speculative.rs
index 04aa7f58..05636652 100644
--- a/openinfer-qwen3-4b/src/scheduler/speculative.rs
+++ b/openinfer-qwen3-4b/src/scheduler/speculative.rs
@@ -1,55 +1,60 @@
 //! Self-contained n-gram speculative decode step for the scheduler.
 //!
 //! When speculation is enabled this replaces the batched single-token decode
-//! for the active set. Per request: propose drafts from the request's token
-//! history, run one speculative verify (or fall back to a single decode when
-//! there is no draft), then stream the committed tokens — applying stop /
-//! max-token handling to each.
+//! for the active set. Per request, IF the request is spec-eligible (greedy and
+//! not asking for decode logprobs): propose drafts from its token history, run
+//! one speculative verify (or fall back to a single decode when there is no
+//! draft), then stream the committed tokens.
 //!
-//! Greedy speculation is lossless, so this emits the same tokens as the normal
-//! decode path, just (often) several per step. It is kept isolated from the
-//! generic plan/resolve/effects pipeline, which assumes exactly one token per
-//! request per step.
+//! Speculation verifies with **argmax** and drops per-token logprobs, so it is
+//! only valid for greedy, no-logprobs requests. Any other request takes a
+//! normal sampled single-token decode this tick (with its own sampling params,
+//! logprobs, and a fresh `random_val`), so enabling speculation never silently
+//! changes a sampled request's output or strips its requested logprobs.
+//!
+//! Greedy speculation is lossless, so eligible requests emit the same tokens as
+//! the normal decode path, just (often) several per step. This is kept isolated
+//! from the generic plan/resolve/effects pipeline, which assumes exactly one
+//! token per request per step.
 
 use log::warn;
-use openinfer_core::engine::FinishReason;
+use openinfer_core::engine::{FinishReason, TokenLogprob};
+use rand::rngs::StdRng;
 
 use crate::executor::{DecodePlan, DecodeStepItem, ModelExecutor, SpeculativeStepItem};
 use crate::speculative::SpeculativeProposer;
 
 use super::{ActiveRequestState, TokenEvent};
 
-/// Advance every active request by one speculative step. Finished requests are
-/// dropped from `active`. The proposer is method-agnostic ([`SpeculativeProposer`]);
-/// everything below it (verify, KV rollback, acceptance, streaming) is reused
-/// unchanged regardless of which proposer is plugged in.
+/// Advance every active request by one step. Spec-eligible requests take a
+/// speculative step; the rest take a normal sampled decode. Finished requests
+/// are dropped from `active`. The proposer is method-agnostic
+/// ([`SpeculativeProposer`]).
 pub(super) fn speculative_decode_step(
     executor: &mut impl ModelExecutor,
     active: &mut Vec<ActiveRequestState>,
     proposer: &dyn SpeculativeProposer,
+    rng: &mut StdRng,
 ) {
     let mut to_retire = Vec::new();
     for idx in 0..active.len() {
         let request_id = active[idx].request_id;
-        let mut drafts = proposer.propose(&active[idx].token_history);
 
-        // A verify step commits at most `drafts.len() + 1` tokens. Cap the
-        // draft count to the request's remaining token budget so the commit can
-        // never exceed it — otherwise the executor's `schedule_speculative`
-        // clamps to the budget and the larger accepted run is rejected. Tokens
-        // past the budget would be truncated by `apply_committed` anyway.
-        let remaining = active[idx]
-            .max_tokens
-            .saturating_sub(active[idx].generated_count);
-        let max_drafts = remaining.saturating_sub(1);
-        if drafts.len() > max_drafts {
-            drafts.truncate(max_drafts);
-        }
+        // Speculation is argmax-based and logprob-free: only greedy requests
+        // that don't ask for decode logprobs may use it. Everyone else takes a
+        // normal sampled decode so their semantics are preserved.
+        let spec_eligible = active[idx].params.is_greedy() && active[idx].logprobs == 0;
+        let outcome = if spec_eligible {
+            speculative_one(executor, &mut active[idx], proposer)
+        } else {
+            sampled_decode_one(executor, &mut active[idx], rng)
+        };
 
-        let committed = match commit_tokens(executor, &active[idx], drafts) {
-            Ok(tokens) => tokens,
+        match outcome {
+            Ok(true) => to_retire.push(idx),
+            Ok(false) => {}
             Err(e) => {
-                warn!("speculative decode failed for {request_id:?}: {e}");
+                warn!("decode step failed for {request_id:?}: {e}");
                 let req = &active[idx];
                 let _ = req.token_tx.send(TokenEvent::Finished {
                     finish_reason: FinishReason::Stop,
@@ -58,12 +63,7 @@ pub(super) fn speculative_decode_step(
                 });
                 let _ = executor.drop_request(request_id);
                 to_retire.push(idx);
-                continue;
             }
-        };
-
-        if apply_committed(executor, &mut active[idx], &committed) {
-            to_retire.push(idx);
         }
     }
     to_retire.sort_unstable();
@@ -73,8 +73,60 @@ pub(super) fn speculative_decode_step(
     }
 }
 
-/// Verify the drafts (or fall back to a single decode) and return the committed
-/// tokens. Always at least one token.
+/// Greedy speculative step for one eligible request: propose, verify, commit,
+/// stream. Returns `true` if the request finished.
+fn speculative_one(
+    executor: &mut impl ModelExecutor,
+    req: &mut ActiveRequestState,
+    proposer: &dyn SpeculativeProposer,
+) -> anyhow::Result<bool> {
+    let mut drafts = proposer.propose(&req.token_history);
+
+    // A verify step commits at most `drafts.len() + 1` tokens. Cap the draft
+    // count to the request's remaining token budget so the commit can never
+    // exceed it — otherwise the executor's `schedule_speculative` clamps to the
+    // budget and the larger accepted run is rejected. Tokens past the budget
+    // would be truncated by `apply_committed` anyway.
+    let remaining = req.max_tokens.saturating_sub(req.generated_count);
+    let max_drafts = remaining.saturating_sub(1);
+    if drafts.len() > max_drafts {
+        drafts.truncate(max_drafts);
+    }
+
+    let committed = commit_tokens(executor, req, drafts)?;
+    Ok(apply_committed(executor, req, &committed))
+}
+
+/// Normal sampled single-token decode for a request that isn't spec-eligible.
+/// Uses the request's own sampling params, logprobs, and a fresh `random_val`,
+/// then streams the one token (with its logprob). Returns `true` if finished.
+fn sampled_decode_one(
+    executor: &mut impl ModelExecutor,
+    req: &mut ActiveRequestState,
+    rng: &mut StdRng,
+) -> anyhow::Result<bool> {
+    let result = executor.execute_decode(DecodePlan {
+        requests: &[DecodeStepItem {
+            request_id: req.request_id,
+            token_id: req.last_token,
+            params: req.params,
+            logprobs: req.logprobs,
+            lora_adapter: req.lora_adapter.clone(),
+            random_val: rand::RngExt::random(rng),
+        }],
+    })?;
+    let out = result
+        .requests
+        .into_iter()
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("decode returned no result for {:?}", req.request_id))?;
+    Ok(emit_one(executor, req, out.token, out.logprob))
+}
+
+/// Verify the drafts (or fall back to a single greedy decode) and return the
+/// committed tokens. Always at least one token. Only called for spec-eligible
+/// (greedy, no-logprobs) requests, so the fallback's `logprobs: 0` /
+/// `random_val: 0.0` are correct.
 fn commit_tokens(
     executor: &mut impl ModelExecutor,
     req: &ActiveRequestState,
@@ -99,48 +151,60 @@ fn commit_tokens(
     }
 }
 
-/// Stream `committed`, updating request state and applying stop / max-token
-/// handling. Returns `true` when the request finished (caller retires it).
+/// Stream `committed` (no per-token logprobs — speculation is greedy), updating
+/// request state and applying stop / max-token handling. Returns `true` when
+/// the request finished (caller retires it).
 fn apply_committed(
     executor: &mut impl ModelExecutor,
     req: &mut ActiveRequestState,
     committed: &[u32],
 ) -> bool {
     for &token in committed {
-        let completion = req.generated_count + 1;
-        let is_eos = !req.params.ignore_eos && executor.is_stop_token(token);
-        if req
-            .token_tx
-            .send(TokenEvent::Token {
-                id: token,
-                logprob: None,
-            })
-            .is_err()
-        {
-            // Client hung up: drop without a Finished event.
-            let _ = executor.drop_request(req.request_id);
+        if emit_one(executor, req, token, None) {
             return true;
         }
-        req.generated_count = completion;
-        req.last_token = token;
-        req.token_history.push(token);
+    }
+    false
+}
 
-        let finish = if is_eos {
-            Some(FinishReason::Stop)
-        } else if completion >= req.max_tokens {
-            Some(FinishReason::Length)
-        } else {
-            None
-        };
-        if let Some(finish_reason) = finish {
-            let _ = req.token_tx.send(TokenEvent::Finished {
-                finish_reason,
-                prompt_tokens: req.prompt_len,
-                completion_tokens: req.generated_count,
-            });
-            let _ = executor.drop_request(req.request_id);
-            return true;
-        }
+/// Emit one token, update request state, and apply stop / max-token handling.
+/// Returns `true` when the request finished (caller retires it).
+fn emit_one(
+    executor: &mut impl ModelExecutor,
+    req: &mut ActiveRequestState,
+    token: u32,
+    logprob: Option<TokenLogprob>,
+) -> bool {
+    let completion = req.generated_count + 1;
+    let is_eos = !req.params.ignore_eos && executor.is_stop_token(token);
+    if req
+        .token_tx
+        .send(TokenEvent::Token { id: token, logprob })
+        .is_err()
+    {
+        // Client hung up: drop without a Finished event.
+        let _ = executor.drop_request(req.request_id);
+        return true;
+    }
+    req.generated_count = completion;
+    req.last_token = token;
+    req.token_history.push(token);
+
+    let finish = if is_eos {
+        Some(FinishReason::Stop)
+    } else if completion >= req.max_tokens {
+        Some(FinishReason::Length)
+    } else {
+        None
+    };
+    if let Some(finish_reason) = finish {
+        let _ = req.token_tx.send(TokenEvent::Finished {
+            finish_reason,
+            prompt_tokens: req.prompt_len,
+            completion_tokens: req.generated_count,
+        });
+        let _ = executor.drop_request(req.request_id);
+        return true;
     }
     false
 }

From 2842d15ea483f8acaf326ad9c2386ab1038cdd78 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 16:35:23 +0800
Subject: [PATCH 20/31] feat(qwen3): n-gram speculative decoding on the shared
 #436 core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Plug host-side n-gram (prompt-lookup) speculation into the method-agnostic
speculative core landed by #436, instead of #349's parallel stack. The shared
verify/accept/KV transaction (speculative.rs, executor/spec.rs, pool.rs) is
reused as-is; only the propose step and a no-capture verify forward are added.

- ngram.rs: keep the suffix-lookup proposer + unit tests; drop the obsolete
  SpeculativeProposer impl (the trait is gone — #436 kept the core concrete).
- executor: NgramProposer + per-request token context (ngram_ctx), maintained
  at prefill (prompt + first token), verify-commit, and decode; dropped on
  retire. execute_speculative_draft proposes host-side (no worker forward);
  execute_speculative_verify reuses the shared KV transaction with the DFlash
  readiness/capture asserts relaxed.
- executor lane: execute_ngram_verify runs batch_prefill_into with zero capture
  layers (no hidden-state capture), argmax, build_verify_results. Verify is
  routed by lane.dflash.is_some() (the methods are mutually exclusive).
- load_ngram_drafter + speculative_enabled/request_ready cover the n-gram method.
- CLI: --ngram-speculative (single-GPU greedy; mutually exclusive with DFlash,
  LoRA, KV offload, decode overlap), threaded through launch -> start_qwen3.
- Remove tests/ngram_speculative.rs (drove #349's deleted low-level
  execute_speculative API); the proposer is unit-tested in ngram.rs. GPU
  losslessness to be validated engine-level on the 5090.

Compiles green: cargo check -p openinfer-qwen3-4b {--lib,--tests} and
-p openinfer-server; ngram unit tests pass. GPU losslessness validation pending.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 openinfer-dynamo-backend/src/engine.rs        |   1 +
 openinfer-qwen3-4b/src/executor.rs            | 166 ++++++++-
 openinfer-qwen3-4b/src/executor/spec.rs       |  28 +-
 openinfer-qwen3-4b/src/lib.rs                 |  25 ++
 openinfer-qwen3-4b/src/ngram.rs               |   6 -
 openinfer-qwen3-4b/src/scheduler.rs           |   6 +
 openinfer-qwen3-4b/src/verify_graph.rs        |   5 +
 .../tests/batch_invariance_reject.rs          |   1 +
 .../tests/dflash_speculative_gate.rs          |   2 +
 .../tests/dflash_speculative_perf.rs          |   2 +
 openinfer-qwen3-4b/tests/ngram_speculative.rs | 339 ------------------
 .../tests/scheduler_robustness.rs             |   1 +
 openinfer-server/src/config.rs                |   7 +
 openinfer-server/src/main.rs                  |   9 +
 14 files changed, 243 insertions(+), 355 deletions(-)
 delete mode 100644 openinfer-qwen3-4b/tests/ngram_speculative.rs

diff --git a/openinfer-dynamo-backend/src/engine.rs b/openinfer-dynamo-backend/src/engine.rs
index 1e1f8123..6affcafc 100644
--- a/openinfer-dynamo-backend/src/engine.rs
+++ b/openinfer-dynamo-backend/src/engine.rs
@@ -153,6 +153,7 @@ impl OpeninferBackend {
             // Speculative decoding is a standalone-server knob; the Dynamo
             // worker never drafts.
             dflash_draft_model_path: None,
+            ngram_speculative: false,
             enable_kv_events,
         };
 
diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 4a1b75e8..c45a57b1 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -501,7 +501,11 @@ fn execute_step_on_lane(
             // each span position) and captures the target hidden states (at the
             // DFlash layers) to seed the next draft — all into reused,
             // pointer-stable scratch (`VerifyGraphBuffers`).
-            let result = lane.execute_dflash_verify(requests, kv_views)?;
+            let result = if lane.dflash.is_some() {
+                lane.execute_dflash_verify(requests, kv_views)?
+            } else {
+                lane.execute_ngram_verify(requests, kv_views)?
+            };
             Ok(WorkerStepOutcome::SpeculativeVerify(result))
         }
         StepCommand::SpeculativeDraft { requests } => Ok(WorkerStepOutcome::SpeculativeDraft(
@@ -833,6 +837,15 @@ pub struct Qwen3Executor {
     /// enters this set when its prompt finishes prefilling with captured target
     /// context, and leaves on retire or a plain (non-speculative) decode.
     dflash_ready_requests: HashSet<RequestId>,
+    /// N-gram (prompt-lookup) proposer; `Some` once host-side n-gram speculative
+    /// decoding is enabled. Mutually exclusive with [`Self::speculative`]
+    /// (DFlash): exactly one drafter is ever loaded.
+    ngram: Option<crate::ngram::NgramProposer>,
+    /// Per-request running token context (prompt + every committed token),
+    /// maintained only while [`Self::ngram`] is set. The n-gram proposer scans
+    /// this to find prompt-lookup matches; it always ends with the request's
+    /// current dangling token so the next draft continues from it.
+    ngram_ctx: HashMap<RequestId, Vec<u32>>,
     /// Opt-in KV block-event feed for a cache-aware router (`Some` only when the
     /// engine was built with events on — single-GPU, no LoRA). `None` on the
     /// plain path, where the whole feed costs nothing.
@@ -954,6 +967,8 @@ impl Qwen3Executor {
             async_prefill: None,
             speculative: None,
             dflash_ready_requests: HashSet::new(),
+            ngram: None,
+            ngram_ctx: HashMap::new(),
             kv_events,
         })
     }
@@ -1183,6 +1198,8 @@ impl Qwen3Executor {
             async_prefill: None,
             speculative: None,
             dflash_ready_requests: HashSet::new(),
+            ngram: None,
+            ngram_ctx: HashMap::new(),
             // KV events are single-GPU only (asserted above); never wired here.
             kv_events: None,
         })
@@ -1293,6 +1310,31 @@ impl Qwen3Executor {
         Ok(())
     }
 
+    /// Enable host-side n-gram (prompt-lookup) speculative decoding. Like DFlash
+    /// it is single-GPU greedy-only and forces the prefix cache off (the verify
+    /// forward writes each request's own speculative KV span); unlike DFlash it
+    /// loads no draft model — drafts come from scanning the request's own token
+    /// context on the executor thread.
+    pub fn load_ngram_drafter(&mut self, config: crate::ngram::NgramConfig) -> Result<()> {
+        anyhow::ensure!(
+            self.workers.is_empty(),
+            "speculative decoding requires the single-GPU path (got {} extra ranks)",
+            self.workers.len()
+        );
+        anyhow::ensure!(
+            self.offload.is_none(),
+            "speculative decoding is not supported together with KV offload"
+        );
+        anyhow::ensure!(
+            self.speculative.is_none(),
+            "n-gram speculative decoding cannot be combined with a DFlash draft model"
+        );
+        log::info!("Qwen3 n-gram speculative decoding enabled: {config}");
+        self.prefix_cache_enabled = false;
+        self.ngram = Some(crate::ngram::NgramProposer::new(config));
+        Ok(())
+    }
+
     /// Whether KV offload is active on this executor.
     pub fn offload_enabled(&self) -> bool {
         self.offload.is_some()
@@ -1662,6 +1704,7 @@ impl ModelExecutor for Qwen3Executor {
             }
         }
         self.saved_cursor.remove(&request_id);
+        self.ngram_ctx.remove(&request_id);
         if self.speculative.is_some() {
             self.dflash_ready_requests.remove(&request_id);
             self.primary.drop_dflash_request(request_id)?;
@@ -1849,6 +1892,26 @@ impl ModelExecutor for Qwen3Executor {
         for req_result in &result.requests {
             self.apply_prefill_result(req_result)?;
         }
+        // Seed the n-gram running context for each freshly-completed prompt:
+        // the full prompt followed by the first sampled token (the request's
+        // current dangling token). Later committed tokens are appended at verify
+        // and decode time so the context always ends with the dangling token.
+        if self.ngram.is_some() {
+            for req_result in &result.requests {
+                if req_result.completed {
+                    if let Some(req) = plan
+                        .requests
+                        .iter()
+                        .find(|r| r.request_id == req_result.request_id)
+                    {
+                        let mut context = Vec::with_capacity(req.prompt_tokens.len() + 1);
+                        context.extend_from_slice(&req.prompt_tokens);
+                        context.push(req_result.first_token);
+                        self.ngram_ctx.insert(req_result.request_id, context);
+                    }
+                }
+            }
+        }
         // A request becomes draft-ready once its prompt is fully prefilled with
         // captured target context. Partial chunks stay pending; ineligible
         // requests drop any stale worker state.
@@ -1923,6 +1986,15 @@ impl ModelExecutor for Qwen3Executor {
                 .expect("request must exist after decode");
             rkv.apply_decode(req_result.token, self.kv_mgr.pool())?;
         }
+        // A plain decode commits one token per request; keep the n-gram context
+        // in sync so it always ends with the request's dangling token.
+        if self.ngram.is_some() {
+            for req_result in &result.requests {
+                if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
+                    context.push(req_result.token);
+                }
+            }
+        }
         // A plain decode advances the sequence outside the speculative path, so
         // any captured draft context is now stale — drop it.
         if self.speculative.is_some() {
@@ -1941,6 +2013,31 @@ impl ModelExecutor for Qwen3Executor {
     }
 
     fn execute_speculative_draft(&mut self, plan: DraftPlan<'_>) -> Result<DraftResult> {
+        // N-gram drafts are produced host-side on the executor thread (no worker
+        // forward): scan each request's own token context for a prompt-lookup
+        // match. DFlash instead runs a draft-model forward on the lane.
+        if let Some(ngram) = self.ngram {
+            let mut requests = Vec::with_capacity(plan.requests.len());
+            for item in plan.requests {
+                anyhow::ensure!(
+                    item.params.is_greedy(),
+                    "n-gram speculative draft currently supports greedy sampling only"
+                );
+                let context = self.ngram_ctx.get(&item.request_id).ok_or_else(|| {
+                    anyhow::anyhow!("missing n-gram context for {:?}", item.request_id)
+                })?;
+                let drafts = ngram.propose(context);
+                // Verify span = [current dangling token, draft_1, …, draft_K].
+                let mut token_ids = Vec::with_capacity(1 + drafts.len());
+                token_ids.push(item.current_token);
+                token_ids.extend(drafts);
+                requests.push(crate::speculative::DraftRequestResult {
+                    request_id: item.request_id,
+                    token_ids,
+                });
+            }
+            return Ok(DraftResult { requests });
+        }
         self.execute_speculative_draft_impl(plan)
     }
 
@@ -1949,11 +2046,17 @@ impl ModelExecutor for Qwen3Executor {
     }
 
     fn speculative_enabled(&self) -> bool {
-        self.speculative.is_some()
+        self.speculative.is_some() || self.ngram.is_some()
     }
 
     fn speculative_request_ready(&self, request_id: RequestId) -> bool {
-        self.dflash_ready_requests.contains(&request_id)
+        if self.ngram.is_some() {
+            // Ready as soon as the prompt is prefilled and its context seeded;
+            // n-gram needs no captured hidden state.
+            self.ngram_ctx.contains_key(&request_id)
+        } else {
+            self.dflash_ready_requests.contains(&request_id)
+        }
     }
 
     fn execute_unified(&mut self, plan: UnifiedPlan<'_>) -> Result<UnifiedResult> {
@@ -2660,6 +2763,63 @@ impl LocalQwen3Lane {
         result
     }
 
+    /// Verify forward for host-side proposers (n-gram): one target forward over
+    /// each request's `K + 1` span with its speculative KV view, returning the
+    /// greedy argmax per position. Unlike [`Self::execute_dflash_verify`] it
+    /// captures no hidden states — a token-only proposer keeps no model state to
+    /// seed — so the verify graph runs with zero capture layers.
+    fn execute_ngram_verify(
+        &mut self,
+        requests: &[VerifyStepItem],
+        kv_views: &[KvView],
+    ) -> Result<VerifyResult> {
+        let span = requests
+            .iter()
+            .map(|req| req.as_slice().len())
+            .max()
+            .unwrap_or(1)
+            .max(1);
+        // (Re)allocate the verify scratch if it is missing or too narrow for this
+        // step's widest span. With a fixed n-gram K this allocates at most once.
+        let needs_alloc = self
+            .verify_bufs
+            .as_ref()
+            .map_or(true, |bufs| bufs.span() < span);
+        if needs_alloc {
+            let max_batch = *BATCH_BUCKETS.last().unwrap();
+            self.verify_bufs = Some(VerifyGraphBuffers::new(
+                &self.model,
+                max_batch,
+                span,
+                0,
+                self.total_blocks,
+            )?);
+        }
+
+        let mut bufs = self.verify_bufs.take().expect("verify buffers just set");
+        let result = (|| -> Result<VerifyResult> {
+            let spans: Vec<&[u32]> = requests.iter().map(VerifyStepItem::as_slice).collect();
+            self.model.batch_prefill_into(
+                &spans,
+                kv_views,
+                self.kv_buffer.buffer(),
+                &self.layout,
+                &[],
+                &mut bufs,
+            )?;
+            let total_tokens: usize = requests.iter().map(|req| req.as_slice().len()).sum();
+            let greedy = SamplingParams::default();
+            let params: Vec<&SamplingParams> = vec![&greedy; total_tokens];
+            let target_tokens = self.select_step_tokens(bufs.all_logits(), &params, 0)?;
+            let request_results = build_verify_results(requests, &target_tokens)?;
+            Ok(VerifyResult {
+                requests: request_results,
+            })
+        })();
+        self.verify_bufs = Some(bufs);
+        result
+    }
+
     fn execute_decode(
         &mut self,
         token_ids: &[u32],
diff --git a/openinfer-qwen3-4b/src/executor/spec.rs b/openinfer-qwen3-4b/src/executor/spec.rs
index 0ec42881..764cd879 100644
--- a/openinfer-qwen3-4b/src/executor/spec.rs
+++ b/openinfer-qwen3-4b/src/executor/spec.rs
@@ -17,8 +17,8 @@ impl Qwen3Executor {
         plan: VerifyPlan<'_>,
     ) -> Result<VerifyResult> {
         anyhow::ensure!(
-            self.speculative.is_some(),
-            "speculative verification requested but no draft model is loaded"
+            self.speculative.is_some() || self.ngram.is_some(),
+            "speculative verification requested but no speculative method is enabled"
         );
         for req in plan.requests {
             anyhow::ensure!(
@@ -30,11 +30,13 @@ impl Qwen3Executor {
                 req.params.is_greedy(),
                 "speculative verification currently supports greedy sampling only"
             );
-            anyhow::ensure!(
-                self.dflash_ready_requests.contains(&req.request_id),
-                "speculative verification requested before DFlash state is ready for {:?}",
-                req.request_id
-            );
+            if self.speculative.is_some() {
+                anyhow::ensure!(
+                    self.dflash_ready_requests.contains(&req.request_id),
+                    "speculative verification requested before DFlash state is ready for {:?}",
+                    req.request_id
+                );
+            }
             anyhow::ensure!(
                 self.request_kvs.contains_key(&req.request_id),
                 "missing RequestKv for {:?}",
@@ -134,6 +136,18 @@ impl Qwen3Executor {
             self.save_sealed_blocks(req_result.request_id);
         }
 
+        // Keep the n-gram running context in sync: append every committed token
+        // so the next draft scans the full history and continues from the new
+        // dangling token. DFlash keeps its own per-request hidden state instead.
+        if self.ngram.is_some() {
+            for req_result in &result.requests {
+                self.ngram_ctx
+                    .entry(req_result.request_id)
+                    .or_default()
+                    .extend_from_slice(&req_result.accepted_tokens);
+            }
+        }
+
         Ok(result)
     }
 
diff --git a/openinfer-qwen3-4b/src/lib.rs b/openinfer-qwen3-4b/src/lib.rs
index e7eacba8..56d6dfa3 100644
--- a/openinfer-qwen3-4b/src/lib.rs
+++ b/openinfer-qwen3-4b/src/lib.rs
@@ -10,6 +10,7 @@ mod executor;
 pub(crate) mod green_ctx;
 pub mod kernel_bench;
 mod lora;
+mod ngram;
 mod prefill;
 mod scheduler;
 mod speculative;
@@ -177,6 +178,10 @@ pub struct Qwen3LaunchOptions {
     /// `Some` enables DFlash speculative decoding with this drafter model.
     /// Single-GPU only and mutually exclusive with LoRA and KV offload.
     pub dflash_draft_model_path: Option<PathBuf>,
+    /// Enables host-side n-gram (prompt-lookup) speculative decoding. Needs no
+    /// draft model; single-GPU greedy only and mutually exclusive with DFlash,
+    /// LoRA, KV offload, and decode overlap.
+    pub ngram_speculative: bool,
     /// Publish KV block store/remove events for an out-of-band cache-aware
     /// router (e.g. a Dynamo KV router). Off for plain single-machine serving;
     /// single-GPU + base-model only (rejected with LoRA or tensor parallel).
@@ -226,6 +231,22 @@ pub fn launch(model_path: &Path, options: Qwen3LaunchOptions) -> Result<EngineHa
          (--decode-overlap): the speculative path never takes the unified overlap \
          route, so the overlap streams would only waste VRAM the drafter needs"
     );
+    anyhow::ensure!(
+        !(options.dflash_draft_model_path.is_some() && options.ngram_speculative),
+        "n-gram speculative decoding cannot be combined with a DFlash draft model"
+    );
+    anyhow::ensure!(
+        !(options.ngram_speculative && options.lora.is_some()),
+        "n-gram speculative decoding cannot be combined with LoRA serving"
+    );
+    anyhow::ensure!(
+        !(options.ngram_speculative && options.offload.enabled),
+        "n-gram speculative decoding cannot be combined with KV offload"
+    );
+    anyhow::ensure!(
+        !(options.ngram_speculative && !matches!(options.decode_overlap, DecodeOverlap::Off)),
+        "n-gram speculative decoding cannot be combined with decode overlap (--decode-overlap)"
+    );
     if options.enable_kv_events && options.lora.is_some() {
         anyhow::bail!("KV block events are not supported with LoRA serving");
     }
@@ -257,6 +278,7 @@ pub fn launch(model_path: &Path, options: Qwen3LaunchOptions) -> Result<EngineHa
             options.decode_overlap,
             options.batch_invariant,
             options.dflash_draft_model_path.as_deref(),
+            options.ngram_speculative,
             options.enable_kv_events,
         ),
     }
@@ -274,6 +296,7 @@ pub fn start_engine(model_path: &Path, options: EngineLoadOptions) -> Result<Eng
         false,
         None,
         false,
+        false,
     )
 }
 
@@ -300,6 +323,7 @@ pub fn start_engine_with_offload(
     decode_overlap: DecodeOverlap,
     batch_invariant: bool,
     dflash_draft_model_path: Option<&Path>,
+    ngram_speculative: bool,
     enable_kv_events: bool,
 ) -> Result<EngineHandle> {
     let EngineLoadOptions {
@@ -330,6 +354,7 @@ pub fn start_engine_with_offload(
         memory_options,
         decode_overlap,
         dflash_draft_model_path,
+        ngram_speculative,
         enable_kv_events,
     )
 }
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index 1aacfaea..16aa71ad 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -123,12 +123,6 @@ impl NgramProposer {
     }
 }
 
-impl crate::speculative::SpeculativeProposer for NgramProposer {
-    fn propose(&self, context: &[u32]) -> Vec<u32> {
-        NgramProposer::propose(self, context)
-    }
-}
-
 /// Find the latest start index `i < len - n` such that `context[i..i + n]`
 /// equals `suffix` (the trailing `n` tokens). "Latest" prefers the most recent
 /// context. Guaranteed to leave at least one following token at `i + n`.
diff --git a/openinfer-qwen3-4b/src/scheduler.rs b/openinfer-qwen3-4b/src/scheduler.rs
index fb679522..4183725f 100644
--- a/openinfer-qwen3-4b/src/scheduler.rs
+++ b/openinfer-qwen3-4b/src/scheduler.rs
@@ -150,6 +150,7 @@ pub(crate) fn start_qwen3(
     memory_options: Qwen3MemoryOptions,
     decode_overlap: crate::DecodeOverlap,
     dflash_draft_model_path: Option<&str>,
+    ngram_speculative: bool,
     enable_kv_events: bool,
 ) -> Result<EngineHandle> {
     let mut executor = Qwen3Executor::from_runtime_with_lora_options(
@@ -171,6 +172,11 @@ pub(crate) fn start_qwen3(
     // was already reserved during profiling from the draft path passed above.
     if let Some(draft_path) = dflash_draft_model_path {
         executor.load_dflash_draft_model(draft_path)?;
+    } else if ngram_speculative {
+        // Host-side n-gram drafter: no draft model to load, just enable the
+        // proposer and force the prefix cache off (same as DFlash) so verify
+        // writes each request's own speculative KV span.
+        executor.load_ngram_drafter(crate::ngram::NgramConfig::from_env())?;
     }
 
     Ok(start_with_executor(executor, seed, max_prefill_tokens))
diff --git a/openinfer-qwen3-4b/src/verify_graph.rs b/openinfer-qwen3-4b/src/verify_graph.rs
index 25c4e626..0e384477 100644
--- a/openinfer-qwen3-4b/src/verify_graph.rs
+++ b/openinfer-qwen3-4b/src/verify_graph.rs
@@ -157,6 +157,11 @@ impl VerifyGraphBuffers {
     pub(crate) fn captured_hidden(&self) -> &HiddenStates {
         &self.captured_hidden
     }
+
+    /// The fixed per-request span width these buffers were allocated for.
+    pub(crate) fn span(&self) -> usize {
+        self.span
+    }
 }
 
 impl Qwen3Model {
diff --git a/openinfer-qwen3-4b/tests/batch_invariance_reject.rs b/openinfer-qwen3-4b/tests/batch_invariance_reject.rs
index 01f04546..26ee842b 100644
--- a/openinfer-qwen3-4b/tests/batch_invariance_reject.rs
+++ b/openinfer-qwen3-4b/tests/batch_invariance_reject.rs
@@ -29,6 +29,7 @@ fn batch_invariant_rejects_decode_overlap() {
         true,
         None,
         false,
+        false,
     )
     .err()
     .expect("--batch-invariant + --decode-overlap must be rejected");
diff --git a/openinfer-qwen3-4b/tests/dflash_speculative_gate.rs b/openinfer-qwen3-4b/tests/dflash_speculative_gate.rs
index 2cf83760..b28d57fb 100644
--- a/openinfer-qwen3-4b/tests/dflash_speculative_gate.rs
+++ b/openinfer-qwen3-4b/tests/dflash_speculative_gate.rs
@@ -116,6 +116,8 @@ fn launch_options(draft: Option<PathBuf>) -> Qwen3LaunchOptions {
         decode_overlap: DecodeOverlap::Off,
         batch_invariant: false,
         dflash_draft_model_path: draft,
+        ngram_speculative: false,
+        enable_kv_events: false,
     }
 }
 
diff --git a/openinfer-qwen3-4b/tests/dflash_speculative_perf.rs b/openinfer-qwen3-4b/tests/dflash_speculative_perf.rs
index ff0e017a..fea3fb0b 100644
--- a/openinfer-qwen3-4b/tests/dflash_speculative_perf.rs
+++ b/openinfer-qwen3-4b/tests/dflash_speculative_perf.rs
@@ -66,6 +66,8 @@ fn launch_options(draft: Option<PathBuf>) -> Qwen3LaunchOptions {
         decode_overlap: DecodeOverlap::Off,
         batch_invariant: false,
         dflash_draft_model_path: draft,
+        ngram_speculative: false,
+        enable_kv_events: false,
     }
 }
 
diff --git a/openinfer-qwen3-4b/tests/ngram_speculative.rs b/openinfer-qwen3-4b/tests/ngram_speculative.rs
deleted file mode 100644
index 143e949e..00000000
--- a/openinfer-qwen3-4b/tests/ngram_speculative.rs
+++ /dev/null
@@ -1,339 +0,0 @@
-//! GPU validation for n-gram speculative decoding: greedy speculation is
-//! lossless, so the token sequence produced with the proposer must match plain
-//! greedy decode token-for-token.
-//!
-//! Methodology note: the verify forward runs the prefill attention kernel while
-//! the reference decode runs the decode attention kernel. The math is identical
-//! but bf16 reduction order can differ, so a near-tie position could in
-//! principle diverge; the repetitive prompt keeps the greedy argmax
-//! well-separated. Prefix caching is disabled so both runs are cold and
-//! identically shaped.
-//!
-//! Requires a CUDA GPU and Qwen3-4B weights; skips cleanly when absent
-//! (point `OPENINFER_TEST_MODEL_PATH` at the weights to run it).
-
-use std::path::Path;
-
-use openinfer_core::sampler::SamplingParams;
-use openinfer_qwen3_4b::ngram::{NgramConfig, NgramProposer};
-use openinfer_qwen3_4b::runtime::{
-    DecodePlan, DecodeStepItem, PrefillPlan, PrefillStepItem, Qwen3Executor, RequestId,
-    SpeculativeStepItem,
-};
-
-const MODEL_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../models/Qwen3-4B");
-const N_GENERATE: usize = 24;
-const MAX_OUTPUT: usize = N_GENERATE + 8;
-
-fn model_path_or_skip() -> Option<String> {
-    match std::env::var("OPENINFER_TEST_MODEL_PATH") {
-        Ok(path) => Some(path),
-        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => {
-            Some(MODEL_PATH.to_string())
-        }
-        Err(_) => {
-            eprintln!(
-                "skipping qwen3 ngram_speculative: {MODEL_PATH}/config.json missing; set OPENINFER_TEST_MODEL_PATH to run it"
-            );
-            None
-        }
-    }
-}
-
-/// Repetitive prompt so the n-gram proposer fires (its suffix recurs earlier).
-fn repetitive_prompt() -> Vec<u32> {
-    let unit: [u32; 8] = [1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007];
-    let mut p = Vec::new();
-    for _ in 0..8 {
-        p.extend_from_slice(&unit);
-    }
-    p
-}
-
-fn prefill_item(id: u64, prompt: &[u32]) -> PrefillStepItem {
-    prefill_item_max(id, prompt, MAX_OUTPUT)
-}
-
-fn prefill_item_max(id: u64, prompt: &[u32], max_output: usize) -> PrefillStepItem {
-    PrefillStepItem::new(
-        RequestId::new(id),
-        prompt.to_vec(),
-        max_output,
-        SamplingParams::default(),
-        0,
-        false,
-        0.0,
-    )
-}
-
-/// Plain greedy decode (spec-off): one model token per step.
-fn greedy_decode(ex: &mut Qwen3Executor, id: u64, prompt: &[u32]) -> Vec<u32> {
-    let pr = ex
-        .execute_prefill(PrefillPlan {
-            requests: &[prefill_item(id, prompt)],
-            echo: false,
-        })
-        .expect("prefill");
-    let mut last = pr.requests[0].first_token;
-    let mut out = vec![last];
-    while out.len() < N_GENERATE {
-        let dr = ex
-            .execute_decode(DecodePlan {
-                requests: &[DecodeStepItem::new(
-                    RequestId::new(id),
-                    last,
-                    SamplingParams::default(),
-                    0,
-                    0.0,
-                )],
-            })
-            .expect("decode");
-        last = dr.requests[0].token;
-        out.push(last);
-    }
-    out
-}
-
-/// Greedy n-gram speculative decode (spec-on).
-fn speculative_decode(ex: &mut Qwen3Executor, id: u64, prompt: &[u32]) -> Vec<u32> {
-    let proposer = NgramProposer::new(NgramConfig {
-        max_ngram: 3,
-        min_ngram: 1,
-        num_speculative: 4,
-    });
-    let pr = ex
-        .execute_prefill(PrefillPlan {
-            requests: &[prefill_item(id, prompt)],
-            echo: false,
-        })
-        .expect("prefill");
-    let first = pr.requests[0].first_token;
-    let mut history = prompt.to_vec();
-    history.push(first);
-    let mut out = vec![first];
-
-    while out.len() < N_GENERATE {
-        let last = *out.last().unwrap();
-        let drafts = proposer.propose(&history);
-        if drafts.is_empty() {
-            let dr = ex
-                .execute_decode(DecodePlan {
-                    requests: &[DecodeStepItem::new(
-                        RequestId::new(id),
-                        last,
-                        SamplingParams::default(),
-                        0,
-                        0.0,
-                    )],
-                })
-                .expect("decode");
-            let t = dr.requests[0].token;
-            history.push(t);
-            out.push(t);
-        } else {
-            let committed = ex
-                .execute_speculative(&SpeculativeStepItem::new(RequestId::new(id), last, drafts))
-                .expect("speculative");
-            for t in committed {
-                history.push(t);
-                out.push(t);
-            }
-        }
-    }
-    out.truncate(N_GENERATE);
-    out
-}
-
-#[test]
-fn ngram_speculative_is_lossless_vs_greedy() {
-    let Some(model_path) = model_path_or_skip() else {
-        return;
-    };
-    let mut ex =
-        Qwen3Executor::from_runtime(&model_path, false, &[0]).expect("build eager executor");
-    // Cold, identically-shaped runs so the comparison is exact.
-    ex.set_prefix_cache_enabled(false);
-
-    let prompt = repetitive_prompt();
-    let reference = greedy_decode(&mut ex, 1, &prompt);
-    let speculative = speculative_decode(&mut ex, 2, &prompt);
-
-    assert_eq!(
-        reference, speculative,
-        "greedy speculative decode must be token-identical to plain greedy decode"
-    );
-}
-
-/// Counters for the speculative run: how many forward passes (verify + fallback
-/// single decodes) it took and how many tokens those passes committed.
-struct SpecStats {
-    tokens: usize,
-    forwards: usize,
-    drafted_forwards: usize,
-    drafted_tokens: usize,
-}
-
-/// Greedy speculative decode that also tallies forward passes / acceptance.
-fn speculative_decode_timed(
-    ex: &mut Qwen3Executor,
-    id: u64,
-    prompt: &[u32],
-    n_generate: usize,
-) -> (Vec<u32>, SpecStats) {
-    let proposer = NgramProposer::new(NgramConfig {
-        max_ngram: 3,
-        min_ngram: 1,
-        num_speculative: 4,
-    });
-    let pr = ex
-        .execute_prefill(PrefillPlan {
-            requests: &[prefill_item_max(id, prompt, n_generate + 8)],
-            echo: false,
-        })
-        .expect("prefill");
-    let first = pr.requests[0].first_token;
-    let mut history = prompt.to_vec();
-    history.push(first);
-    let mut out = vec![first];
-    let mut stats = SpecStats {
-        tokens: 0,
-        forwards: 0,
-        drafted_forwards: 0,
-        drafted_tokens: 0,
-    };
-
-    while out.len() < n_generate {
-        let last = *out.last().unwrap();
-        let drafts = proposer.propose(&history);
-        if drafts.is_empty() {
-            let dr = ex
-                .execute_decode(DecodePlan {
-                    requests: &[DecodeStepItem::new(
-                        RequestId::new(id),
-                        last,
-                        SamplingParams::default(),
-                        0,
-                        0.0,
-                    )],
-                })
-                .expect("decode");
-            let t = dr.requests[0].token;
-            history.push(t);
-            out.push(t);
-            stats.forwards += 1;
-        } else {
-            let committed = ex
-                .execute_speculative(&SpeculativeStepItem::new(RequestId::new(id), last, drafts))
-                .expect("speculative");
-            stats.forwards += 1;
-            stats.drafted_forwards += 1;
-            stats.drafted_tokens += committed.len();
-            for t in committed {
-                history.push(t);
-                out.push(t);
-            }
-        }
-    }
-    out.truncate(n_generate);
-    stats.tokens = n_generate;
-    (out, stats)
-}
-
-/// Wall-clock + acceptance benchmark; ignored by default (needs a GPU + weights
-/// and is timing-sensitive). Run with:
-/// `cargo test -p openinfer-qwen3-4b --release --test ngram_speculative \
-///   ngram_speculative_speedup -- --ignored --nocapture`
-#[test]
-#[ignore]
-fn ngram_speculative_speedup() {
-    use std::time::Instant;
-
-    let Some(model_path) = model_path_or_skip() else {
-        return;
-    };
-    let n_generate = std::env::var("OPENINFER_BENCH_TOKENS")
-        .ok()
-        .and_then(|v| v.parse().ok())
-        .unwrap_or(192usize);
-
-    let mut ex =
-        Qwen3Executor::from_runtime(&model_path, false, &[0]).expect("build eager executor");
-    ex.set_prefix_cache_enabled(false);
-    let prompt = repetitive_prompt();
-
-    // Warm up CUDA/cuBLAS so the first run's lazy init doesn't skew timings.
-    let _ = greedy_decode(&mut ex, 100, &prompt);
-
-    let t0 = Instant::now();
-    let baseline = {
-        // Inline greedy decode for `n_generate` tokens (greedy_decode is fixed at
-        // N_GENERATE), counting decode forwards.
-        let pr = ex
-            .execute_prefill(PrefillPlan {
-                requests: &[prefill_item_max(1, &prompt, n_generate + 8)],
-                echo: false,
-            })
-            .expect("prefill");
-        let mut last = pr.requests[0].first_token;
-        let mut out = vec![last];
-        while out.len() < n_generate {
-            let dr = ex
-                .execute_decode(DecodePlan {
-                    requests: &[DecodeStepItem::new(
-                        RequestId::new(1),
-                        last,
-                        SamplingParams::default(),
-                        0,
-                        0.0,
-                    )],
-                })
-                .expect("decode");
-            last = dr.requests[0].token;
-            out.push(last);
-        }
-        out
-    };
-    let baseline_dt = t0.elapsed();
-    let baseline_forwards = baseline.len() - 1; // one decode forward per token after prefill
-
-    let t1 = Instant::now();
-    let (spec_out, stats) = speculative_decode_timed(&mut ex, 2, &prompt, n_generate);
-    let spec_dt = t1.elapsed();
-
-    assert_eq!(
-        baseline, spec_out,
-        "speculative output must stay token-identical to greedy"
-    );
-
-    let baseline_tpot = baseline_dt.as_secs_f64() / baseline_forwards as f64 * 1e3;
-    let spec_tpot = spec_dt.as_secs_f64() / (stats.tokens - 1) as f64 * 1e3;
-    let accept_per_verify = stats.drafted_tokens as f64 / stats.drafted_forwards.max(1) as f64;
-
-    eprintln!("=== n-gram speculative decode benchmark ===");
-    eprintln!("tokens generated   : {}", stats.tokens);
-    eprintln!(
-        "baseline (greedy)  : {:>7.1} ms, {} forwards, {:.3} ms/token",
-        baseline_dt.as_secs_f64() * 1e3,
-        baseline_forwards,
-        baseline_tpot
-    );
-    eprintln!(
-        "speculative        : {:>7.1} ms, {} forwards ({} verify), {:.3} ms/token",
-        spec_dt.as_secs_f64() * 1e3,
-        stats.forwards,
-        stats.drafted_forwards,
-        spec_tpot
-    );
-    eprintln!(
-        "accepted/verify    : {:.2} tokens (K=4, so up to 5 per verify)",
-        accept_per_verify
-    );
-    eprintln!(
-        "forward-pass saving: {:.1}% fewer model calls",
-        (1.0 - stats.forwards as f64 / baseline_forwards as f64) * 100.0
-    );
-    eprintln!(
-        "wall-clock speedup : {:.2}x",
-        baseline_dt.as_secs_f64() / spec_dt.as_secs_f64()
-    );
-}
diff --git a/openinfer-qwen3-4b/tests/scheduler_robustness.rs b/openinfer-qwen3-4b/tests/scheduler_robustness.rs
index fdaec356..cde5098c 100644
--- a/openinfer-qwen3-4b/tests/scheduler_robustness.rs
+++ b/openinfer-qwen3-4b/tests/scheduler_robustness.rs
@@ -106,6 +106,7 @@ fn scheduler_survives_consumer_drop() {
         true,
         None,
         false,
+        false,
     )
     .expect("failed to start engine");
     assert_eq!(
diff --git a/openinfer-server/src/config.rs b/openinfer-server/src/config.rs
index d13c06b4..26c803e9 100644
--- a/openinfer-server/src/config.rs
+++ b/openinfer-server/src/config.rs
@@ -95,6 +95,13 @@ pub(crate) struct Args {
     #[arg(long = "dflash-draft-model-path")]
     pub dflash_draft_model_path: Option<PathBuf>,
 
+    /// Enable Qwen3 host-side n-gram (prompt-lookup) speculative decoding. Needs
+    /// no draft model; single-GPU greedy only, and forces the prefix cache off.
+    /// Mutually exclusive with --dflash-draft-model-path, --enable-lora,
+    /// --kv-offload, and --decode-overlap.
+    #[arg(long = "ngram-speculative", default_value_t = false)]
+    pub ngram_speculative: bool,
+
     /// Cap on total prompt tokens forwarded in one Qwen3 scheduler step
     /// (chunked prefill). Prefill activation scratch scales with the step's
     /// prompt tokens, so this bounds peak VRAM under request bursts; prompts
diff --git a/openinfer-server/src/main.rs b/openinfer-server/src/main.rs
index b660e393..dc474e05 100644
--- a/openinfer-server/src/main.rs
+++ b/openinfer-server/src/main.rs
@@ -119,6 +119,10 @@ fn load_engine(args: &Args, model_type: ModelType) -> anyhow::Result<EngineHandl
         args.dflash_draft_model_path.is_none() || is_qwen3,
         "--dflash-draft-model-path is only supported for Qwen3 (got {model_type:?})"
     );
+    anyhow::ensure!(
+        !args.ngram_speculative || is_qwen3,
+        "--ngram-speculative is only supported for Qwen3 (got {model_type:?})"
+    );
     let handle = match model_type {
         #[cfg(feature = "deepseek-v4")]
         ModelType::DeepSeekV4 => openinfer_deepseek_v4::launch(
@@ -182,6 +186,10 @@ fn load_engine(args: &Args, model_type: ModelType) -> anyhow::Result<EngineHandl
                 }
                 None => None,
             };
+            anyhow::ensure!(
+                !args.ngram_speculative || args.tp_size == 1,
+                "--ngram-speculative currently requires --tp-size=1"
+            );
             openinfer_qwen3_4b::launch(
                 &args.model_path,
                 Qwen3LaunchOptions {
@@ -200,6 +208,7 @@ fn load_engine(args: &Args, model_type: ModelType) -> anyhow::Result<EngineHandl
                     decode_overlap: args.decode_overlap.resolve(args.decode_sm_pct),
                     batch_invariant: args.batch_invariant,
                     dflash_draft_model_path,
+                    ngram_speculative: args.ngram_speculative,
                     // KV block events are a Dynamo-backend concern; the plain
                     // server never publishes them.
                     enable_kv_events: false,

From 5fb92a291ce0298972aa124b98da3107f7dd4760 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 17:24:31 +0800
Subject: [PATCH 21/31] test(qwen3): engine-level n-gram speculative
 losslessness gate

Greedy n-gram speculation must be lossless: spec-on tokens equal plain greedy
decode (modulo the benign prefill-vs-decode bf16 tie flip). Mirrors the DFlash
gate's regret check (re-prefill the shared context at a divergence, tolerate
only picks within MARGIN_TOL of the prefill-kernel argmax), driven through the
engine with --ngram-speculative (no draft model). Repetitive/structured prompts
make the proposer fire so the verify path is actually exercised.

- ngram_speculative_greedy_matches_plain_greedy: bs=1 over 5 prompts.
- ngram_concurrent_heterogeneous_is_lossless: 4 concurrent staggered budgets,
  exercising the bs>1 verify-span path and per-request context isolation.

GPU + Qwen3-4B weights required; skips cleanly without OPENINFER_TEST_MODEL_PATH.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../tests/ngram_speculative_gate.rs           | 475 ++++++++++++++++++
 1 file changed, 475 insertions(+)
 create mode 100644 openinfer-qwen3-4b/tests/ngram_speculative_gate.rs

diff --git a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
new file mode 100644
index 00000000..e5dabcd3
--- /dev/null
+++ b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
@@ -0,0 +1,475 @@
+//! N-gram (prompt-lookup) speculative-decoding losslessness gate.
+//!
+//! Greedy speculative decoding must be *lossless*: every draft is verified by a
+//! target forward and only the matching-argmax prefix (plus one bonus) is
+//! committed, so the accepted tokens are the target model's own greedy
+//! continuation. The catch is pure numerics — the verify path runs the
+//! *prefill* attention kernel over the `K + 1` span while a plain decode runs
+//! the *decode* kernel, and the two differ by ~1 bf16 ULP. On a near-tie that
+//! flips one argmax, and from there two greedy runs fan out completely.
+//!
+//! So an exact `spec == baseline` token match is the wrong gate: it false-fails
+//! on a benign tie flip. We use the same *regret* test as the DFlash gate (and
+//! `hf_golden_gate`): at the first position the two sequences disagree (where
+//! they still share an identical context, so the comparison is valid) we ask how
+//! far below the argmax the speculative pick sits — measured *in the prefill
+//! kernel's own distribution*, because that is the kernel the verify path runs.
+//! A re-prefill of the shared context (`prefill_next`) gives that reference.
+//! Within `MARGIN_TOL` of the prefill argmax ⇒ a benign numerical tie. Clearly
+//! worse (or outside the prefill top-K) ⇒ the verify/accept logic chose a token
+//! the forward never favored — a real bug.
+//!
+//! Unlike the DFlash gate this needs no draft model: n-gram drafts come from
+//! scanning each request's own token context, so the only knob is
+//! `--ngram-speculative`. Prompts are repetitive/structured so the proposer
+//! actually fires (its suffix recurs earlier) and the verify path is exercised;
+//! losslessness holds regardless, but a no-op proposer would make the gate
+//! vacuous.
+//!
+//! Requires a CUDA GPU and Qwen3-4B weights; skips cleanly when absent (set
+//! `OPENINFER_TEST_MODEL_PATH` to the weights to run it).
+
+use std::path::Path;
+use std::time::Duration;
+
+use openinfer_core::engine::{EngineHandle, GenerateRequest, TokenEvent, TokenSink};
+use openinfer_core::sampler::SamplingParams;
+use openinfer_qwen3_4b::{
+    DEFAULT_KV_CACHE_MEMORY_MARGIN_BYTES, DEFAULT_MAX_PREFILL_TOKENS, DecodeOverlap,
+    Qwen3LaunchOptions, Qwen3MemoryOptions, Qwen3OffloadOptions,
+};
+use vllm_text::tokenizer::DynTokenizer;
+
+mod common;
+
+const MODEL_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../models/Qwen3-4B");
+const GENERATED_TOKENS: usize = 64;
+/// Top-K logprobs requested from the baseline; wide enough that the speculative
+/// pick is in the set on any real tie (a pick outside the top-K is itself a red
+/// flag the gate should catch).
+const LOGPROBS: usize = 20;
+/// Max acceptable regret: how far below the baseline's argmax (in the baseline's
+/// own logprobs) the speculative pick may sit at the divergence point. ~3 bf16
+/// ULP at typical logit magnitudes — mirrors `hf_golden_gate`'s `MARGIN_TOL`.
+const MARGIN_TOL: f32 = 0.20;
+
+/// Both tests launch a Qwen3-4B engine, and two at once overflow a 16 GB card.
+/// Cargo runs tests in one binary concurrently, so serialize the engine-holding
+/// bodies — only one engine is ever resident on the GPU.
+static GPU: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+fn target_path_or_skip() -> Option<String> {
+    match std::env::var("OPENINFER_TEST_MODEL_PATH") {
+        Ok(path) => Some(path),
+        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => Some(MODEL_PATH.to_string()),
+        Err(_) => {
+            eprintln!(
+                "skipping ngram gate: {MODEL_PATH}/config.json missing; set OPENINFER_TEST_MODEL_PATH"
+            );
+            None
+        }
+    }
+}
+
+/// `ngram = true` switches on host-side n-gram speculation (no draft model). The
+/// baseline (`false`) still forces the prefix cache off so both runs take the
+/// same cold prefill path.
+fn launch_options(ngram: bool) -> Qwen3LaunchOptions {
+    Qwen3LaunchOptions {
+        device_ordinal: 0,
+        tp_size: 1,
+        cuda_graph: true,
+        offload: Qwen3OffloadOptions::disabled(),
+        no_prefix_cache: true,
+        max_prefill_tokens: DEFAULT_MAX_PREFILL_TOKENS,
+        memory: Qwen3MemoryOptions::new(0.85, DEFAULT_KV_CACHE_MEMORY_MARGIN_BYTES)
+            .validate()
+            .expect("valid memory options"),
+        lora: None,
+        decode_overlap: DecodeOverlap::Off,
+        batch_invariant: false,
+        dflash_draft_model_path: None,
+        ngram_speculative: ngram,
+        enable_kv_events: false,
+    }
+}
+
+/// One decoded position: the chosen token and (when requested) the top-K
+/// `(token, logprob)` distribution that produced it.
+struct Step {
+    id: u32,
+    top_logprobs: Vec<(u32, f32)>,
+}
+
+/// Submit one greedy request and collect the decoded steps until `Finished`.
+fn generate(
+    handle: &EngineHandle,
+    prompt_tokens: Vec<u32>,
+    logprobs: usize,
+    max_tokens: usize,
+) -> Vec<Step> {
+    let (token_tx, mut rx) = TokenSink::standalone();
+    handle
+        .submit(GenerateRequest {
+            request_id: None,
+            queued_at_unix_s: None,
+            prompt_tokens,
+            params: SamplingParams::default(),
+            max_tokens,
+            lora_adapter: None,
+            token_tx,
+            logprobs,
+            echo: false,
+        })
+        .expect("submit failed");
+
+    let mut steps = Vec::new();
+    loop {
+        match rx.blocking_recv().map(|(_, event)| event) {
+            Some(TokenEvent::Token { id, logprob }) => steps.push(Step {
+                id,
+                top_logprobs: logprob.map(|lp| lp.top_logprobs).unwrap_or_default(),
+            }),
+            Some(TokenEvent::Scheduled { .. } | TokenEvent::PromptTokens { .. }) => {}
+            Some(TokenEvent::Finished { .. }) => return steps,
+            Some(TokenEvent::Error { message, .. }) => panic!("generation failed: {message}"),
+            Some(TokenEvent::Rejected { message, .. }) => panic!("generation rejected: {message}"),
+            None => panic!("scheduler channel closed without Finished"),
+        }
+    }
+}
+
+/// Submit several greedy requests at once, then collect each one's steps. They
+/// run concurrently in the one engine — the scheduler batches them — so with
+/// heterogeneous `max_tokens` the verify spans differ across a batch, exercising
+/// the real bs>1 verify path. Each tuple is `(prompt_tokens, max_tokens)`;
+/// logprobs are off so the speculative path stays active. Returns one step list
+/// per request, in submission order.
+fn generate_concurrent(handle: &EngineHandle, requests: Vec<(Vec<u32>, usize)>) -> Vec<Vec<Step>> {
+    let receivers: Vec<_> = requests
+        .into_iter()
+        .map(|(prompt_tokens, max_tokens)| {
+            let (token_tx, rx) = TokenSink::standalone();
+            handle
+                .submit(GenerateRequest {
+                    request_id: None,
+                    queued_at_unix_s: None,
+                    prompt_tokens,
+                    params: SamplingParams::default(),
+                    max_tokens,
+                    lora_adapter: None,
+                    token_tx,
+                    logprobs: 0,
+                    echo: false,
+                })
+                .expect("submit failed");
+            rx
+        })
+        .collect();
+
+    receivers
+        .into_iter()
+        .map(|mut rx| {
+            let mut steps = Vec::new();
+            loop {
+                match rx.blocking_recv().map(|(_, event)| event) {
+                    Some(TokenEvent::Token { id, logprob }) => steps.push(Step {
+                        id,
+                        top_logprobs: logprob.map(|lp| lp.top_logprobs).unwrap_or_default(),
+                    }),
+                    Some(TokenEvent::Scheduled { .. } | TokenEvent::PromptTokens { .. }) => {}
+                    Some(TokenEvent::Finished { .. }) => break,
+                    Some(TokenEvent::Error { message, .. }) => {
+                        panic!("generation failed: {message}")
+                    }
+                    Some(TokenEvent::Rejected { message, .. }) => {
+                        panic!("generation rejected: {message}")
+                    }
+                    None => panic!("scheduler channel closed without Finished"),
+                }
+            }
+            steps
+        })
+        .collect()
+}
+
+/// Prefill `context` (echo) and return the next-token distribution the *prefill*
+/// kernel produces — the kernel the speculative verify path also uses. This is
+/// the reference the spec pick should match (vs the plain-decode baseline, whose
+/// kernel resolves bifurcation ties to the other side).
+fn prefill_next(handle: &EngineHandle, context: Vec<u32>, logprobs: usize) -> Step {
+    let (token_tx, mut rx) = TokenSink::standalone();
+    handle
+        .submit(GenerateRequest {
+            request_id: None,
+            queued_at_unix_s: None,
+            prompt_tokens: context,
+            params: SamplingParams::default(),
+            max_tokens: 1,
+            lora_adapter: None,
+            token_tx,
+            logprobs,
+            echo: true,
+        })
+        .expect("submit failed");
+
+    loop {
+        match rx.blocking_recv().map(|(_, event)| event) {
+            Some(TokenEvent::Token { id, logprob }) => {
+                return Step {
+                    id,
+                    top_logprobs: logprob.map(|lp| lp.top_logprobs).unwrap_or_default(),
+                };
+            }
+            Some(TokenEvent::Scheduled { .. } | TokenEvent::PromptTokens { .. }) => {}
+            Some(TokenEvent::Finished { .. }) => panic!("echo prefill finished without a token"),
+            Some(TokenEvent::Error { message, .. }) => panic!("prefill failed: {message}"),
+            Some(TokenEvent::Rejected { message, .. }) => panic!("prefill rejected: {message}"),
+            None => panic!("scheduler channel closed without a token"),
+        }
+    }
+}
+
+/// Compare one prompt's speculative `spec` steps against its plain-greedy `base`,
+/// tolerating only the benign prefill-vs-decode kernel-gap tie flip (the spec
+/// pick sits within `MARGIN_TOL` of the prefill kernel's own argmax, measured in
+/// the prefill distribution the verify path actually runs). `Ok(())` ⇒ lossless
+/// or a benign tie; `Err(diagnostic)` ⇒ a real spec bug. `handle` must be the
+/// live speculative engine — at a divergence it re-prefills the shared context
+/// (`prompt_tokens` + the matched prefix) to read that prefill-kernel reference.
+fn check_lossless(
+    handle: &EngineHandle,
+    tokenizer: &DynTokenizer,
+    i: usize,
+    prompt: &str,
+    prompt_tokens: &[u32],
+    base: &[Step],
+    spec: &[Step],
+) -> Result<(), String> {
+    let matched = base
+        .iter()
+        .zip(spec)
+        .take_while(|(b, s)| b.id == s.id)
+        .count();
+
+    if matched == base.len().min(spec.len()) {
+        eprintln!(
+            "prompt {i} ({prompt:?}): {matched}/{} tokens identical (100% lossless)",
+            base.len()
+        );
+        return Ok(());
+    }
+
+    let spec_id = spec[matched].id;
+    let decode_argmax = base[matched].top_logprobs[0].0;
+
+    {
+        let lo = matched.saturating_sub(2);
+        let hi = (matched + 3).min(base.len()).min(spec.len());
+        let base_ids: Vec<u32> = base[..hi].iter().map(|s| s.id).collect();
+        eprintln!("  [diag] prompt {i} matched={matched}");
+        eprintln!(
+            "  [diag] context+gen base ids {:?} = {:?}",
+            &base_ids,
+            tokenizer.decode(&base_ids, false).unwrap_or_default()
+        );
+        eprintln!(
+            "  [diag] base[{lo}..{hi}] = {:?}",
+            base[lo..hi]
+                .iter()
+                .map(|s| (s.id, tokenizer.decode(&[s.id], false).unwrap_or_default()))
+                .collect::<Vec<_>>()
+        );
+        eprintln!(
+            "  [diag] spec[{lo}..{hi}] = {:?}",
+            spec[lo..hi]
+                .iter()
+                .map(|s| (s.id, tokenizer.decode(&[s.id], false).unwrap_or_default()))
+                .collect::<Vec<_>>()
+        );
+    }
+
+    let mut context = prompt_tokens.to_vec();
+    context.extend(base[..matched].iter().map(|s| s.id));
+    let prefill_ref = prefill_next(handle, context, LOGPROBS);
+
+    if prefill_ref.id == spec_id {
+        let decode_lp = base[matched]
+            .top_logprobs
+            .iter()
+            .find(|(t, _)| *t == spec_id)
+            .map(|(_, lp)| base[matched].top_logprobs[0].1 - lp);
+        eprintln!(
+            "prompt {i} ({prompt:?}): kernel-gap flip at token {matched} — verify(prefill)→{spec_id}, \
+             decode→{decode_argmax}; spec matches prefill greedy (decode-margin {:?}). Not a spec bug.",
+            decode_lp
+        );
+        return Ok(());
+    }
+
+    let prefill_regret = prefill_ref
+        .top_logprobs
+        .iter()
+        .find(|(t, _)| *t == spec_id)
+        .map(|(_, lp)| prefill_ref.top_logprobs[0].1 - lp);
+
+    if let Some(regret) = prefill_regret {
+        if regret <= MARGIN_TOL {
+            eprintln!(
+                "prompt {i} ({prompt:?}): tie flip at token {matched} — \
+                 verify(prefill)→{}, spec→{spec_id}, decode→{decode_argmax}; \
+                 spec pick is #2 in the prefill distribution (regret {regret:.3} ≤ {MARGIN_TOL}). \
+                 Not a spec bug.",
+                prefill_ref.id,
+            );
+            return Ok(());
+        }
+    }
+
+    let decode_regret = base[matched]
+        .top_logprobs
+        .iter()
+        .find(|(t, _)| *t == spec_id)
+        .map(|(_, lp)| base[matched].top_logprobs[0].1 - lp);
+    Err(format!(
+        "prompt {i}: at token {matched} spec chose {spec_id} but prefill greedy says {} and \
+         decode greedy says {decode_argmax} (spec regret in prefill dist: {prefill_regret:?} > \
+         {MARGIN_TOL}; in decode dist: {decode_regret:?}) — real spec bug",
+        prefill_ref.id,
+    ))
+}
+
+/// bs=1 losslessness: each prompt's greedy n-gram speculative decode must match
+/// its plain-greedy baseline token-for-token (modulo the benign bf16 tie flip).
+#[test]
+fn ngram_speculative_greedy_matches_plain_greedy() {
+    let Some(model_path) = target_path_or_skip() else {
+        return;
+    };
+    let _gpu = GPU.lock().unwrap_or_else(|p| p.into_inner());
+
+    // Repetitive / structured prompts so the n-gram proposer fires often (its
+    // suffix recurs earlier in the context) and the verify path is exercised.
+    let prompts = [
+        "1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4",
+        "def fibonacci(n):\n    if n < 2:\n        return n\n    return fibonacci(n - 1) + fibonacci(",
+        "The quick brown fox. The quick brown fox. The quick brown fox.",
+        "Q: What is 17 multiplied by 23? A: Let's think step by step.",
+        "{\"a\": 1, \"b\": 2, \"a\": 1, \"b\": 2, \"a\": 1, \"b\": 2,",
+    ];
+
+    let tokenizer = common::load_tokenizer(&model_path);
+    let encoded: Vec<Vec<u32>> = prompts
+        .iter()
+        .map(|p| tokenizer.encode(p, false).expect("encode failed"))
+        .collect();
+
+    // 1. Baseline: plain greedy decode (speculative off), with logprobs so the
+    //    regret check has the reference distribution at the divergence point.
+    let baseline: Vec<Vec<Step>> = {
+        let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(false))
+            .expect("failed to start baseline engine");
+        let out = encoded
+            .iter()
+            .map(|t| generate(&handle, t.clone(), LOGPROBS, GENERATED_TOKENS))
+            .collect();
+        drop(handle);
+        // Free the target before the speculative engine loads the same weights.
+        std::thread::sleep(Duration::from_secs(2));
+        out
+    };
+
+    // 2. Speculative: n-gram propose + verify (logprobs off ⇒ spec path active).
+    //    Keep the engine alive through analysis: at a divergence it re-prefills
+    //    the shared context to read the prefill-kernel reference.
+    let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(true))
+        .expect("failed to start speculative engine");
+
+    let mut failures = Vec::new();
+    for (i, &prompt) in prompts.iter().enumerate() {
+        let spec = generate(&handle, encoded[i].clone(), 0, GENERATED_TOKENS);
+        if let Err(failure) =
+            check_lossless(&handle, &tokenizer, i, prompt, &encoded[i], &baseline[i], &spec)
+        {
+            failures.push(failure);
+        }
+    }
+
+    drop(handle);
+
+    assert!(
+        failures.is_empty(),
+        "n-gram speculative greedy decode is not lossless:\n{}",
+        failures.join("\n")
+    );
+}
+
+/// Concurrent, heterogeneous-`max_tokens` losslessness for the bs>1 verify path:
+/// several greedy requests at staggered budgets run in one engine so the in-flight
+/// batch mixes full and near-budget (truncated) verify spans. Each must stay
+/// lossless vs its own plain-greedy baseline. A per-request context mix-up (the
+/// proposer scanning the wrong request's history) or a bs>1 verify-span indexing
+/// bug would surface here as a real (non-tie) divergence.
+#[test]
+fn ngram_concurrent_heterogeneous_is_lossless() {
+    let Some(model_path) = target_path_or_skip() else {
+        return;
+    };
+    let _gpu = GPU.lock().unwrap_or_else(|p| p.into_inner());
+
+    let cases: [(&str, usize); 4] = [
+        ("def fibonacci(n):\n    if n < 2:\n        return n\n    return fibonacci(", 64),
+        ("1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4", 24),
+        ("Q: What is 17 multiplied by 23? A: Let's think step by step.", 48),
+        ("The quick brown fox. The quick brown fox. The quick brown fox.", 40),
+    ];
+
+    let tokenizer = common::load_tokenizer(&model_path);
+    let encoded: Vec<Vec<u32>> = cases
+        .iter()
+        .map(|(p, _)| tokenizer.encode(p, false).expect("encode failed"))
+        .collect();
+
+    // 1. Baselines: each prompt's plain-greedy decode (spec off) at ITS budget,
+    //    with logprobs for the regret reference. Sequential, one engine.
+    let baselines: Vec<Vec<Step>> = {
+        let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(false))
+            .expect("failed to start baseline engine");
+        let out = encoded
+            .iter()
+            .zip(&cases)
+            .map(|(t, (_, max_tokens))| generate(&handle, t.clone(), LOGPROBS, *max_tokens))
+            .collect();
+        drop(handle);
+        std::thread::sleep(Duration::from_secs(2));
+        out
+    };
+
+    // 2. Speculative engine: submit all four at once so they form real batches.
+    let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(true))
+        .expect("failed to start speculative engine");
+    let specs = generate_concurrent(
+        &handle,
+        encoded
+            .iter()
+            .zip(&cases)
+            .map(|(t, (_, max_tokens))| (t.clone(), *max_tokens))
+            .collect(),
+    );
+
+    let mut failures = Vec::new();
+    for (i, (prompt, _)) in cases.iter().enumerate() {
+        if let Err(failure) =
+            check_lossless(&handle, &tokenizer, i, prompt, &encoded[i], &baselines[i], &specs[i])
+        {
+            failures.push(failure);
+        }
+    }
+    drop(handle);
+
+    assert!(
+        failures.is_empty(),
+        "concurrent heterogeneous n-gram speculative decode is not lossless:\n{}",
+        failures.join("\n")
+    );
+}

From 8cf77c2cb5732a4128ecc1663626c825a83e4230 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 17:39:18 +0800
Subject: [PATCH 22/31] refactor(qwen3): unify speculative method state into
 one enum
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Collapse the two mutually-exclusive executor fields (speculative: Option<DFlashMeta>
and ngram: Option<NgramProposer>) into a single speculative_method:
Option<SpeculativeMethod> { Dflash(..) | Ngram(..) }. One source of truth for
'is speculation on and which kind', removing the 'assert not both set' smell.
dflash_meta()/ngram_proposer() accessors keep every call site a one-liner and
DFlash behaviour bit-identical.

This is the natural pre-trait consolidation now that n-gram is the second method
#436 was waiting for. A full SpeculativeProposer trait object is deliberately
not introduced yet: DFlash's propose is a worker RPC (needs &mut executor +
lane), so it can't be a self-contained proposer object without borrow gymnastics
— the enum-dispatch is the honest shape until a third method (EAGLE, #325) makes
a shared trait pay off.

cargo check -p openinfer-qwen3-4b {--lib,--tests} green; ngram units pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/executor.rs      | 80 ++++++++++++++++---------
 openinfer-qwen3-4b/src/executor/spec.rs |  8 +--
 2 files changed, 57 insertions(+), 31 deletions(-)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index c45a57b1..723e47d8 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -797,6 +797,17 @@ struct Qwen3ExecutorMetadata {
     config: Config,
 }
 
+/// The loaded speculative-decode method. Exactly one is ever active; the
+/// *propose* step is the only method-specific part (DFlash runs a draft-model
+/// forward on the worker; n-gram scans the request's token context host-side),
+/// while verify, greedy acceptance, and the KV transaction are shared.
+enum SpeculativeMethod {
+    /// DFlash draft model: worker-side draft forward with hidden-state capture.
+    Dflash(DFlashMeta),
+    /// Host-side n-gram / prompt-lookup proposer; needs no draft model.
+    Ngram(crate::ngram::NgramProposer),
+}
+
 pub struct Qwen3Executor {
     metadata: Qwen3ExecutorMetadata,
     kv_mgr: KvCacheManager,
@@ -830,21 +841,20 @@ pub struct Qwen3Executor {
     /// In-flight async prefill state. Populated by the SplitConcurrent step,
     /// consumed by `poll_async_prefill`.
     async_prefill: Option<AsyncPrefillState>,
-    /// DFlash draft metadata; `Some` once a draft model is loaded into the
-    /// primary lane. Speculative decoding is enabled iff this is set.
-    speculative: Option<DFlashMeta>,
+    /// The active speculative-decode method and its loaded state — DFlash draft
+    /// metadata or the n-gram proposer — or `None` when speculation is off.
+    /// Exactly one method is ever loaded; this is the single source of truth for
+    /// whether speculation is on and which kind. Only the *propose* step varies
+    /// between methods; verify, accept, and the KV transaction are shared.
+    speculative_method: Option<SpeculativeMethod>,
     /// Requests whose DFlash context is captured and ready to draft. A request
     /// enters this set when its prompt finishes prefilling with captured target
     /// context, and leaves on retire or a plain (non-speculative) decode.
     dflash_ready_requests: HashSet<RequestId>,
-    /// N-gram (prompt-lookup) proposer; `Some` once host-side n-gram speculative
-    /// decoding is enabled. Mutually exclusive with [`Self::speculative`]
-    /// (DFlash): exactly one drafter is ever loaded.
-    ngram: Option<crate::ngram::NgramProposer>,
     /// Per-request running token context (prompt + every committed token),
-    /// maintained only while [`Self::ngram`] is set. The n-gram proposer scans
-    /// this to find prompt-lookup matches; it always ends with the request's
-    /// current dangling token so the next draft continues from it.
+    /// maintained only under the n-gram method. The proposer scans this for
+    /// prompt-lookup matches; it always ends with the request's current dangling
+    /// token so the next draft continues from it.
     ngram_ctx: HashMap<RequestId, Vec<u32>>,
     /// Opt-in KV block-event feed for a cache-aware router (`Some` only when the
     /// engine was built with events on — single-GPU, no LoRA). `None` on the
@@ -965,9 +975,8 @@ impl Qwen3Executor {
             l1_retention_disabled: false,
             overlap: None,
             async_prefill: None,
-            speculative: None,
+            speculative_method: None,
             dflash_ready_requests: HashSet::new(),
-            ngram: None,
             ngram_ctx: HashMap::new(),
             kv_events,
         })
@@ -1196,9 +1205,8 @@ impl Qwen3Executor {
             l1_retention_disabled: false,
             overlap: None,
             async_prefill: None,
-            speculative: None,
+            speculative_method: None,
             dflash_ready_requests: HashSet::new(),
-            ngram: None,
             ngram_ctx: HashMap::new(),
             // KV events are single-GPU only (asserted above); never wired here.
             kv_events: None,
@@ -1290,6 +1298,22 @@ impl Qwen3Executor {
     /// is incompatible with KV offload. Disables the prefix cache: speculative
     /// capture needs clean, uncached target hidden states for every prompt
     /// token, and a prefix-cache hit skips the forward that would produce them.
+    /// The loaded DFlash draft metadata, or `None` under no/other method.
+    fn dflash_meta(&self) -> Option<&DFlashMeta> {
+        match &self.speculative_method {
+            Some(SpeculativeMethod::Dflash(meta)) => Some(meta),
+            _ => None,
+        }
+    }
+
+    /// The loaded n-gram proposer, or `None` under no/other method.
+    fn ngram_proposer(&self) -> Option<crate::ngram::NgramProposer> {
+        match &self.speculative_method {
+            Some(SpeculativeMethod::Ngram(proposer)) => Some(*proposer),
+            _ => None,
+        }
+    }
+
     pub fn load_dflash_draft_model(&mut self, draft_path: &str) -> Result<()> {
         anyhow::ensure!(
             self.workers.is_empty(),
@@ -1306,7 +1330,7 @@ impl Qwen3Executor {
             meta.block_size
         );
         self.prefix_cache_enabled = false;
-        self.speculative = Some(meta);
+        self.speculative_method = Some(SpeculativeMethod::Dflash(meta));
         Ok(())
     }
 
@@ -1326,12 +1350,14 @@ impl Qwen3Executor {
             "speculative decoding is not supported together with KV offload"
         );
         anyhow::ensure!(
-            self.speculative.is_none(),
+            self.speculative_method.is_none(),
             "n-gram speculative decoding cannot be combined with a DFlash draft model"
         );
         log::info!("Qwen3 n-gram speculative decoding enabled: {config}");
         self.prefix_cache_enabled = false;
-        self.ngram = Some(crate::ngram::NgramProposer::new(config));
+        self.speculative_method = Some(SpeculativeMethod::Ngram(crate::ngram::NgramProposer::new(
+            config,
+        )));
         Ok(())
     }
 
@@ -1649,7 +1675,7 @@ impl ModelExecutor for Qwen3Executor {
 
     fn max_context_tokens(&self) -> usize {
         let target = self.metadata.config.max_position_embeddings;
-        match &self.speculative {
+        match self.dflash_meta() {
             // The draft's fixed-width in-fill block writes `block_size` positions
             // past the committed length each step, so a request may use at most
             // `draft.max_pos - block_size` tokens before the draft cache would
@@ -1705,7 +1731,7 @@ impl ModelExecutor for Qwen3Executor {
         }
         self.saved_cursor.remove(&request_id);
         self.ngram_ctx.remove(&request_id);
-        if self.speculative.is_some() {
+        if self.dflash_meta().is_some() {
             self.dflash_ready_requests.remove(&request_id);
             self.primary.drop_dflash_request(request_id)?;
         }
@@ -1896,7 +1922,7 @@ impl ModelExecutor for Qwen3Executor {
         // the full prompt followed by the first sampled token (the request's
         // current dangling token). Later committed tokens are appended at verify
         // and decode time so the context always ends with the dangling token.
-        if self.ngram.is_some() {
+        if self.ngram_proposer().is_some() {
             for req_result in &result.requests {
                 if req_result.completed {
                     if let Some(req) = plan
@@ -1915,7 +1941,7 @@ impl ModelExecutor for Qwen3Executor {
         // A request becomes draft-ready once its prompt is fully prefilled with
         // captured target context. Partial chunks stay pending; ineligible
         // requests drop any stale worker state.
-        if self.speculative.is_some() {
+        if self.dflash_meta().is_some() {
             for req_result in &result.requests {
                 let captured = result
                     .dflash_context_captured_requests
@@ -1988,7 +2014,7 @@ impl ModelExecutor for Qwen3Executor {
         }
         // A plain decode commits one token per request; keep the n-gram context
         // in sync so it always ends with the request's dangling token.
-        if self.ngram.is_some() {
+        if self.ngram_proposer().is_some() {
             for req_result in &result.requests {
                 if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
                     context.push(req_result.token);
@@ -1997,7 +2023,7 @@ impl ModelExecutor for Qwen3Executor {
         }
         // A plain decode advances the sequence outside the speculative path, so
         // any captured draft context is now stale — drop it.
-        if self.speculative.is_some() {
+        if self.dflash_meta().is_some() {
             for req_result in &result.requests {
                 if self.dflash_ready_requests.remove(&req_result.request_id) {
                     self.primary.drop_dflash_request(req_result.request_id)?;
@@ -2016,7 +2042,7 @@ impl ModelExecutor for Qwen3Executor {
         // N-gram drafts are produced host-side on the executor thread (no worker
         // forward): scan each request's own token context for a prompt-lookup
         // match. DFlash instead runs a draft-model forward on the lane.
-        if let Some(ngram) = self.ngram {
+        if let Some(ngram) = self.ngram_proposer() {
             let mut requests = Vec::with_capacity(plan.requests.len());
             for item in plan.requests {
                 anyhow::ensure!(
@@ -2046,11 +2072,11 @@ impl ModelExecutor for Qwen3Executor {
     }
 
     fn speculative_enabled(&self) -> bool {
-        self.speculative.is_some() || self.ngram.is_some()
+        self.speculative_method.is_some()
     }
 
     fn speculative_request_ready(&self, request_id: RequestId) -> bool {
-        if self.ngram.is_some() {
+        if self.ngram_proposer().is_some() {
             // Ready as soon as the prompt is prefilled and its context seeded;
             // n-gram needs no captured hidden state.
             self.ngram_ctx.contains_key(&request_id)
@@ -2130,7 +2156,7 @@ impl ModelExecutor for Qwen3Executor {
                 // now stale — drop it, mirroring execute_decode. (Eligible pending
                 // are routed to a dedicated prefill step, so unified prefills never
                 // need DFlash mark-ready here.)
-                if self.speculative.is_some() {
+                if self.dflash_meta().is_some() {
                     for req_result in &result.decode_requests {
                         if self.dflash_ready_requests.remove(&req_result.request_id) {
                             self.primary.drop_dflash_request(req_result.request_id)?;
diff --git a/openinfer-qwen3-4b/src/executor/spec.rs b/openinfer-qwen3-4b/src/executor/spec.rs
index 764cd879..d3cccb26 100644
--- a/openinfer-qwen3-4b/src/executor/spec.rs
+++ b/openinfer-qwen3-4b/src/executor/spec.rs
@@ -17,7 +17,7 @@ impl Qwen3Executor {
         plan: VerifyPlan<'_>,
     ) -> Result<VerifyResult> {
         anyhow::ensure!(
-            self.speculative.is_some() || self.ngram.is_some(),
+            self.speculative_method.is_some(),
             "speculative verification requested but no speculative method is enabled"
         );
         for req in plan.requests {
@@ -30,7 +30,7 @@ impl Qwen3Executor {
                 req.params.is_greedy(),
                 "speculative verification currently supports greedy sampling only"
             );
-            if self.speculative.is_some() {
+            if self.dflash_meta().is_some() {
                 anyhow::ensure!(
                     self.dflash_ready_requests.contains(&req.request_id),
                     "speculative verification requested before DFlash state is ready for {:?}",
@@ -139,7 +139,7 @@ impl Qwen3Executor {
         // Keep the n-gram running context in sync: append every committed token
         // so the next draft scans the full history and continues from the new
         // dangling token. DFlash keeps its own per-request hidden state instead.
-        if self.ngram.is_some() {
+        if self.ngram_proposer().is_some() {
             for req_result in &result.requests {
                 self.ngram_ctx
                     .entry(req_result.request_id)
@@ -156,7 +156,7 @@ impl Qwen3Executor {
         plan: DraftPlan<'_>,
     ) -> Result<DraftResult> {
         anyhow::ensure!(
-            self.speculative.is_some(),
+            self.dflash_meta().is_some(),
             "speculative draft requested but no draft model is loaded"
         );
         for req in plan.requests {

From c17975a4d1989122df0f2046771ac6ebf0dc9266 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 17:42:02 +0800
Subject: [PATCH 23/31] style(qwen3): rustfmt the n-gram losslessness gate

cargo fmt --all --check clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../tests/ngram_speculative_gate.rs           | 43 ++++++++++++++-----
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
index e5dabcd3..1c7d8c03 100644
--- a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
+++ b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
@@ -61,7 +61,9 @@ static GPU: std::sync::Mutex<()> = std::sync::Mutex::new(());
 fn target_path_or_skip() -> Option<String> {
     match std::env::var("OPENINFER_TEST_MODEL_PATH") {
         Ok(path) => Some(path),
-        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => Some(MODEL_PATH.to_string()),
+        Err(_) if Path::new(MODEL_PATH).join("config.json").exists() => {
+            Some(MODEL_PATH.to_string())
+        }
         Err(_) => {
             eprintln!(
                 "skipping ngram gate: {MODEL_PATH}/config.json missing; set OPENINFER_TEST_MODEL_PATH"
@@ -388,9 +390,15 @@ fn ngram_speculative_greedy_matches_plain_greedy() {
     let mut failures = Vec::new();
     for (i, &prompt) in prompts.iter().enumerate() {
         let spec = generate(&handle, encoded[i].clone(), 0, GENERATED_TOKENS);
-        if let Err(failure) =
-            check_lossless(&handle, &tokenizer, i, prompt, &encoded[i], &baseline[i], &spec)
-        {
+        if let Err(failure) = check_lossless(
+            &handle,
+            &tokenizer,
+            i,
+            prompt,
+            &encoded[i],
+            &baseline[i],
+            &spec,
+        ) {
             failures.push(failure);
         }
     }
@@ -418,10 +426,19 @@ fn ngram_concurrent_heterogeneous_is_lossless() {
     let _gpu = GPU.lock().unwrap_or_else(|p| p.into_inner());
 
     let cases: [(&str, usize); 4] = [
-        ("def fibonacci(n):\n    if n < 2:\n        return n\n    return fibonacci(", 64),
+        (
+            "def fibonacci(n):\n    if n < 2:\n        return n\n    return fibonacci(",
+            64,
+        ),
         ("1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4", 24),
-        ("Q: What is 17 multiplied by 23? A: Let's think step by step.", 48),
-        ("The quick brown fox. The quick brown fox. The quick brown fox.", 40),
+        (
+            "Q: What is 17 multiplied by 23? A: Let's think step by step.",
+            48,
+        ),
+        (
+            "The quick brown fox. The quick brown fox. The quick brown fox.",
+            40,
+        ),
     ];
 
     let tokenizer = common::load_tokenizer(&model_path);
@@ -459,9 +476,15 @@ fn ngram_concurrent_heterogeneous_is_lossless() {
 
     let mut failures = Vec::new();
     for (i, (prompt, _)) in cases.iter().enumerate() {
-        if let Err(failure) =
-            check_lossless(&handle, &tokenizer, i, prompt, &encoded[i], &baselines[i], &specs[i])
-        {
+        if let Err(failure) = check_lossless(
+            &handle,
+            &tokenizer,
+            i,
+            prompt,
+            &encoded[i],
+            &baselines[i],
+            &specs[i],
+        ) {
             failures.push(failure);
         }
     }

From 1d7b6c983b8bf33d04c0372b17cef771c973c1f8 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 22:40:40 +0800
Subject: [PATCH 24/31] fix(qwen3): sync ngram_ctx on the unified decode path;
 address #349 review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review of #349 surfaced a real gap plus polish:

- 🔴 The fused unified decode path (execute_unified) committed tokens via
  apply_decode but never appended them to ngram_ctx — only execute_decode and
  the verify path did. Reachable whenever a non-greedy request shares the batch
  (n-gram does not reject temperature>0): should_speculative_decode flips false,
  and the co-resident greedy request decodes via Unified. Not a correctness bug
  (the verify seed is the request's real last_token, not ngram_ctx), but the
  context tracker drifts, silently degrading n-gram acceptance once the batch
  returns to pure greedy and violating the 'always ends with the dangling token'
  invariant. Fixed by mirroring execute_decode's sync in the Unified branch, and
  defensively in the SplitDecodeReady branch (unreachable today — overlap is off
  under speculation — but guards a future relaxation).
- Add ngram_mixed_greedy_and_sampling_is_lossless: a greedy request must stay
  token-identical to its solo baseline while a sampling request holds the batch
  non-greedy off the speculative path.
- 🟡 Fix the misplaced doc comment: the load_dflash_draft_model header had been
  split from its body by the inserted dflash_meta()/ngram_proposer() accessors.
- 🟢 max_context_tokens: document why n-gram needs no DFlash-style block_size
  reduction (verify span is budget-clamped, so it never writes past the window).
- 🟢 Soften the verify-buffer 'allocates at most once' comment (it grows 1->K+1)
  and note propose()'s O(context_len) per-step scan + the vLLM look-back-window
  option for long contexts.

cargo check -p openinfer-qwen3-4b {--lib,--tests} green; fmt clean; ngram units pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/executor.rs            |  48 +++++--
 openinfer-qwen3-4b/src/ngram.rs               |   7 +
 .../tests/ngram_speculative_gate.rs           | 135 ++++++++++++++++++
 3 files changed, 182 insertions(+), 8 deletions(-)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 723e47d8..0311d6e1 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -1291,13 +1291,6 @@ impl Qwen3Executor {
         }
     }
 
-    /// Enable speculative decoding by loading a DFlash draft model into the
-    /// primary lane.
-    ///
-    /// Requires the single-GPU topology (tensor parallel shards KV per rank) and
-    /// is incompatible with KV offload. Disables the prefix cache: speculative
-    /// capture needs clean, uncached target hidden states for every prompt
-    /// token, and a prefix-cache hit skips the forward that would produce them.
     /// The loaded DFlash draft metadata, or `None` under no/other method.
     fn dflash_meta(&self) -> Option<&DFlashMeta> {
         match &self.speculative_method {
@@ -1314,6 +1307,13 @@ impl Qwen3Executor {
         }
     }
 
+    /// Enable speculative decoding by loading a DFlash draft model into the
+    /// primary lane.
+    ///
+    /// Requires the single-GPU topology (tensor parallel shards KV per rank) and
+    /// is incompatible with KV offload. Disables the prefix cache: speculative
+    /// capture needs clean, uncached target hidden states for every prompt
+    /// token, and a prefix-cache hit skips the forward that would produce them.
     pub fn load_dflash_draft_model(&mut self, draft_path: &str) -> Result<()> {
         anyhow::ensure!(
             self.workers.is_empty(),
@@ -1681,6 +1681,12 @@ impl ModelExecutor for Qwen3Executor {
             // `draft.max_pos - block_size` tokens before the draft cache would
             // overflow. Reject the rest at admission instead of crashing mid-prefill.
             Some(meta) => target.min(meta.max_position_embeddings.saturating_sub(meta.block_size)),
+            // n-gram (or no speculation): full window. n-gram has no separate
+            // draft cache, and its verify span is clamped to the request's
+            // remaining output budget (see `build_speculative_verify_items`), so
+            // the furthest KV position it writes is `prompt + generated + span <=
+            // prompt + max_tokens <= max_position_embeddings` — never past the
+            // window. No DFlash-style block_size reduction is needed.
             None => target,
         }
     }
@@ -2151,6 +2157,19 @@ impl ModelExecutor for Qwen3Executor {
                         .expect("request must exist after unified decode");
                     rkv.apply_decode(req_result.token, self.kv_mgr.pool())?;
                 }
+                // A unified step commits one token per active request outside the
+                // speculative path; keep the n-gram context in sync (mirroring
+                // execute_decode) so it stays anchored on the dangling token when
+                // the batch returns to pure-greedy n-gram speculation. Reachable
+                // whenever a non-greedy request shares the batch and pushes the
+                // greedy ones onto the fused decode path.
+                if self.ngram_proposer().is_some() {
+                    for req_result in &result.decode_requests {
+                        if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
+                            context.push(req_result.token);
+                        }
+                    }
+                }
                 // A plain decode via the fused unified step advances the sequence
                 // outside the speculative path, so any captured draft context is
                 // now stale — drop it, mirroring execute_decode. (Eligible pending
@@ -2184,6 +2203,17 @@ impl ModelExecutor for Qwen3Executor {
                         .expect("request must exist after split decode");
                     rkv.apply_decode(req_result.token, self.kv_mgr.pool())?;
                 }
+                // Defensive n-gram context sync (mirrors the Unified branch). The
+                // overlap path is unreachable under n-gram today (speculation forces
+                // decode-overlap off), but keep the context anchored if that
+                // mutual-exclusion is ever relaxed.
+                if self.ngram_proposer().is_some() {
+                    for req_result in &decode_result.requests {
+                        if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
+                            context.push(req_result.token);
+                        }
+                    }
+                }
                 for req_result in &decode_result.requests {
                     self.save_sealed_blocks(req_result.request_id);
                 }
@@ -2806,7 +2836,9 @@ impl LocalQwen3Lane {
             .unwrap_or(1)
             .max(1);
         // (Re)allocate the verify scratch if it is missing or too narrow for this
-        // step's widest span. With a fixed n-gram K this allocates at most once.
+        // step's widest span. The widest span grows from 1 (no match) up to K + 1
+        // as matches lengthen, so this may reallocate a few times early on before
+        // settling at the K + 1 ceiling — never per step in steady state.
         let needs_alloc = self
             .verify_bufs
             .as_ref()
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index 16aa71ad..dfc09dac 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -102,6 +102,13 @@ impl NgramProposer {
     /// Returns an empty `Vec` when no usable n-gram match exists or when
     /// `num_speculative == 0`. Tries the longest configured suffix first and
     /// falls back to shorter suffixes.
+    ///
+    /// Cost: each call is a linear reverse scan of `context` per suffix length,
+    /// so a miss is `O(context_len * max_ngram)` on the executor thread, every
+    /// speculative step. Fine for the target case (repetitive / bounded
+    /// contexts); a very long context could erode the speedup it buys. If that
+    /// becomes a concern, bound the look-back window (as vLLM's prompt-lookup
+    /// does) rather than scanning the whole history.
     #[must_use]
     pub fn propose(&self, context: &[u32]) -> Vec<u32> {
         if self.config.num_speculative == 0 {
diff --git a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
index 1c7d8c03..d55e679b 100644
--- a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
+++ b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
@@ -496,3 +496,138 @@ fn ngram_concurrent_heterogeneous_is_lossless() {
         failures.join("\n")
     );
 }
+
+/// Mixed greedy + sampling traffic must not break the greedy request.
+///
+/// A non-greedy request sharing the batch flips `should_speculative_decode` to
+/// false (it requires *every* active request to be greedy), so the co-resident
+/// greedy request decodes outside the speculative path — via plain decode, or
+/// via the fused unified step when a non-greedy request is pending while the
+/// greedy one is active. Both paths must keep `ngram_ctx` anchored on the
+/// dangling token; a missed sync there silently degrades n-gram acceptance once
+/// the batch returns to pure greedy (it is not a correctness bug — the verify
+/// seed comes from the real KV, not `ngram_ctx`). This guards that mixed-traffic
+/// class: the greedy request must stay token-identical to its solo plain-greedy
+/// baseline. The sampling request's output is non-deterministic, so only the
+/// greedy request is checked; it is there purely to hold the batch non-greedy.
+///
+/// Note: deterministically landing the greedy request on the *unified* branch
+/// (vs plain decode) depends on scheduler arrival timing the public API can't
+/// pin down; the unified context-sync itself mirrors the proven `execute_decode`
+/// path, so this test guards the broader property rather than that one branch.
+#[test]
+fn ngram_mixed_greedy_and_sampling_is_lossless() {
+    let Some(model_path) = target_path_or_skip() else {
+        return;
+    };
+    let _gpu = GPU.lock().unwrap_or_else(|p| p.into_inner());
+
+    let greedy_prompt = "def fibonacci(n):\n    if n < 2:\n        return n\n    return fibonacci(";
+    const GREEDY_MAX: usize = 48;
+    let tokenizer = common::load_tokenizer(&model_path);
+    let greedy_tokens = tokenizer
+        .encode(greedy_prompt, false)
+        .expect("encode failed");
+    let sampling_tokens = tokenizer
+        .encode("Tell me a long and winding story about a dragon.", false)
+        .expect("encode failed");
+
+    // 1. Baseline: the greedy prompt's solo plain-greedy decode (spec off) with
+    //    logprobs for the regret reference.
+    let baseline = {
+        let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(false))
+            .expect("failed to start baseline engine");
+        let out = generate(&handle, greedy_tokens.clone(), LOGPROBS, GREEDY_MAX);
+        drop(handle);
+        std::thread::sleep(Duration::from_secs(2));
+        out
+    };
+
+    // 2. Speculative engine (n-gram on). Submit a long *sampling* request and the
+    //    greedy request concurrently so they co-reside: the sampling request keeps
+    //    the batch non-greedy, forcing the greedy one off the speculative path.
+    let handle = openinfer_qwen3_4b::launch(Path::new(&model_path), launch_options(true))
+        .expect("failed to start speculative engine");
+
+    // Non-greedy (temperature >= eps, top_k != 1) so the batch is held non-greedy;
+    // start from Default to stay robust to added fields.
+    let sampling_params = SamplingParams {
+        temperature: 0.8,
+        top_p: 0.95,
+        ..SamplingParams::default()
+    };
+    let (sampling_tx, mut sampling_rx) = TokenSink::standalone();
+    handle
+        .submit(GenerateRequest {
+            request_id: None,
+            queued_at_unix_s: None,
+            prompt_tokens: sampling_tokens,
+            params: sampling_params,
+            max_tokens: 96,
+            lora_adapter: None,
+            token_tx: sampling_tx,
+            logprobs: 0,
+            echo: false,
+        })
+        .expect("submit sampling failed");
+    let (greedy_tx, mut greedy_rx) = TokenSink::standalone();
+    handle
+        .submit(GenerateRequest {
+            request_id: None,
+            queued_at_unix_s: None,
+            prompt_tokens: greedy_tokens.clone(),
+            params: SamplingParams::default(),
+            max_tokens: GREEDY_MAX,
+            lora_adapter: None,
+            token_tx: greedy_tx,
+            logprobs: 0,
+            echo: false,
+        })
+        .expect("submit greedy failed");
+
+    // Drain the greedy request's steps; the sampling request just runs alongside.
+    let mut greedy_steps = Vec::new();
+    loop {
+        match greedy_rx.blocking_recv().map(|(_, event)| event) {
+            Some(TokenEvent::Token { id, logprob }) => greedy_steps.push(Step {
+                id,
+                top_logprobs: logprob.map(|lp| lp.top_logprobs).unwrap_or_default(),
+            }),
+            Some(TokenEvent::Scheduled { .. } | TokenEvent::PromptTokens { .. }) => {}
+            Some(TokenEvent::Finished { .. }) => break,
+            Some(TokenEvent::Error { message, .. }) => {
+                panic!("greedy generation failed: {message}")
+            }
+            Some(TokenEvent::Rejected { message, .. }) => {
+                panic!("greedy generation rejected: {message}")
+            }
+            None => panic!("scheduler channel closed without Finished"),
+        }
+    }
+    // Drain the sampling request to completion so the engine tears down cleanly.
+    while let Some((_, event)) = sampling_rx.blocking_recv() {
+        if matches!(
+            event,
+            TokenEvent::Finished { .. } | TokenEvent::Error { .. } | TokenEvent::Rejected { .. }
+        ) {
+            break;
+        }
+    }
+
+    let result = check_lossless(
+        &handle,
+        &tokenizer,
+        0,
+        greedy_prompt,
+        &greedy_tokens,
+        &baseline,
+        &greedy_steps,
+    );
+    drop(handle);
+
+    assert!(
+        result.is_ok(),
+        "greedy n-gram request diverged from plain greedy under mixed greedy+sampling traffic:\n{}",
+        result.unwrap_err()
+    );
+}

From 3f27ad7adaba7079843ad9685d9ff0854349cf79 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 22:51:36 +0800
Subject: [PATCH 25/31] style(qwen3): pub(crate) the n-gram proposer to clear
 unreachable_pub
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

NgramProposer/new/propose are only used within the crate (mod ngram is private;
the gate test drives through the engine, not the proposer directly), so their
pub visibility tripped the workspace unreachable_pub lint. NgramConfig stays pub
— it appears in load_ngram_drafter's public signature.

GPU-validated: ngram_speculative_gate passes 3/3 on RTX 4090 (Qwen3-4B), every
prompt 100% token-identical to plain greedy across bs=1, concurrent, and
mixed greedy+sampling.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/ngram.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index dfc09dac..abe1b86c 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -75,7 +75,7 @@ fn env_usize(name: &str) -> Option<usize> {
 /// The proposer keeps no history of its own; each call scans the supplied
 /// context, so the caller is free to reuse a single instance across requests.
 #[derive(Clone, Copy, Debug)]
-pub struct NgramProposer {
+pub(crate) struct NgramProposer {
     config: NgramConfig,
 }
 
@@ -84,7 +84,7 @@ impl NgramProposer {
     /// `max_ngram`, and `max_ngram` to at least 1, so the config is always
     /// usable regardless of caller input.
     #[must_use]
-    pub fn new(config: NgramConfig) -> Self {
+    pub(crate) fn new(config: NgramConfig) -> Self {
         let max_ngram = config.max_ngram.max(1);
         let min_ngram = config.min_ngram.clamp(1, max_ngram);
         Self {
@@ -110,7 +110,7 @@ impl NgramProposer {
     /// becomes a concern, bound the look-back window (as vLLM's prompt-lookup
     /// does) rather than scanning the whole history.
     #[must_use]
-    pub fn propose(&self, context: &[u32]) -> Vec<u32> {
+    pub(crate) fn propose(&self, context: &[u32]) -> Vec<u32> {
         if self.config.num_speculative == 0 {
             return Vec::new();
         }

From 8c5b5adc4aa2bb436d62d2aaac34367193d544b1 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Wed, 24 Jun 2026 23:23:04 +0800
Subject: [PATCH 26/31] perf(qwen3): route no-draft n-gram steps through plain
 decode
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses #349 review (P2): when n-gram proposes nothing (non-repetitive
prompt, or OPENINFER_QWEN3_NGRAM_TOKENS=0) the verify span is just
[current_token], yet the scheduler had already picked SpeculativeDecode — so the
no-hit token went through the prefill-style verify forward instead of the
optimized decode kernel. That cost a full verify for one token and could flip a
near-tie greedy pick by the prefill-vs-decode bf16 gap, with zero speculative
upside.

Now execute_plan partitions the post-propose batch: requests with drafts (verify
span > 1) take the verify forward; requests with no draft fall back to
execute_decode — same kernel, same cost, and bit-identical to baseline. All-miss
steps (common on non-repetitive text) degrade exactly to plain decode, so
enabling n-gram never regresses below baseline. DFlash is unaffected: its draft
always emits block_size tokens, so its spans are never length 1.

GPU-validated: ngram_speculative_gate still 3/3 on RTX 4090 (Qwen3-4B), all
prompts 100% token-identical (the 17x23 reasoning prompt exercises the no-draft
fallback path). cargo check --lib/--tests green; fmt clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/scheduler/plan.rs | 64 ++++++++++++++++++++----
 1 file changed, 55 insertions(+), 9 deletions(-)

diff --git a/openinfer-qwen3-4b/src/scheduler/plan.rs b/openinfer-qwen3-4b/src/scheduler/plan.rs
index 19b1bf7e..e295ece3 100644
--- a/openinfer-qwen3-4b/src/scheduler/plan.rs
+++ b/openinfer-qwen3-4b/src/scheduler/plan.rs
@@ -6,7 +6,8 @@ use crate::executor::{
     PrefillStepItem, UnifiedPlan, UnifiedResult,
 };
 use crate::speculative::{
-    DraftPlan, DraftRequestResult, DraftStepItem, VerifyPlan, VerifyResult, VerifyStepItem,
+    DraftPlan, DraftRequestResult, DraftStepItem, VerifyPlan, VerifyRequestResult, VerifyResult,
+    VerifyStepItem,
 };
 
 use super::{ActiveRequestState, PendingRequest};
@@ -113,20 +114,65 @@ pub(super) fn execute_plan(
             Ok(ExecutionArtifacts::Decode { result })
         }
         ExecutionPlan::SpeculativeDecode => {
-            // Two executor calls per step: draft proposes K tokens per request,
-            // then a single target forward verifies the K+1 span. Both index by
-            // request_id; sorting keeps draft and verify results aligned.
+            // Propose first, then route each request by whether it actually got a
+            // draft. Drafted requests (verify span > 1) take the speculative
+            // verify forward. Requests with no draft (n-gram miss, or K = 0) would
+            // otherwise pay a full verify forward for a single token — swapping the
+            // optimized decode kernel for the prefill-style verify kernel, which
+            // both costs more and perturbs near-tie greedy picks by the
+            // prefill-vs-decode bf16 gap. Route those through the plain decode path
+            // instead: same kernel and cost as baseline, committing one token each.
+            // All-miss steps (common on non-repetitive text) thus degrade exactly
+            // to plain decode. All index by request_id; sorting keeps results
+            // aligned for resolve.
             let draft_requests = build_speculative_draft_items(active);
             let mut draft = executor.execute_speculative_draft(DraftPlan {
                 requests: &draft_requests,
             })?;
             draft.requests.sort_by_key(|result| result.request_id);
             let verify_requests = build_speculative_verify_items(active, &draft.requests);
-            let mut verify = executor.execute_speculative_verify(VerifyPlan {
-                requests: &verify_requests,
-            })?;
-            verify.requests.sort_by_key(|result| result.request_id);
-            Ok(ExecutionArtifacts::SpeculativeDecode { verify })
+            let (drafted, undrafted): (Vec<VerifyStepItem>, Vec<VerifyStepItem>) = verify_requests
+                .into_iter()
+                .partition(|item| item.token_ids.len() > 1);
+
+            let mut request_results: Vec<VerifyRequestResult> = Vec::with_capacity(active.len());
+            if !drafted.is_empty() {
+                let mut verify =
+                    executor.execute_speculative_verify(VerifyPlan { requests: &drafted })?;
+                request_results.append(&mut verify.requests);
+            }
+            if !undrafted.is_empty() {
+                // logprobs are off and LoRA is absent on the speculative path
+                // (should_speculative_decode requires both), so a minimal decode
+                // item is faithful to these requests.
+                let decode_items: Vec<DecodeStepItem> = undrafted
+                    .iter()
+                    .map(|item| DecodeStepItem {
+                        request_id: item.request_id,
+                        token_id: item.token_ids[0],
+                        params: item.params,
+                        logprobs: 0,
+                        lora_adapter: None,
+                    })
+                    .collect();
+                let decode = executor.execute_decode(DecodePlan {
+                    requests: &decode_items,
+                    sample_seed: rand::RngExt::random(rng),
+                })?;
+                for result in decode.requests {
+                    request_results.push(VerifyRequestResult {
+                        request_id: result.request_id,
+                        matched_draft_tokens: 0,
+                        accepted_tokens: vec![result.token],
+                    });
+                }
+            }
+            request_results.sort_by_key(|result| result.request_id);
+            Ok(ExecutionArtifacts::SpeculativeDecode {
+                verify: VerifyResult {
+                    requests: request_results,
+                },
+            })
         }
         ExecutionPlan::Unified { pending } => {
             let scheduled_at_unix_s = openinfer_core::engine::unix_now_s();

From 569af201dcfa811edfa56b8aaebb022c9e7c6778 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Thu, 25 Jun 2026 16:10:43 +0800
Subject: [PATCH 27/31] test(qwen3): fail the n-gram lossless gate on
 strict-prefix outputs; list the design note
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Address #349 review.

The gate's success check `matched == base.len().min(spec.len())` returned
Ok(()) whenever every overlapping token matched, even when the two runs had
different lengths — so a speculative path that emits EOS/Length early (or runs
on past the baseline) passed as "100% lossless". It also risked an
out-of-bounds panic on the divergence diagnostic when `spec` was the shorter
run. Reject length mismatches explicitly: greedy speculation is lossless only
if the streams are token-for-token AND length identical.

Also list `models/qwen3/ngram-speculative.md` in docs/index.md per the docs
workflow (every doc must appear in the routing table).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/index.md                                  |  1 +
 .../tests/ngram_speculative_gate.rs            | 18 +++++++++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/index.md b/docs/index.md
index 8cce9e77..2b41d187 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -31,6 +31,7 @@ Organized by domain (model line / subsystem / playbook / lesson) instead of by l
 | `models/qwen3/model-crate.md` | `openinfer-qwen3-4b` owns Qwen3 config/weights/executor/scheduler/tests/kernel plan; root sees generic `EngineHandle`; split-K retuned to `256/64`, with 4k/64 serving TPOT p50 at `6.46ms` on RTX 5090. |
 | `models/qwen3/prefix-cache.md` | Prefix caching on by default for Qwen3-4B: full-block kvbm radix matching at the executor, suffix-only prefill. Repeated ~1900-token prompt TTFT 141.8 → 16.3ms p50 (8.7×); warm TTFT ≈ TPOT + ~5ms setup. Includes the RoPE scalar-path corruption fix and the drain-the-stream TTFT measurement pitfall. |
 | `models/qwen3/dflash-speculative-decoding.md` | DFlash speculative decoding behind `--dflash-draft-model-path`, modelled as an optimistic transaction (propose K → verify K+1 span → accept longest argmax prefix + 1 bonus → commit/roll back KV). Lossless up to bf16 tie-flips (bit-identical multi-token accepts; lm-eval gsm8k strict-match identical spec on/off). Single-stream decode 1.82× on 5070 Ti, 1.56× on 5090. Concurrent throughput fixed by batching the draft forward, then a piecewise verify CUDA Graph (dense ops captured, attention eager) closed single-stream: 5090 greedy c1 274 ≈ vLLM 278, c8 1525 > 1240, c16 1834 ≈ 1846 — all batch sizes now ≥ vLLM. Accept measured equal (9.1% vs 8.85%, same drafter); draft-side piecewise graph tracked next. Proposer trait deferred to EAGLE. |
+| `models/qwen3/ngram-speculative.md` | Draft-model-free (n-gram / prompt-lookup) speculative decoding for Qwen3-4B behind `--ngram-speculative`, built on the same #436 verify/accept/KV transaction as DFlash — only *propose* differs (host-side longest-suffix lookup over the request's own token history, no worker forward, no draft model). Greedy verification is **lossless** (bit-for-bit identical to plain greedy, modulo bf16 tie flips), GPU-validated 3/3 in `tests/ngram_speculative_gate.rs`. Method state collapsed into one `SpeculativeMethod{ Dflash \| Ngram }` enum; full proposer trait deferred to EAGLE (#325). Single-GPU greedy only; mutually exclusive with DFlash / LoRA / KV-offload / decode-overlap; forces prefix cache off. |
 | `models/qwen3/accuracy-gate.md` | Qwen3-4B instance of the logits golden gate (`tests/hf_golden_gate.rs`): 48 teacher-forced sequences / 816 positions vs a stored HF bf16 golden, replayed over bs=1 / batched eager / CUDA-graph. Strict guards: regret check + mean ≤ 0.06 + p99 ≤ 0.20; absolute max printed but not asserted (coverage-unstable). Methodology in `subsystems/correctness/`. |
 | `models/qwen3/kernels-crate.md` | Phase 1 split implemented and 5090-verified: Qwen3-4B kernel surface lives in `openinfer-kernels`; release build, test-target compile, accuracy gate, and bench snapshot pass. |
 | `models/qwen3/tp-design.md` | Qwen3 tensor-parallel design: `TP=2` milestone scope plus the controller/worker broadcast execution model, request identity, and coarse-grained step protocol for future TP/MoE work. |
diff --git a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
index d55e679b..7ef4f087 100644
--- a/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
+++ b/openinfer-qwen3-4b/tests/ngram_speculative_gate.rs
@@ -254,7 +254,23 @@ fn check_lossless(
         .take_while(|(b, s)| b.id == s.id)
         .count();
 
-    if matched == base.len().min(spec.len()) {
+    // A strict-prefix relationship (one run stops early, or runs on past the
+    // other) must fail the gate even though every overlapping token matched:
+    // greedy speculation is lossless only if the two streams are token-for-token
+    // *and* length identical. Reject length mismatches before the divergence
+    // diagnostic below, which indexes `spec[matched]` and would otherwise panic
+    // out of bounds when `spec` is the shorter run.
+    if base.len() != spec.len() {
+        return Err(format!(
+            "prompt {i} ({prompt:?}): length mismatch — baseline produced {} tokens, \
+             speculative produced {} tokens ({matched} prefix tokens matched); greedy \
+             speculation must be token-for-token identical, not merely a shared prefix",
+            base.len(),
+            spec.len(),
+        ));
+    }
+
+    if matched == base.len() {
         eprintln!(
             "prompt {i} ({prompt:?}): {matched}/{} tokens identical (100% lossless)",
             base.len()

From 8ae1b3344c81068168a9b9be67497fa83f874b02 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Mon, 29 Jun 2026 11:10:04 +0800
Subject: [PATCH 28/31] perf(qwen3): gate n-gram speculation on draft
 acceptance
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On non-repetitive text (prose) prompt-lookup drafts are almost never the
target's greedy continuation (~5-7% acceptance), so every speculative step
pays a K+1-wide verify forward — plus, when only some requests in the batch
matched, a second plain-decode pass — for ~no accepted tokens. At higher
concurrency the GPU has no spare compute to hide that, so n-gram regressed
serving throughput badly (sonnet -26% c1 / -40% c4 on Qwen3-4B / RTX 4090).

Add an engine-wide acceptance gate (`NgramGate`): an EWMA of accepted draft
tokens per drafted step. When it falls below `accept_threshold` the proposer
returns nothing for the step, so the existing draft/undraft partition routes
the whole batch to plain decode — no verify forward, no regression. A periodic
probe re-opens the gate so a shift into repetitive text is picked back up.
Gating only changes whether we speculate, never the committed tokens, so the
losslessness gate still passes 3/3.

Serving A/B (vllm bench serve, Qwen3-4B, RTX 4090), n-gram vs plain:
  sonnet c1  -26.2% -> +0.6% | c4  -40.0% -> -2.2%
  random c1  -10.3% -> +21.8%| c4  -33.8% -> -20.0%
Prose regression is recovered (now better than vLLM's -6%, which keeps
speculating); random c1 matches vLLM (+21.8% vs +19.3%). random c4 stays
negative at high (~37%) acceptance — that gap is the verify path itself
(two forwards + prefill-kernel verify vs vLLM's single fused forward), tracked
as a follow-up in docs/models/qwen3/ngram-speculative.md.

Tunable via OPENINFER_QWEN3_NGRAM_ACCEPT_THRESHOLD (default 0.3, 0 disables).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 docs/models/qwen3/ngram-speculative.md  |  91 ++++++++++++---
 openinfer-qwen3-4b/src/executor.rs      |  34 +++++-
 openinfer-qwen3-4b/src/executor/spec.rs |  12 ++
 openinfer-qwen3-4b/src/ngram.rs         | 148 +++++++++++++++++++++++-
 4 files changed, 259 insertions(+), 26 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 36966787..66f7765d 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -7,14 +7,14 @@ longest greedy-agreed prefix plus one model token. Greedy verification is
 **lossless** — output is bit-for-bit identical to plain greedy decode, just
 fewer forward passes on repetitive / structured text (code, quoting, JSON).
 The KV layer (kvbm) already supports speculative scheduling natively, so
-rejected drafts need no manual rollback.
+rejected drafts need no manual rollback. An **acceptance gate** (`NgramGate`)
+closes speculation when recent draft acceptance is too low to pay for the verify
+forward, so low-acceptance traffic (prose) falls back to plain decode instead of
+regressing — see *Serving A/B* below.
 
-Last touched: 2026-06 · Status: **end-to-end implemented and gated behind a
-(default-off) `SpeculativeConfig`: proposer, greedy acceptance, KV speculative
-pass-throughs, the executor verify forward (`execute_speculative`,
-GPU-validated lossless), and the scheduler serving-loop wiring
-(`speculative_decode_step`, mock-tested). Remaining: a public config knob to
-turn it on, and a speedup measurement.**
+Last touched: 2026-06 · Enabled via `--ngram-speculative`; proposer, greedy
+acceptance, KV speculative pass-throughs, the verify forward, scheduler wiring,
+and the acceptance gate are all landed and GPU-validated lossless.
 
 ## Why this is cheap to add here
 
@@ -137,6 +137,9 @@ parses its own knobs (`NgramConfig::from_env`):
 - `OPENINFER_QWEN3_SPEC=1` — turn speculation on (generic).
 - `OPENINFER_QWEN3_NGRAM_TOKENS=K` — draft count (n-gram, default 4).
 - `OPENINFER_QWEN3_NGRAM_MAX_NGRAM=N` — longest matched suffix (n-gram, default 3).
+- `OPENINFER_QWEN3_NGRAM_ACCEPT_THRESHOLD=f` — acceptance gate threshold (mean
+  accepted draft tokens/step below which speculation falls back to plain decode;
+  default 0.3, `0` disables the gate). See *Serving A/B* below.
 
 Only the non-LoRA `scheduler_loop` reads it; the unified prefill+decode tick
 still uses plain decode.
@@ -150,17 +153,6 @@ request takes a normal sampled single-token decode (its own params, logprobs,
 and a fresh `random_val`) on that tick, so enabling speculation never changes a
 sampled request's output or strips requested logprobs.
 
-## Remaining work
-
-1. **First-class config knob**: env var is the current switch; thread a typed
-   knob through `start_qwen3*` / `start_engine*` (and a server flag) so it shows
-   up in the engine config rather than the environment.
-2. **Speedup measurement** (initial numbers below — generalize to realistic
-   prompts and the scheduler loop).
-3. **Batched / vLLM-style verify** (perf): fold the verify tokens into the
-   unified batched forward (FlashInfer varlen) with a GPU rejection step,
-   instead of the current per-request `batch_prefill`-based verify.
-
 ## Measured speedup (best case)
 
 `tests/ngram_speculative.rs::ngram_speculative_speedup` (ignored; needs GPU +
@@ -181,6 +173,69 @@ Run: `cargo test -p openinfer-qwen3-4b --release --test ngram_speculative \
 ngram_speculative_speedup -- --ignored --nocapture` (`OPENINFER_BENCH_TOKENS`
 overrides the 192-token default).
 
+## Serving A/B (`vllm bench serve`) and the acceptance gate
+
+Synthetic best-case is the ceiling; serving throughput on real datasets is the
+contract. Single RTX 4090, Qwen3-4B, `temperature=0`, `--ignore-eos`,
+`--no-prefix-cache`, 12 prompts × 128 output tokens, sonnet (550/128, prefix 200)
+and random (550/128). `Δ` is n-gram vs plain decode on the **same** engine;
+vLLM (0.21 / 0.23, ngram `K=4`, `prompt_lookup_max=3`) is the reference.
+
+| dataset / conc | acceptance | vLLM 0.23 Δ | openinfer (no gate) Δ | **openinfer (gate) Δ** |
+| --- | --- | --- | --- | --- |
+| sonnet c1 | ~5% | −5.8% | −26.2% | **+0.6%** |
+| sonnet c4 | ~7% | −5.8% | −40.0% | **−2.2%** |
+| random c1 | ~33% | +19.3% | −10.3% | **+21.8%** |
+| random c4 | ~37% | +17.0% | −33.8% | **−20.0%** |
+
+Reading this:
+
+- **Prose (sonnet) acceptance is ~5–7%** — prompt-lookup drafts are almost never
+  the model's greedy continuation, so *every* engine loses here (vLLM included).
+  Without the gate the loss is severe (−40% at c4) because each step pays the
+  verify forward (a `K+1`-wide prefill-kernel pass) plus, when only some requests
+  matched, a second plain-decode pass — both wasted. The gate detects the low
+  acceptance and falls back to plain decode, recovering the regression to ≈0 and
+  **beating vLLM**, which keeps speculating and eats its −6%.
+- **random c1 matches vLLM** (+21.8% vs +19.3%): low concurrency leaves the GPU
+  compute-idle, so the gate stays open and the ~33% acceptance is a free win.
+- **random c4 is the open gap**: vLLM **+17%** vs openinfer **−20%** at the *same*
+  ~37% acceptance (proven by the matching c1 numbers). This is purely the verify
+  path — see below. The gate can't help here: acceptance is high, so it correctly
+  stays open; the loss is in *how* we verify, not *whether* we should.
+
+**Root cause of the random-c4 gap (nsys, `--cuda-graph-trace=node`).** At c4 the
+GPU is already ~92% busy on plain decode. Under n-gram the same 22 s window shows
+the attention running through the **prefill** kernel (`BatchPrefill`, ~23 µs)
+instead of the decode kernel (`BatchDecode`, ~15 µs), ~989 verify forwards **plus**
+~887 decode forwards (the batch splits two ways every mixed step), and ~43% more
+GEMM. vLLM instead runs **one fused forward**: draft tokens are extra rows in the
+same batch (a missed request is just `query_len=1`), verified by one rejection
+step — no second pass, no prefill-kernel penalty, no ragged-span CUDA-graph
+thrash. Folding our verify into that single unified forward is the way to turn
+random-c4 positive; it is tracked as the follow-up below, not in this PR.
+
+## The acceptance gate (`NgramGate`)
+
+`openinfer-qwen3-4b/src/ngram.rs`. Engine-wide EWMA of accepted draft tokens per
+drafted step. `should_draft` opens while in warmup, while the EWMA clears
+`accept_threshold`, or on a periodic probe; otherwise the proposer returns
+nothing for the step and the existing draft/undraft partition routes the whole
+batch to plain decode (no verify forward). The probe (every `PROBE_INTERVAL`
+steps) re-opens it so a shift into repetitive text is picked back up. Lossless
+either way — gating only changes *whether* we speculate, never the committed
+tokens. Unit-tested (`gate_*` in `ngram.rs`); the engine-level losslessness gate
+(`tests/ngram_speculative_gate.rs`) still passes with the gate live.
+
+## Remaining work / follow-up
+
+1. **Single fused verify forward** (the random-c4 fix, separate PR): fold the
+   verify tokens into one unified batched forward (FlashInfer varlen) with a GPU
+   rejection step — a missed request becomes a `query_len=1` row in the same
+   batch — instead of the current `batch_prefill`-based verify plus the second
+   plain-decode pass for undrafted requests. The *Serving A/B* above quantifies
+   the prize: closing the ~37-point random-c4 gap to vLLM (+17% vs −20%).
+
 ## Scope / deferred
 
 First cut is **single-request, greedy, non-CUDA-graph**. Deferred: batched
diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 74808b93..1f123611 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -856,6 +856,11 @@ pub struct Qwen3Executor {
     /// prompt-lookup matches; it always ends with the request's current dangling
     /// token so the next draft continues from it.
     ngram_ctx: HashMap<RequestId, Vec<u32>>,
+    /// Engine-wide acceptance gate for the n-gram method. Closes speculation when
+    /// recent draft acceptance is too low to pay for the verify forward (e.g.
+    /// prose), so the batch falls back to plain decode instead of regressing —
+    /// most visibly at higher concurrency. Idle/`None` under DFlash.
+    ngram_gate: crate::ngram::NgramGate,
     /// Opt-in KV block-event feed for a cache-aware router (`Some` only when the
     /// engine was built with events on — single-GPU, no LoRA). `None` on the
     /// plain path, where the whole feed costs nothing.
@@ -978,6 +983,7 @@ impl Qwen3Executor {
             speculative_method: None,
             dflash_ready_requests: HashSet::new(),
             ngram_ctx: HashMap::new(),
+            ngram_gate: crate::ngram::NgramGate::new(),
             kv_events,
         })
     }
@@ -1208,6 +1214,7 @@ impl Qwen3Executor {
             speculative_method: None,
             dflash_ready_requests: HashSet::new(),
             ngram_ctx: HashMap::new(),
+            ngram_gate: crate::ngram::NgramGate::new(),
             // KV events are single-GPU only (asserted above); never wired here.
             kv_events: None,
         })
@@ -2049,16 +2056,28 @@ impl ModelExecutor for Qwen3Executor {
         // forward): scan each request's own token context for a prompt-lookup
         // match. DFlash instead runs a draft-model forward on the lane.
         if let Some(ngram) = self.ngram_proposer() {
+            // One gate decision per step. When closed (recent acceptance too low
+            // to pay for the verify forward), propose nothing: every request then
+            // has a length-1 verify span and the scheduler routes the whole batch
+            // to plain decode — no verify forward, no regression. The gate probes
+            // periodically so a shift into repetitive text re-opens it.
+            let drafting = self.ngram_gate.should_draft(ngram.config().accept_threshold);
+            let mut any_drafted = false;
             let mut requests = Vec::with_capacity(plan.requests.len());
             for item in plan.requests {
                 anyhow::ensure!(
                     item.params.is_greedy(),
                     "n-gram speculative draft currently supports greedy sampling only"
                 );
-                let context = self.ngram_ctx.get(&item.request_id).ok_or_else(|| {
-                    anyhow::anyhow!("missing n-gram context for {:?}", item.request_id)
-                })?;
-                let drafts = ngram.propose(context);
+                let drafts = if drafting {
+                    let context = self.ngram_ctx.get(&item.request_id).ok_or_else(|| {
+                        anyhow::anyhow!("missing n-gram context for {:?}", item.request_id)
+                    })?;
+                    ngram.propose(context)
+                } else {
+                    Vec::new()
+                };
+                any_drafted |= !drafts.is_empty();
                 // Verify span = [current dangling token, draft_1, …, draft_K].
                 let mut token_ids = Vec::with_capacity(1 + drafts.len());
                 token_ids.push(item.current_token);
@@ -2068,6 +2087,13 @@ impl ModelExecutor for Qwen3Executor {
                     token_ids,
                 });
             }
+            // No draft produced (gate closed, or open but every request missed)
+            // means no verify forward runs this step, so advance the gate's probe
+            // clock toward the next re-check. A step that did draft updates the
+            // acceptance estimate from its verify result instead (see spec.rs).
+            if !any_drafted {
+                self.ngram_gate.record_skipped();
+            }
             return Ok(DraftResult { requests });
         }
         self.execute_speculative_draft_impl(plan)
diff --git a/openinfer-qwen3-4b/src/executor/spec.rs b/openinfer-qwen3-4b/src/executor/spec.rs
index d3cccb26..c074265d 100644
--- a/openinfer-qwen3-4b/src/executor/spec.rs
+++ b/openinfer-qwen3-4b/src/executor/spec.rs
@@ -146,6 +146,18 @@ impl Qwen3Executor {
                     .or_default()
                     .extend_from_slice(&req_result.accepted_tokens);
             }
+            // Feed the acceptance gate: this verify forward ran because the gate
+            // was open, so fold its mean accepted-draft-tokens into the estimate.
+            // `result.requests` is the drafted subset, so it is non-empty here.
+            if !result.requests.is_empty() {
+                let accepted: usize = result
+                    .requests
+                    .iter()
+                    .map(|r| r.matched_draft_tokens)
+                    .sum();
+                let mean = accepted as f32 / result.requests.len() as f32;
+                self.ngram_gate.record_drafted(mean);
+            }
         }
 
         Ok(result)
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index abe1b86c..f86e585f 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -15,7 +15,7 @@
 //! module so the lookup logic can be unit-tested in isolation.
 
 /// Configuration for [`NgramProposer`].
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub struct NgramConfig {
     /// Largest suffix length to match. Longer suffixes are tried first because
     /// they are more specific (a longer match is a stronger predictor).
@@ -24,6 +24,13 @@ pub struct NgramConfig {
     pub min_ngram: usize,
     /// Maximum number of speculative tokens proposed per step.
     pub num_speculative: usize,
+    /// Mean accepted-draft-tokens-per-step below which speculation is judged not
+    /// worth its verify forward, so the engine falls back to plain decode (see
+    /// [`NgramGate`]). On non-repetitive text (e.g. prose) prompt-lookup drafts
+    /// are almost never accepted, and at higher concurrency the wasted verify
+    /// compute is a net throughput loss; gating recovers it. `0.0` disables the
+    /// gate (always speculate).
+    pub accept_threshold: f32,
 }
 
 impl Default for NgramConfig {
@@ -32,6 +39,7 @@ impl Default for NgramConfig {
             max_ngram: 3,
             min_ngram: 1,
             num_speculative: 4,
+            accept_threshold: 0.3,
         }
     }
 }
@@ -40,8 +48,8 @@ impl std::fmt::Display for NgramConfig {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "K={}, max_ngram={}",
-            self.num_speculative, self.max_ngram
+            "K={}, max_ngram={}, accept_threshold={}",
+            self.num_speculative, self.max_ngram, self.accept_threshold
         )
     }
 }
@@ -54,6 +62,7 @@ impl NgramConfig {
     ///
     /// * `OPENINFER_QWEN3_NGRAM_TOKENS` = draft count `K`.
     /// * `OPENINFER_QWEN3_NGRAM_MAX_NGRAM` = longest suffix to match.
+    /// * `OPENINFER_QWEN3_NGRAM_ACCEPT_THRESHOLD` = gate threshold (`0` disables).
     #[must_use]
     pub fn from_env() -> Self {
         let defaults = Self::default();
@@ -62,6 +71,8 @@ impl NgramConfig {
             min_ngram: defaults.min_ngram,
             num_speculative: env_usize("OPENINFER_QWEN3_NGRAM_TOKENS")
                 .unwrap_or(defaults.num_speculative),
+            accept_threshold: env_f32("OPENINFER_QWEN3_NGRAM_ACCEPT_THRESHOLD")
+                .unwrap_or(defaults.accept_threshold),
         }
     }
 }
@@ -70,6 +81,10 @@ fn env_usize(name: &str) -> Option<usize> {
     std::env::var(name).ok().and_then(|v| v.parse().ok())
 }
 
+fn env_f32(name: &str) -> Option<f32> {
+    std::env::var(name).ok().and_then(|v| v.parse().ok())
+}
+
 /// Stateless n-gram / prompt-lookup proposer.
 ///
 /// The proposer keeps no history of its own; each call scans the supplied
@@ -91,11 +106,17 @@ impl NgramProposer {
             config: NgramConfig {
                 max_ngram,
                 min_ngram,
-                num_speculative: config.num_speculative,
+                ..config
             },
         }
     }
 
+    /// The (clamped) configuration this proposer was built with.
+    #[must_use]
+    pub(crate) fn config(&self) -> NgramConfig {
+        self.config
+    }
+
     /// Propose up to `num_speculative` continuation tokens for `context`
     /// (the full token sequence so far: prompt followed by generated tokens).
     ///
@@ -145,6 +166,75 @@ fn latest_earlier_match(context: &[u32], suffix: &[u32]) -> Option<usize> {
     (0..len - n).rev().find(|&i| &context[i..i + n] == suffix)
 }
 
+/// Engine-wide switch that decides whether n-gram speculation is currently
+/// paying for itself.
+///
+/// Speculation is only a win when enough drafted tokens are accepted to offset
+/// the verify forward (which costs ~`K + 1`× a plain decode). On repetitive text
+/// acceptance is high and this stays open; on prose it collapses to near zero, so
+/// the gate closes and decoding falls back to plain — recovering the throughput
+/// the wasted verify would otherwise burn, which matters most at the higher batch
+/// sizes where the GPU has no spare compute to hide it.
+///
+/// The estimate is a single EWMA of accepted draft tokens per step, updated only
+/// when we actually drafted. While closed the gate **probes** once every
+/// `PROBE_INTERVAL` steps so a shift into repetitive text re-opens it. An initial
+/// warmup keeps it open until there is enough data to judge.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct NgramGate {
+    /// EWMA of accepted draft tokens per drafted step.
+    accept_ewma: f32,
+    /// Drafted steps observed so far (capped at `WARMUP_STEPS`).
+    warmup_left: u32,
+    /// Steps elapsed since the gate last allowed a draft (drives probing).
+    since_draft: u32,
+}
+
+impl NgramGate {
+    /// Smoothing for the acceptance EWMA: low enough to react within a handful
+    /// of steps when a request switches between repetitive and prose regions.
+    const EWMA_ALPHA: f32 = 0.2;
+    /// Always speculate for this many drafted steps before trusting the EWMA.
+    const WARMUP_STEPS: u32 = 8;
+    /// While closed, allow one probing draft every this many steps.
+    const PROBE_INTERVAL: u32 = 32;
+
+    #[must_use]
+    pub(crate) fn new() -> Self {
+        Self {
+            accept_ewma: 0.0,
+            warmup_left: Self::WARMUP_STEPS,
+            since_draft: 0,
+        }
+    }
+
+    /// Whether this step should draft. Open during warmup, while recent
+    /// acceptance clears the threshold, or on a periodic probe. A non-positive
+    /// threshold disables gating (always open).
+    #[must_use]
+    pub(crate) fn should_draft(&self, threshold: f32) -> bool {
+        threshold <= 0.0
+            || self.warmup_left > 0
+            || self.accept_ewma >= threshold
+            || self.since_draft >= Self::PROBE_INTERVAL
+    }
+
+    /// Record the outcome of a step that drafted: `accepted` is the mean number
+    /// of draft tokens verify took across the step's drafted requests. Resets the
+    /// probe clock and folds the count into the EWMA.
+    pub(crate) fn record_drafted(&mut self, accepted: f32) {
+        self.since_draft = 0;
+        self.warmup_left = self.warmup_left.saturating_sub(1);
+        self.accept_ewma += Self::EWMA_ALPHA * (accepted - self.accept_ewma);
+    }
+
+    /// Record a step that did not draft (gate closed, or no match found), so the
+    /// probe clock advances toward the next re-check.
+    pub(crate) fn record_skipped(&mut self) {
+        self.since_draft = self.since_draft.saturating_add(1);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -154,6 +244,7 @@ mod tests {
             max_ngram,
             min_ngram,
             num_speculative: k,
+            ..NgramConfig::default()
         })
     }
 
@@ -216,10 +307,59 @@ mod tests {
             max_ngram: 0,
             min_ngram: 5,
             num_speculative: 2,
+            ..NgramConfig::default()
         });
         // With max_ngram clamped to 1, a single-token recurrence still works:
         // suffix [3] recurs at index 0, so the follower [1, 3] is proposed.
         let ctx = [3u32, 1, 3];
         assert_eq!(p.propose(&ctx), vec![1, 3]);
     }
+
+    #[test]
+    fn gate_disabled_with_nonpositive_threshold() {
+        let mut gate = NgramGate::new();
+        // Drive acceptance to zero; a zero threshold must keep speculating.
+        for _ in 0..50 {
+            gate.record_drafted(0.0);
+        }
+        assert!(gate.should_draft(0.0));
+    }
+
+    #[test]
+    fn gate_closes_after_warmup_on_low_acceptance() {
+        let mut gate = NgramGate::new();
+        // Warmup keeps it open even while acceptance reads zero.
+        for _ in 0..NgramGate::WARMUP_STEPS {
+            assert!(gate.should_draft(0.3));
+            gate.record_drafted(0.0);
+        }
+        // Warmup spent + EWMA below threshold -> closed.
+        assert!(!gate.should_draft(0.3));
+    }
+
+    #[test]
+    fn gate_stays_open_on_high_acceptance() {
+        let mut gate = NgramGate::new();
+        for _ in 0..20 {
+            gate.record_drafted(3.0);
+        }
+        assert!(gate.should_draft(0.3));
+    }
+
+    #[test]
+    fn gate_probes_after_cooldown_then_recovers() {
+        let mut gate = NgramGate::new();
+        for _ in 0..NgramGate::WARMUP_STEPS {
+            gate.record_drafted(0.0);
+        }
+        assert!(!gate.should_draft(0.3), "closed after low-acceptance warmup");
+        // Skipping for PROBE_INTERVAL steps re-opens it for one probe.
+        for _ in 0..NgramGate::PROBE_INTERVAL {
+            gate.record_skipped();
+        }
+        assert!(gate.should_draft(0.3), "probe re-opens the gate");
+        // A probe that finds repetitive text (high acceptance) keeps it open.
+        gate.record_drafted(4.0);
+        assert!(gate.should_draft(0.3));
+    }
 }

From 9a955c1981a58c9878414ab0fe3732e37dc60187 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Mon, 29 Jun 2026 11:15:59 +0800
Subject: [PATCH 29/31] style(qwen3): rustfmt the n-gram acceptance gate

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/executor.rs      | 4 +++-
 openinfer-qwen3-4b/src/executor/spec.rs | 6 +-----
 openinfer-qwen3-4b/src/ngram.rs         | 5 ++++-
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 1f123611..0f91435f 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -2061,7 +2061,9 @@ impl ModelExecutor for Qwen3Executor {
             // has a length-1 verify span and the scheduler routes the whole batch
             // to plain decode — no verify forward, no regression. The gate probes
             // periodically so a shift into repetitive text re-opens it.
-            let drafting = self.ngram_gate.should_draft(ngram.config().accept_threshold);
+            let drafting = self
+                .ngram_gate
+                .should_draft(ngram.config().accept_threshold);
             let mut any_drafted = false;
             let mut requests = Vec::with_capacity(plan.requests.len());
             for item in plan.requests {
diff --git a/openinfer-qwen3-4b/src/executor/spec.rs b/openinfer-qwen3-4b/src/executor/spec.rs
index c074265d..a64ca011 100644
--- a/openinfer-qwen3-4b/src/executor/spec.rs
+++ b/openinfer-qwen3-4b/src/executor/spec.rs
@@ -150,11 +150,7 @@ impl Qwen3Executor {
             // was open, so fold its mean accepted-draft-tokens into the estimate.
             // `result.requests` is the drafted subset, so it is non-empty here.
             if !result.requests.is_empty() {
-                let accepted: usize = result
-                    .requests
-                    .iter()
-                    .map(|r| r.matched_draft_tokens)
-                    .sum();
+                let accepted: usize = result.requests.iter().map(|r| r.matched_draft_tokens).sum();
                 let mean = accepted as f32 / result.requests.len() as f32;
                 self.ngram_gate.record_drafted(mean);
             }
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index f86e585f..878dd9b7 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -352,7 +352,10 @@ mod tests {
         for _ in 0..NgramGate::WARMUP_STEPS {
             gate.record_drafted(0.0);
         }
-        assert!(!gate.should_draft(0.3), "closed after low-acceptance warmup");
+        assert!(
+            !gate.should_draft(0.3),
+            "closed after low-acceptance warmup"
+        );
         // Skipping for PROBE_INTERVAL steps re-opens it for one probe.
         for _ in 0..NgramGate::PROBE_INTERVAL {
             gate.record_skipped();

From 3ac292c0ed5b8ab071832d67f448bf8693953c38 Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Mon, 29 Jun 2026 22:45:49 +0800
Subject: [PATCH 30/31] refactor(qwen3): consolidate n-gram state into
 NgramRuntime

The n-gram lifecycle was spread across executor.rs: a context HashMap field,
an acceptance-gate field, and manual maintenance of both in every execution
path (prefill seed, decode/unified/overlap append, verify append + gate
record, retire drop, and the draft step's gate decision).

Move all of it behind crate::ngram::NgramRuntime, which owns the proposer, the
gate, and the per-request context map, and exposes intent-level methods:
seed / append_committed / drop_request / is_ready / propose_step /
record_verified. SpeculativeMethod::Ngram now holds the runtime, so the two
executor fields (ngram_ctx, ngram_gate) and the per-path bookkeeping collapse
to single calls through ngram_runtime()/ngram_runtime_mut(). No behaviour
change: unit tests 12/12, losslessness gate 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 openinfer-qwen3-4b/src/executor.rs      | 113 +++++++-----------------
 openinfer-qwen3-4b/src/executor/spec.rs |  16 ++--
 openinfer-qwen3-4b/src/ngram.rs         | 111 +++++++++++++++++++++++
 3 files changed, 150 insertions(+), 90 deletions(-)

diff --git a/openinfer-qwen3-4b/src/executor.rs b/openinfer-qwen3-4b/src/executor.rs
index 0f91435f..b1132081 100644
--- a/openinfer-qwen3-4b/src/executor.rs
+++ b/openinfer-qwen3-4b/src/executor.rs
@@ -804,8 +804,9 @@ struct Qwen3ExecutorMetadata {
 enum SpeculativeMethod {
     /// DFlash draft model: worker-side draft forward with hidden-state capture.
     Dflash(DFlashMeta),
-    /// Host-side n-gram / prompt-lookup proposer; needs no draft model.
-    Ngram(crate::ngram::NgramProposer),
+    /// Host-side n-gram / prompt-lookup proposer; needs no draft model. The
+    /// runtime owns the proposer, the acceptance gate, and per-request context.
+    Ngram(crate::ngram::NgramRuntime),
 }
 
 pub struct Qwen3Executor {
@@ -851,16 +852,6 @@ pub struct Qwen3Executor {
     /// enters this set when its prompt finishes prefilling with captured target
     /// context, and leaves on retire or a plain (non-speculative) decode.
     dflash_ready_requests: HashSet<RequestId>,
-    /// Per-request running token context (prompt + every committed token),
-    /// maintained only under the n-gram method. The proposer scans this for
-    /// prompt-lookup matches; it always ends with the request's current dangling
-    /// token so the next draft continues from it.
-    ngram_ctx: HashMap<RequestId, Vec<u32>>,
-    /// Engine-wide acceptance gate for the n-gram method. Closes speculation when
-    /// recent draft acceptance is too low to pay for the verify forward (e.g.
-    /// prose), so the batch falls back to plain decode instead of regressing —
-    /// most visibly at higher concurrency. Idle/`None` under DFlash.
-    ngram_gate: crate::ngram::NgramGate,
     /// Opt-in KV block-event feed for a cache-aware router (`Some` only when the
     /// engine was built with events on — single-GPU, no LoRA). `None` on the
     /// plain path, where the whole feed costs nothing.
@@ -982,8 +973,6 @@ impl Qwen3Executor {
             async_prefill: None,
             speculative_method: None,
             dflash_ready_requests: HashSet::new(),
-            ngram_ctx: HashMap::new(),
-            ngram_gate: crate::ngram::NgramGate::new(),
             kv_events,
         })
     }
@@ -1213,8 +1202,6 @@ impl Qwen3Executor {
             async_prefill: None,
             speculative_method: None,
             dflash_ready_requests: HashSet::new(),
-            ngram_ctx: HashMap::new(),
-            ngram_gate: crate::ngram::NgramGate::new(),
             // KV events are single-GPU only (asserted above); never wired here.
             kv_events: None,
         })
@@ -1306,10 +1293,18 @@ impl Qwen3Executor {
         }
     }
 
-    /// The loaded n-gram proposer, or `None` under no/other method.
-    fn ngram_proposer(&self) -> Option<crate::ngram::NgramProposer> {
+    /// The n-gram runtime (proposer + gate + per-request context), or `None`
+    /// under no/other method.
+    fn ngram_runtime(&self) -> Option<&crate::ngram::NgramRuntime> {
         match &self.speculative_method {
-            Some(SpeculativeMethod::Ngram(proposer)) => Some(*proposer),
+            Some(SpeculativeMethod::Ngram(runtime)) => Some(runtime),
+            _ => None,
+        }
+    }
+
+    fn ngram_runtime_mut(&mut self) -> Option<&mut crate::ngram::NgramRuntime> {
+        match &mut self.speculative_method {
+            Some(SpeculativeMethod::Ngram(runtime)) => Some(runtime),
             _ => None,
         }
     }
@@ -1362,7 +1357,7 @@ impl Qwen3Executor {
         );
         log::info!("Qwen3 n-gram speculative decoding enabled: {config}");
         self.prefix_cache_enabled = false;
-        self.speculative_method = Some(SpeculativeMethod::Ngram(crate::ngram::NgramProposer::new(
+        self.speculative_method = Some(SpeculativeMethod::Ngram(crate::ngram::NgramRuntime::new(
             config,
         )));
         Ok(())
@@ -1743,7 +1738,9 @@ impl ModelExecutor for Qwen3Executor {
             }
         }
         self.saved_cursor.remove(&request_id);
-        self.ngram_ctx.remove(&request_id);
+        if let Some(runtime) = self.ngram_runtime_mut() {
+            runtime.drop_request(request_id);
+        }
         if self.dflash_meta().is_some() {
             self.dflash_ready_requests.remove(&request_id);
             self.primary.drop_dflash_request(request_id)?;
@@ -1935,7 +1932,7 @@ impl ModelExecutor for Qwen3Executor {
         // the full prompt followed by the first sampled token (the request's
         // current dangling token). Later committed tokens are appended at verify
         // and decode time so the context always ends with the dangling token.
-        if self.ngram_proposer().is_some() {
+        if let Some(runtime) = self.ngram_runtime_mut() {
             for req_result in &result.requests {
                 if req_result.completed {
                     if let Some(req) = plan
@@ -1946,7 +1943,7 @@ impl ModelExecutor for Qwen3Executor {
                         let mut context = Vec::with_capacity(req.prompt_tokens.len() + 1);
                         context.extend_from_slice(&req.prompt_tokens);
                         context.push(req_result.first_token);
-                        self.ngram_ctx.insert(req_result.request_id, context);
+                        runtime.seed(req_result.request_id, context);
                     }
                 }
             }
@@ -2027,11 +2024,9 @@ impl ModelExecutor for Qwen3Executor {
         }
         // A plain decode commits one token per request; keep the n-gram context
         // in sync so it always ends with the request's dangling token.
-        if self.ngram_proposer().is_some() {
+        if let Some(runtime) = self.ngram_runtime_mut() {
             for req_result in &result.requests {
-                if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
-                    context.push(req_result.token);
-                }
+                runtime.append_committed(req_result.request_id, &[req_result.token]);
             }
         }
         // A plain decode advances the sequence outside the speculative path, so
@@ -2053,49 +2048,11 @@ impl ModelExecutor for Qwen3Executor {
 
     fn execute_speculative_draft(&mut self, plan: DraftPlan<'_>) -> Result<DraftResult> {
         // N-gram drafts are produced host-side on the executor thread (no worker
-        // forward): scan each request's own token context for a prompt-lookup
-        // match. DFlash instead runs a draft-model forward on the lane.
-        if let Some(ngram) = self.ngram_proposer() {
-            // One gate decision per step. When closed (recent acceptance too low
-            // to pay for the verify forward), propose nothing: every request then
-            // has a length-1 verify span and the scheduler routes the whole batch
-            // to plain decode — no verify forward, no regression. The gate probes
-            // periodically so a shift into repetitive text re-opens it.
-            let drafting = self
-                .ngram_gate
-                .should_draft(ngram.config().accept_threshold);
-            let mut any_drafted = false;
-            let mut requests = Vec::with_capacity(plan.requests.len());
-            for item in plan.requests {
-                anyhow::ensure!(
-                    item.params.is_greedy(),
-                    "n-gram speculative draft currently supports greedy sampling only"
-                );
-                let drafts = if drafting {
-                    let context = self.ngram_ctx.get(&item.request_id).ok_or_else(|| {
-                        anyhow::anyhow!("missing n-gram context for {:?}", item.request_id)
-                    })?;
-                    ngram.propose(context)
-                } else {
-                    Vec::new()
-                };
-                any_drafted |= !drafts.is_empty();
-                // Verify span = [current dangling token, draft_1, …, draft_K].
-                let mut token_ids = Vec::with_capacity(1 + drafts.len());
-                token_ids.push(item.current_token);
-                token_ids.extend(drafts);
-                requests.push(crate::speculative::DraftRequestResult {
-                    request_id: item.request_id,
-                    token_ids,
-                });
-            }
-            // No draft produced (gate closed, or open but every request missed)
-            // means no verify forward runs this step, so advance the gate's probe
-            // clock toward the next re-check. A step that did draft updates the
-            // acceptance estimate from its verify result instead (see spec.rs).
-            if !any_drafted {
-                self.ngram_gate.record_skipped();
-            }
+        // forward): the runtime scans each request's own token context for a
+        // prompt-lookup match, gated by recent acceptance. DFlash instead runs a
+        // draft-model forward on the lane.
+        if let Some(runtime) = self.ngram_runtime_mut() {
+            let requests = runtime.propose_step(plan.requests)?;
             return Ok(DraftResult { requests });
         }
         self.execute_speculative_draft_impl(plan)
@@ -2110,10 +2067,10 @@ impl ModelExecutor for Qwen3Executor {
     }
 
     fn speculative_request_ready(&self, request_id: RequestId) -> bool {
-        if self.ngram_proposer().is_some() {
+        if let Some(runtime) = self.ngram_runtime() {
             // Ready as soon as the prompt is prefilled and its context seeded;
             // n-gram needs no captured hidden state.
-            self.ngram_ctx.contains_key(&request_id)
+            runtime.is_ready(request_id)
         } else {
             self.dflash_ready_requests.contains(&request_id)
         }
@@ -2191,11 +2148,9 @@ impl ModelExecutor for Qwen3Executor {
                 // the batch returns to pure-greedy n-gram speculation. Reachable
                 // whenever a non-greedy request shares the batch and pushes the
                 // greedy ones onto the fused decode path.
-                if self.ngram_proposer().is_some() {
+                if let Some(runtime) = self.ngram_runtime_mut() {
                     for req_result in &result.decode_requests {
-                        if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
-                            context.push(req_result.token);
-                        }
+                        runtime.append_committed(req_result.request_id, &[req_result.token]);
                     }
                 }
                 // A plain decode via the fused unified step advances the sequence
@@ -2235,11 +2190,9 @@ impl ModelExecutor for Qwen3Executor {
                 // overlap path is unreachable under n-gram today (speculation forces
                 // decode-overlap off), but keep the context anchored if that
                 // mutual-exclusion is ever relaxed.
-                if self.ngram_proposer().is_some() {
+                if let Some(runtime) = self.ngram_runtime_mut() {
                     for req_result in &decode_result.requests {
-                        if let Some(context) = self.ngram_ctx.get_mut(&req_result.request_id) {
-                            context.push(req_result.token);
-                        }
+                        runtime.append_committed(req_result.request_id, &[req_result.token]);
                     }
                 }
                 for req_result in &decode_result.requests {
diff --git a/openinfer-qwen3-4b/src/executor/spec.rs b/openinfer-qwen3-4b/src/executor/spec.rs
index a64ca011..9c469f3a 100644
--- a/openinfer-qwen3-4b/src/executor/spec.rs
+++ b/openinfer-qwen3-4b/src/executor/spec.rs
@@ -139,20 +139,16 @@ impl Qwen3Executor {
         // Keep the n-gram running context in sync: append every committed token
         // so the next draft scans the full history and continues from the new
         // dangling token. DFlash keeps its own per-request hidden state instead.
-        if self.ngram_proposer().is_some() {
+        // This verify forward ran because the gate was open, so also fold its
+        // mean accepted-draft-tokens into the gate (`result.requests` is the
+        // drafted subset, hence non-empty here).
+        if let Some(runtime) = self.ngram_runtime_mut() {
             for req_result in &result.requests {
-                self.ngram_ctx
-                    .entry(req_result.request_id)
-                    .or_default()
-                    .extend_from_slice(&req_result.accepted_tokens);
+                runtime.append_committed(req_result.request_id, &req_result.accepted_tokens);
             }
-            // Feed the acceptance gate: this verify forward ran because the gate
-            // was open, so fold its mean accepted-draft-tokens into the estimate.
-            // `result.requests` is the drafted subset, so it is non-empty here.
             if !result.requests.is_empty() {
                 let accepted: usize = result.requests.iter().map(|r| r.matched_draft_tokens).sum();
-                let mean = accepted as f32 / result.requests.len() as f32;
-                self.ngram_gate.record_drafted(mean);
+                runtime.record_verified(accepted as f32 / result.requests.len() as f32);
             }
         }
 
diff --git a/openinfer-qwen3-4b/src/ngram.rs b/openinfer-qwen3-4b/src/ngram.rs
index 878dd9b7..45487927 100644
--- a/openinfer-qwen3-4b/src/ngram.rs
+++ b/openinfer-qwen3-4b/src/ngram.rs
@@ -14,6 +14,13 @@
 //! handled by the decode path and are intentionally out of scope for this
 //! module so the lookup logic can be unit-tested in isolation.
 
+use std::collections::HashMap;
+
+use anyhow::Result;
+
+use crate::executor::RequestId;
+use crate::speculative::{DraftRequestResult, DraftStepItem};
+
 /// Configuration for [`NgramProposer`].
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub struct NgramConfig {
@@ -235,6 +242,110 @@ impl NgramGate {
     }
 }
 
+/// Owns all per-engine n-gram speculative state: the stateless [`NgramProposer`],
+/// the [`NgramGate`], and each request's running token context (prompt + every
+/// committed token, always ending on the dangling token the next draft continues
+/// from). This is the single home for the method's state so the executor drives
+/// it through intent-level calls (`seed` / `append_committed` / `drop_request` /
+/// `propose_step` / `record_verified`) instead of threading three fields through
+/// every execution path. Held inside [`crate::executor::SpeculativeMethod::Ngram`].
+pub(crate) struct NgramRuntime {
+    proposer: NgramProposer,
+    gate: NgramGate,
+    /// Per-request running context. Present once a request's prompt is prefilled
+    /// (see [`Self::seed`]); the proposer scans it for prompt-lookup matches.
+    ctx: HashMap<RequestId, Vec<u32>>,
+}
+
+impl NgramRuntime {
+    #[must_use]
+    pub(crate) fn new(config: NgramConfig) -> Self {
+        Self {
+            proposer: NgramProposer::new(config),
+            gate: NgramGate::new(),
+            ctx: HashMap::new(),
+        }
+    }
+
+    /// Seed a freshly-prefilled request's context (`prompt + first_token`). Later
+    /// commits extend it via [`Self::append_committed`].
+    pub(crate) fn seed(&mut self, request_id: RequestId, context: Vec<u32>) {
+        self.ctx.insert(request_id, context);
+    }
+
+    /// Append committed tokens to a request's context, keeping it anchored on the
+    /// new dangling token. No-op for an unseeded request (defensive).
+    pub(crate) fn append_committed(&mut self, request_id: RequestId, tokens: &[u32]) {
+        if let Some(context) = self.ctx.get_mut(&request_id) {
+            context.extend_from_slice(tokens);
+        }
+    }
+
+    /// Forget a retired request's context.
+    pub(crate) fn drop_request(&mut self, request_id: RequestId) {
+        self.ctx.remove(&request_id);
+    }
+
+    /// Whether a request can speculate: its prompt is prefilled and context
+    /// seeded (n-gram keeps no captured hidden state, so that is all it needs).
+    #[must_use]
+    pub(crate) fn is_ready(&self, request_id: RequestId) -> bool {
+        self.ctx.contains_key(&request_id)
+    }
+
+    /// Build the verify spans for one speculative step. The gate is consulted
+    /// once: when closed, no request drafts, so every span is just
+    /// `[current_token]` and the scheduler routes the whole batch to plain decode.
+    /// Advances the gate's probe clock when the step produced no draft.
+    pub(crate) fn propose_step(
+        &mut self,
+        items: &[DraftStepItem],
+    ) -> Result<Vec<DraftRequestResult>> {
+        let drafting = self
+            .gate
+            .should_draft(self.proposer.config().accept_threshold);
+        let mut any_drafted = false;
+        let mut results = Vec::with_capacity(items.len());
+        for item in items {
+            anyhow::ensure!(
+                item.params.is_greedy(),
+                "n-gram speculative draft currently supports greedy sampling only"
+            );
+            let drafts = if drafting {
+                let context = self.ctx.get(&item.request_id).ok_or_else(|| {
+                    anyhow::anyhow!("missing n-gram context for {:?}", item.request_id)
+                })?;
+                self.proposer.propose(context)
+            } else {
+                Vec::new()
+            };
+            any_drafted |= !drafts.is_empty();
+            // Verify span = [current dangling token, draft_1, …, draft_K].
+            let mut token_ids = Vec::with_capacity(1 + drafts.len());
+            token_ids.push(item.current_token);
+            token_ids.extend(drafts);
+            results.push(DraftRequestResult {
+                request_id: item.request_id,
+                token_ids,
+            });
+        }
+        // No draft produced (gate closed, or open but every request missed) means
+        // no verify forward runs this step, so advance the probe clock. A step
+        // that drafted updates the acceptance estimate via `record_verified`.
+        if !any_drafted {
+            self.gate.record_skipped();
+        }
+        Ok(results)
+    }
+
+    /// Fold a verify forward's mean accepted-draft-tokens into the gate. Called
+    /// only when a verify ran, i.e. the gate was open and at least one request
+    /// drafted.
+    pub(crate) fn record_verified(&mut self, mean_accepted: f32) {
+        self.gate.record_drafted(mean_accepted);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From ebfac0ae318f11f0c635af958e08b7759942516b Mon Sep 17 00:00:00 2001
From: wjinxu <1299461899@qq.com>
Date: Mon, 29 Jun 2026 22:45:49 +0800
Subject: [PATCH 31/31] docs(qwen3): refresh n-gram doc to match the shipped
 code path

The doc still described an earlier version of the branch: a SpeculativeProposer
trait, scheduler/speculative.rs::speculative_decode_step, OPENINFER_QWEN3_SPEC,
tests/ngram_speculative.rs, and a standalone verify_forward. Rewrite it to the
actual path: --ngram-speculative, the SpeculativeMethod{ Dflash, Ngram } enum,
NgramRuntime owning proposer+gate+context, host-side propose, and the shared
verify/accept/KV transaction. Keeps the serving A/B, acceptance-gate, and
single-fused-forward follow-up sections.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 docs/models/qwen3/ngram-speculative.md | 350 ++++++++++---------------
 1 file changed, 145 insertions(+), 205 deletions(-)

diff --git a/docs/models/qwen3/ngram-speculative.md b/docs/models/qwen3/ngram-speculative.md
index 66f7765d..7ac4eebc 100644
--- a/docs/models/qwen3/ngram-speculative.md
+++ b/docs/models/qwen3/ngram-speculative.md
@@ -1,185 +1,132 @@
 # Qwen3-4B n-gram speculative decoding (design)
 
 **TL;DR**: Draft-model-free speculative decoding for Qwen3-4B. An n-gram
-(prompt-lookup) proposer suggests `K` continuation tokens from the running
-context; the target model verifies them in one forward pass and commits the
-longest greedy-agreed prefix plus one model token. Greedy verification is
-**lossless** — output is bit-for-bit identical to plain greedy decode, just
-fewer forward passes on repetitive / structured text (code, quoting, JSON).
-The KV layer (kvbm) already supports speculative scheduling natively, so
-rejected drafts need no manual rollback. An **acceptance gate** (`NgramGate`)
-closes speculation when recent draft acceptance is too low to pay for the verify
-forward, so low-acceptance traffic (prose) falls back to plain decode instead of
-regressing — see *Serving A/B* below.
-
-Last touched: 2026-06 · Enabled via `--ngram-speculative`; proposer, greedy
-acceptance, KV speculative pass-throughs, the verify forward, scheduler wiring,
-and the acceptance gate are all landed and GPU-validated lossless.
-
-## Why this is cheap to add here
-
-Three of the four pieces already exist or are trivial:
-
-- **Proposer** — `openinfer-qwen3-4b/src/ngram.rs` (`NgramProposer`). Done, unit-tested.
-- **Acceptance** — `openinfer-qwen3-4b/src/speculative.rs` (`accept_greedy`,
-  `SpeculativeConfig`). Done, unit-tested.
-- **KV scheduling / rollback** — kvbm's `SchedulableSequence` already implements
-  `schedule_speculative` / `apply_speculative` with **automatic LIFO release of
-  the blocks pre-allocated for rejected drafts**. Exposed on `RequestKv`
-  (`openinfer-kv-cache/src/pool.rs`). Done, unit-tested.
-- **GPU verify forward + orchestration** — the only remaining work (below).
-
-## Per-step data flow (single request, greedy)
-
-Layering mirrors vLLM V1 (proposer in the runner/scheduler layer that owns
-token history; the executor reserves KV, runs the target forward, and accepts):
-
-```
-scheduler owns the request's token history (prompt + generated)
-  1. drafts = NgramProposer.propose(history)        # K candidates; empty -> plain decode
-  2. verify_inputs = [d0, c0, .., c_{K-1}]           # d0 = last committed token (dangling)
-  3. RequestKv.schedule_speculative(K + 1)           # room for drafts + bonus token
-  4. argmax[0..K] = verify_forward(verify_inputs, prefill_view(1 + K))
-  5. committed = accept_greedy(drafts, argmax)        # m accepted drafts + 1 model token
-  6. RequestKv.apply_speculative(committed)           # kvbm releases rejected blocks
-  7. scheduler appends committed to history; applies stop / max_tokens;
-     committed.last() becomes the next step's d0
+(prompt-lookup) proposer suggests up to `K` continuation tokens from the
+request's running context; the target model verifies them in one forward pass
+and commits the longest greedy-agreed prefix plus one model token. Greedy
+verification is **lossless** — output is bit-for-bit identical to plain greedy
+decode, just fewer forward passes on repetitive / structured text. It is a second
+*propose* implementation on the shared speculative core (#436/#442): verify,
+greedy acceptance, and the KV transaction are reused unchanged; only *propose*
+differs. An **acceptance gate** closes speculation when recent draft acceptance
+is too low to pay for the verify forward, so low-acceptance traffic (prose) falls
+back to plain decode instead of regressing — see *Serving A/B*.
+
+Last touched: 2026-06 · Enabled with `--ngram-speculative` (off by default).
+Proposer, host-side propose, the shared verify/accept/KV transaction, scheduler
+wiring, and the acceptance gate are landed and GPU-validated lossless.
+
+## Shape: one transaction, only *propose* varies
+
+Speculative decoding is a single optimistic transaction; the only method-specific
+part is where the drafts come from. The loaded method is one enum on the
+executor (`openinfer-qwen3-4b/src/executor.rs`):
+
+```rust
+enum SpeculativeMethod {
+    Dflash(DFlashMeta),     // worker-side draft-model forward, captures hidden state
+    Ngram(NgramRuntime),    // host-side prompt-lookup, no draft model
+}
+// Qwen3Executor::speculative_method: Option<SpeculativeMethod>  — single source of truth
 ```
 
-### Token / KV accounting (matches kvbm's verified contract)
-
-- The verify forward computes KV for `1 + K` positions (`d0` + `K` drafts) at
-  `base_pos = kv_position`. Structurally this is a prefill of `1 + K` tokens at
-  the current position, so the `KvView` is `RequestKv::prefill_view(1 + K)` and
-  the forward reuses the existing paged-prefill attention path.
-- `argmax[i]` is the model's greedy token *after* consuming `verify_inputs[i]`:
-  `argmax[0]` follows `d0` (the true next token), `argmax[i]` follows `c_{i-1}`,
-  `argmax[K]` is the bonus continuation. This index convention is exactly what
-  `accept_greedy(proposed = drafts, target_argmax = argmax)` expects.
-- `apply_speculative(committed)` advances `kv_position` by `committed.len()`
-  (`m + 1`); kvbm LIFO-drops the over-allocated blocks. `committed.last()` (the
-  model token) becomes the new dangling token. `schedule_speculative(K + 1)`
-  guarantees `m + 1 <= K + 1`.
-
-### Why it is lossless
-
-`accept_greedy` only keeps the prefix where `draft[i] == argmax[i]` and then
-appends one of the model's own tokens, so the committed sequence is identical
-to what plain greedy decode would have produced one token at a time. This gives
-a free correctness oracle: **speculative-on must equal speculative-off** for the
-same prompt under greedy params.
-
-## Landed: GPU verify forward + executor step
-
-- **`LocalQwen3Lane::execute_verify(verify_tokens, kv_view, lora)`** — reuses
-  `batch_prefill(echo = true)` for per-position logits over the existing paged
-  KV, then a GPU argmax (`argmax_batch_bf16_into`) returns one token per verify
-  position. Only the position ids cross to host, never the `[vocab, n]` logits
-  (vLLM-style). Kept additive: `batch_prefill` / `batch_decode` are untouched.
-- **`StepCommand::SpeculativeVerify` + `Qwen3Executor::execute_speculative`** —
-  `schedule_speculative(K + 1)` → `prefill_view(1 + K)` → verify step → argmax →
-  `accept_greedy` → `apply_speculative`, returning the committed tokens. The
-  rank-worker channel carries it (TP-safe).
-
-## Validated
-
-- **Lossless on GPU** — `tests/ngram_speculative.rs` runs real Qwen3-4B and
-  asserts greedy n-gram speculative decode is token-identical to plain greedy
-  decode (prefix cache off; repetitive prompt). Confirms the full pipeline
-  (proposer → schedule_speculative → verify forward → accept_greedy →
-  apply_speculative) end-to-end.
-
-## Landed: scheduler serving-loop wiring
-
-- `ActiveRequestState` carries `token_history` (prompt + generated), maintained
-  on promote and each committed decode token.
-- `scheduler/speculative.rs::speculative_decode_step` proposes per active
-  request, runs `execute_speculative` (or a single decode when no draft), and
-  streams the committed tokens with stop / max-token handling — isolated from
-  the one-token-per-step plan/resolve/effects pipeline. `scheduler_loop` routes
-  pure-decode ticks through it when `SpeculativeConfig.enabled`.
-- Mock-tested (`FakeExecutor::execute_speculative`): streams every committed
-  token + advances state; commits past `max_tokens` truncate, finish, retire.
-
-## Proposer seam (closed-set, n-gram-sized)
-
-The proposer is factored out as the one piece meant to vary between methods:
-
-- `speculative::SpeculativeProposer` — `fn propose(&self, context: &[u32]) -> Vec<u32>`.
-- `SpeculativeConfig.method: SpeculativeMethod` (a *closed* enum, one variant per
-  method) + `build_proposer()` factory. `scheduler_loop` builds one boxed
-  `dyn SpeculativeProposer` at startup; `speculative_decode_step` takes
-  `&dyn SpeculativeProposer`. This is closed-set enum dispatch, not an open
-  plugin system — the idiomatic Rust choice for a small known set.
-
-This is a good **n-gram** seam, not yet a general proposer abstraction. The
-trait fits stateless, token-emitting proposers; a draft-model / EAGLE proposer
-would need a wider trait (`&mut self` + per-request create/drop lifecycle, the
-request id, and returning draft probabilities for rejection sampling) **and**
-changes to the scheduler step and verify path. Concretely, the parts below the
-proposer are **greedy-specific**, not method-agnostic:
-
-- the verify forward returns argmax (part of the greedy acceptance rule; sampling
-  acceptance needs distributions),
-- `accept_greedy` is greedy-only,
-- `speculative_decode_step` assumes a stateless proposer (no per-request
-  create/drop).
-
-Widening these is deferred until a second proposer actually lands, so the shapes
-are validated against a real implementation rather than guessed at now.
+This is the pre-trait consolidation #436 deferred. A full `SpeculativeProposer`
+trait object is **not** introduced: DFlash's propose is a worker RPC (needs
+`&mut executor` + lane), so it can't be a self-contained proposer without borrow
+gymnastics. Enum dispatch is the honest shape until a third method (EAGLE, #325)
+makes a shared trait pay off.
+
+Per step (`scheduler/plan.rs`, `ExecutionPlan::SpeculativeDecode`):
+
+1. **Propose** — produce up to `K` candidate tokens per request.
+2. **Verify** — one target forward over each request's `K + 1` span
+   `[current, draft_1..draft_K]`, argmax at every position.
+3. **Accept** — `accept_greedy` keeps the longest prefix matching the target
+   argmax, then appends the target's own token at the first mismatch (always
+   commits `1..=K + 1` tokens).
+4. **Commit / roll back** — accepted KV committed; the unused draft tail is
+   LIFO-released by kvbm (`RequestKv::apply_speculative`).
+
+Because the draft↔verify boundary is a pure token span, `accept_greedy` /
+`build_verify_results`, the verify transaction
+(`executor/spec.rs::execute_speculative_verify_impl`), and the
+`schedule/apply/revert_speculative` KV lifecycle are **reused unchanged** from
+the DFlash core. Only two things are method-specific: where drafts come from, and
+whether verify captures hidden states.
+
+## What n-gram adds
+
+- **Proposer** (`ngram.rs`, `NgramProposer`) — stateless longest-suffix
+  prompt-lookup over a request's own token history: tries the longest configured
+  suffix first, falls back to shorter. Unit-tested in isolation.
+- **`NgramRuntime`** (`ngram.rs`) — owns the method's entire state: the proposer,
+  the acceptance gate (`NgramGate`), and the per-request running context map. The
+  executor drives it through intent-level calls — `seed`, `append_committed`,
+  `drop_request`, `is_ready`, `propose_step`, `record_verified` — instead of
+  maintaining three parallel fields across every execution path.
+- **Host-side propose** — `Qwen3Executor::execute_speculative_draft` calls
+  `NgramRuntime::propose_step` on the executor thread: no worker forward, no draft
+  model, no readiness handshake. (DFlash instead dispatches
+  `StepCommand::SpeculativeDraft` to the lane.)
+- **No-capture verify** — `LocalQwen3Lane::execute_ngram_verify` reuses
+  `batch_prefill_into` with **zero capture layers** (a token-only proposer keeps
+  no model state to seed), then argmax + `build_verify_results`. The worker routes
+  verify by `lane.dflash.is_some()`; the methods are mutually exclusive.
+- **Running context** (`NgramRuntime.ctx`) — per-request token context, seeded at
+  prefill (`prompt + first_token`), appended at every commit path (verify / plain
+  decode / fused unified decode, plus a defensive overlap branch), dropped on
+  retire. Always ends with the request's dangling token so the next draft
+  continues from it.
 
 ## Enabling it
 
-`scheduler_loop` builds the config via `SpeculativeConfig::from_env()`
-(default-off). The generic switch lives on `SpeculativeConfig`; each method
-parses its own knobs (`NgramConfig::from_env`):
+`--ngram-speculative` (server flag) → `Qwen3LaunchOptions.ngram_speculative` →
+`scheduler::start_qwen3` → `Qwen3Executor::load_ngram_drafter(NgramConfig::from_env())`.
+
+- Single-GPU greedy only; **requires `--tp-size=1`**; mutually exclusive with
+  `--dflash-draft-model-path`, `--enable-lora`, `--kv-offload`, and
+  `--decode-overlap` (validated in `lib.rs::launch` and `server/src/main.rs`).
+- Forces the prefix cache off — the verify forward writes each request's own
+  speculative KV span, so cross-request prefix reuse is unsafe here.
 
-- `OPENINFER_QWEN3_SPEC=1` — turn speculation on (generic).
-- `OPENINFER_QWEN3_NGRAM_TOKENS=K` — draft count (n-gram, default 4).
-- `OPENINFER_QWEN3_NGRAM_MAX_NGRAM=N` — longest matched suffix (n-gram, default 3).
+Proposer knobs (`NgramConfig::from_env`):
+
+- `OPENINFER_QWEN3_NGRAM_TOKENS=K` — draft count (default 4).
+- `OPENINFER_QWEN3_NGRAM_MAX_NGRAM=N` — longest matched suffix (default 3).
 - `OPENINFER_QWEN3_NGRAM_ACCEPT_THRESHOLD=f` — acceptance gate threshold (mean
   accepted draft tokens/step below which speculation falls back to plain decode;
-  default 0.3, `0` disables the gate). See *Serving A/B* below.
-
-Only the non-LoRA `scheduler_loop` reads it; the unified prefill+decode tick
-still uses plain decode.
-
-**Per-request eligibility.** Even with the switch on, only requests that are
-greedy (`SamplingParams::is_greedy()`) **and** ask for no decode logprobs
-(`logprobs == 0`) take the speculative path. Speculation verifies with argmax
-and emits no per-token logprobs, so a sampled request would otherwise be forced
-to argmax and a logprobs request would silently lose them. Any ineligible
-request takes a normal sampled single-token decode (its own params, logprobs,
-and a fresh `random_val`) on that tick, so enabling speculation never changes a
-sampled request's output or strips requested logprobs.
-
-## Measured speedup (best case)
-
-`tests/ngram_speculative.rs::ngram_speculative_speedup` (ignored; needs GPU +
-weights) times greedy vs. speculative on Qwen3-4B (eager, single request, 192
-tokens). On the perfectly periodic synthetic prompt:
-
-| metric            | greedy   | speculative |
-| ----------------- | -------- | ----------- |
-| forward passes    | 191      | 39          |
-| ms / token        | 9.99     | 2.52        |
-| accepted / verify | —        | 5.00 (max with K=4) |
-| wall-clock        | 1908 ms  | 481 ms (**3.96x**) |
-
-This is the ceiling: the prompt is exactly periodic so every draft is accepted.
-Real prompts accept a fraction of drafts, so expect smaller wins; the benchmark
-exists to track acceptance-rate / TPOT as the proposer and verify path evolve.
-Run: `cargo test -p openinfer-qwen3-4b --release --test ngram_speculative \
-ngram_speculative_speedup -- --ignored --nocapture` (`OPENINFER_BENCH_TOKENS`
-overrides the 192-token default).
+  default 0.3, `0` disables the gate). See *Serving A/B*.
+
+**Per-request eligibility.** `should_speculative_decode` takes the speculative
+path only when *every* active request is greedy (`SamplingParams::is_greedy()`),
+asks for no decode logprobs (`logprobs == 0`), and is non-LoRA — a single
+ineligible request falls the whole batch back to plain decode for that step. So
+enabling speculation never changes a sampled request's output or strips requested
+logprobs. A miss (length-1 verify span) already degrades to a plain single-token
+decode in `build_verify_results`, so it needs no special fallback.
+
+## Why it is lossless
+
+`accept_greedy` keeps only the prefix where `draft[i] == target_argmax[i]`, then
+appends one of the target's own tokens, so the committed sequence is exactly what
+plain greedy decode would have produced one token at a time. This is a built-in
+oracle: **spec-on must equal spec-off** token-for-token under greedy params
+(modulo the benign prefill-vs-decode bf16 tie flip). The gate only changes
+*whether* we speculate, never the committed tokens, so it preserves this.
+
+GPU-validated by `tests/ngram_speculative_gate.rs` (3 tests, real Qwen3-4B):
+greedy-matches-plain (bs=1), concurrent heterogeneous (bs>1 verify-span path),
+and mixed greedy+sampling. The proposer and gate also have isolated unit tests in
+`ngram.rs`.
 
 ## Serving A/B (`vllm bench serve`) and the acceptance gate
 
-Synthetic best-case is the ceiling; serving throughput on real datasets is the
-contract. Single RTX 4090, Qwen3-4B, `temperature=0`, `--ignore-eos`,
-`--no-prefix-cache`, 12 prompts × 128 output tokens, sonnet (550/128, prefix 200)
-and random (550/128). `Δ` is n-gram vs plain decode on the **same** engine;
-vLLM (0.21 / 0.23, ngram `K=4`, `prompt_lookup_max=3`) is the reference.
+Greedy speculation can only win when enough drafts are accepted to offset the
+verify forward (which costs ~`K + 1`× a plain decode). Single RTX 4090, Qwen3-4B,
+`temperature=0`, `--ignore-eos`, `--no-prefix-cache`, 12 prompts × 128 output
+tokens, sonnet (550/128, prefix 200) and random (550/128). `Δ` is n-gram vs plain
+decode on the **same** engine; vLLM (0.21 / 0.23, ngram `K=4`,
+`prompt_lookup_max=3`) is the reference.
 
 | dataset / conc | acceptance | vLLM 0.23 Δ | openinfer (no gate) Δ | **openinfer (gate) Δ** |
 | --- | --- | --- | --- | --- |
@@ -201,60 +148,53 @@ Reading this:
   compute-idle, so the gate stays open and the ~33% acceptance is a free win.
 - **random c4 is the open gap**: vLLM **+17%** vs openinfer **−20%** at the *same*
   ~37% acceptance (proven by the matching c1 numbers). This is purely the verify
-  path — see below. The gate can't help here: acceptance is high, so it correctly
+  path — see below. The gate can't help: acceptance is high, so it correctly
   stays open; the loss is in *how* we verify, not *whether* we should.
 
 **Root cause of the random-c4 gap (nsys, `--cuda-graph-trace=node`).** At c4 the
 GPU is already ~92% busy on plain decode. Under n-gram the same 22 s window shows
-the attention running through the **prefill** kernel (`BatchPrefill`, ~23 µs)
-instead of the decode kernel (`BatchDecode`, ~15 µs), ~989 verify forwards **plus**
-~887 decode forwards (the batch splits two ways every mixed step), and ~43% more
-GEMM. vLLM instead runs **one fused forward**: draft tokens are extra rows in the
-same batch (a missed request is just `query_len=1`), verified by one rejection
-step — no second pass, no prefill-kernel penalty, no ragged-span CUDA-graph
-thrash. Folding our verify into that single unified forward is the way to turn
-random-c4 positive; it is tracked as the follow-up below, not in this PR.
+attention running through the **prefill** kernel (`BatchPrefill`, ~23 µs) instead
+of the decode kernel (`BatchDecode`, ~15 µs), ~989 verify forwards **plus** ~887
+decode forwards (the batch splits two ways every mixed step), and ~43% more GEMM.
+vLLM instead runs **one fused forward**: draft tokens are extra rows in the same
+batch (a missed request is just `query_len=1`), verified by one rejection step —
+no second pass, no prefill-kernel penalty, no ragged-span CUDA-graph thrash.
+Folding our verify into that single unified forward is the way to turn random-c4
+positive; tracked as the follow-up below, not in this PR.
 
 ## The acceptance gate (`NgramGate`)
 
-`openinfer-qwen3-4b/src/ngram.rs`. Engine-wide EWMA of accepted draft tokens per
-drafted step. `should_draft` opens while in warmup, while the EWMA clears
-`accept_threshold`, or on a periodic probe; otherwise the proposer returns
-nothing for the step and the existing draft/undraft partition routes the whole
-batch to plain decode (no verify forward). The probe (every `PROBE_INTERVAL`
-steps) re-opens it so a shift into repetitive text is picked back up. Lossless
-either way — gating only changes *whether* we speculate, never the committed
-tokens. Unit-tested (`gate_*` in `ngram.rs`); the engine-level losslessness gate
-(`tests/ngram_speculative_gate.rs`) still passes with the gate live.
+`ngram.rs`, owned by `NgramRuntime`. An engine-wide EWMA of accepted draft tokens
+per drafted step. `should_draft` opens while in warmup, while the EWMA clears
+`accept_threshold`, or on a periodic probe; otherwise `propose_step` produces no
+drafts for the step and the existing draft/undraft partition in `plan.rs` routes
+the whole batch to plain decode (no verify forward). The probe (every
+`PROBE_INTERVAL` steps) re-opens it so a shift into repetitive text is picked back
+up. `record_verified` (called from the verify transaction in `spec.rs`) feeds the
+step's mean acceptance back in. Lossless either way. Unit-tested (`gate_*` in
+`ngram.rs`); the engine-level losslessness gate still passes with the gate live.
 
 ## Remaining work / follow-up
 
 1. **Single fused verify forward** (the random-c4 fix, separate PR): fold the
    verify tokens into one unified batched forward (FlashInfer varlen) with a GPU
    rejection step — a missed request becomes a `query_len=1` row in the same
-   batch — instead of the current `batch_prefill`-based verify plus the second
-   plain-decode pass for undrafted requests. The *Serving A/B* above quantifies
-   the prize: closing the ~37-point random-c4 gap to vLLM (+17% vs −20%).
+   batch — instead of the current `batch_prefill_into`-based verify plus the
+   second plain-decode pass for undrafted requests. The *Serving A/B* above
+   quantifies the prize: closing the ~37-point random-c4 gap to vLLM
+   (+17% vs −20%).
 
 ## Scope / deferred
 
-First cut is **single-request, greedy, non-CUDA-graph**. Deferred: batched
-speculation (ragged verify across requests), sampling (non-greedy) acceptance,
-CUDA-graph-captured verify, interaction with the unified prefill+decode step,
-and pipelined ahead-of-time proposal.
-
-## Open questions for review
-
-1. `verify_forward` as a standalone model method (preferred) vs. reusing
-   `batch_prefill(echo = true)`.
-2. Confirm the GPU-side argmax via `select_batch_tokens_into` over the `K + 1`
-   verify positions is acceptable (vs. a dedicated argmax kernel later).
-3. Proposer placement in the scheduler layer (owns token history), matching
-   vLLM's runner / `request.spec_token_ids` split.
+Greedy, single-GPU, no LoRA. Deferred: sampling (non-greedy) acceptance (needs
+draft distributions + a rejection sampler, not argmax), the unified fused verify
+forward above, and a general `SpeculativeProposer` trait (revisit when EAGLE
+#325 lands a third method).
 
 ## Prior art
 
-- vLLM V1 n-gram / prompt-lookup spec decode (`NgramProposer` in the runner,
-  GPU rejection sampler, scheduler reserves KV for `k` draft tokens).
+- vLLM V1 n-gram / prompt-lookup spec decode (`NgramProposer` in the runner, GPU
+  rejection sampler, scheduler reserves KV for `k` draft tokens). Its single
+  fused forward is what the follow-up above borrows.
 - kvbm `SchedulableSequence` speculative lifecycle (`schedule_speculative` /
-  `apply_speculative`, LIFO block release).
+  `apply_speculative`, LIFO block release), reused unchanged here.