feat(kv_index): the chain index, a lossless KV-event relay with state snapshots, and liveness-aware cache routing

.gitignore

@@ -116,6 +116,8 @@ work_dirs/
 
 # Rust target directory
 target/
+# Older builds of the kv-index churn bench wrote this into the crate directory under cargo test
+churn_result.json
 
 # VSCode
 .vscode

.pre-commit-config.yaml

@@ -7,6 +7,8 @@ repos:
       - id: check-symlinks
       - id: destroyed-symlinks
       - id: trailing-whitespace
+        # Patch files keep their whitespace: a blank context line is a single space.
+        exclude: \.patch$
       - id: end-of-file-fixer
       - id: check-yaml
         args: [--allow-multiple-documents, --unsafe]

bindings/python/src/lib.rs

@@ -528,6 +528,7 @@ struct Router {
     worker_overload_waiting_requests: Option<usize>,
     worker_overload_token_usage: Option<f64>,
     worker_overload_protection: bool,
+    worker_overload_shed: bool,
     disable_load_monitoring: bool,
     max_buffered_request_bytes: u64,
     kv_connector_annotation: String,
@@ -682,6 +683,8 @@ impl Router {
                     cache_index: self.parse_cache_index()?,
                     cache_ttl_secs: self.cache_ttl_secs,
                     cache_boundaries: self.cache_boundaries.clone(),
+                    selection_policy: None,
+                    selection_accounting_ttl_ms: 0,
                 },
                 PolicyType::PowerOfTwo => ConfigPolicyConfig::PowerOfTwo {
                     load_check_interval_secs: self.load_monitor_interval,
@@ -899,6 +902,7 @@ impl Router {
             .worker_overload_waiting_requests(self.worker_overload_waiting_requests)
             .worker_overload_token_usage(self.worker_overload_token_usage)
             .worker_overload_protection(self.worker_overload_protection)
+            .worker_overload_shed(self.worker_overload_shed)
             .disable_load_monitoring(self.disable_load_monitoring)
             .load_monitor_interval_secs(self.load_monitor_interval)
             .pd_admission_wait_secs(self.pd_admission_wait_secs)
@@ -1159,9 +1163,10 @@ impl Router {
         cache_ttl_secs = 180,
         job_queue_capacity = 1000,
         job_queue_concurrency = 200,
-        worker_overload_waiting_requests = None,
-        worker_overload_token_usage = None,
-        worker_overload_protection = false,
+        worker_overload_waiting_requests = Some(8),
+        worker_overload_token_usage = Some(0.8),
+        worker_overload_protection = true,
+        worker_overload_shed = false,
         disable_load_monitoring = false,
         max_buffered_request_bytes = 1_048_576,
         kv_connector_annotation = String::from("smg.ai/kv-connector"),
@@ -1331,6 +1336,7 @@ impl Router {
         worker_overload_waiting_requests: Option<usize>,
         worker_overload_token_usage: Option<f64>,
         worker_overload_protection: bool,
+        worker_overload_shed: bool,
         disable_load_monitoring: bool,
         max_buffered_request_bytes: u64,
         kv_connector_annotation: String,
@@ -1527,6 +1533,7 @@ impl Router {
             worker_overload_waiting_requests,
             worker_overload_token_usage,
             worker_overload_protection,
+            worker_overload_shed,
             disable_load_monitoring,
             max_buffered_request_bytes,
             kv_connector_annotation,

bindings/python/src/servicer.rs

@@ -787,6 +787,7 @@ impl PyTokenSpeedGrpcServer {
         max_running_requests = 0,
         data_parallel_size = 1,
         kv_events_endpoint = String::new(),
+        kv_events_replay_endpoint = String::new(),
         kv_events_topic = String::new(),
         engine_startup_timeout_secs = None,
     ))]
@@ -822,6 +823,7 @@ impl PyTokenSpeedGrpcServer {
         max_running_requests: i32,
         data_parallel_size: i32,
         kv_events_endpoint: String,
+        kv_events_replay_endpoint: String,
         kv_events_topic: String,
         engine_startup_timeout_secs: Option<f64>,
     ) -> PyResult<Self> {
@@ -850,6 +852,7 @@ impl PyTokenSpeedGrpcServer {
             max_running_requests,
             data_parallel_size,
             kv_events_endpoint,
+            kv_events_replay_endpoint,
             kv_events_topic,
         };
         let config = TokenSpeedServicerConfig {
@@ -952,6 +955,9 @@ impl PySglangGrpcServer {
         sglang_version = String::new(),
         max_running_requests = 0,
         data_parallel_size = 1,
+        kv_events_endpoint = String::new(),
+        kv_events_replay_endpoint = String::new(),
+        kv_events_topic = String::new(),
         engine_startup_timeout_secs = None,
     ))]
     #[expect(clippy::too_many_arguments)]
@@ -985,6 +991,9 @@ impl PySglangGrpcServer {
         sglang_version: String,
         max_running_requests: i32,
         data_parallel_size: i32,
+        kv_events_endpoint: String,
+        kv_events_replay_endpoint: String,
+        kv_events_topic: String,
         engine_startup_timeout_secs: Option<f64>,
     ) -> PyResult<Self> {
         let model = SglangModelInfo {
@@ -1011,6 +1020,9 @@ impl PySglangGrpcServer {
             sglang_version,
             max_running_requests,
             data_parallel_size,
+            kv_events_endpoint,
+            kv_events_replay_endpoint,
+            kv_events_topic,
         };
         let config = SglangServicerConfig {
             bind_address,

crates/engine_servicer/Cargo.toml

@@ -36,8 +36,17 @@ zeromq.workspace = true
 
 [dev-dependencies]
 engine-zmq-client = { workspace = true, features = ["mock-engine"] }
+criterion = { version = "0.8", features = ["html_reports"] }
+# The normalized KV-event streams' round trip into the gateway's reference index.
+kv-index.workspace = true
+# The engines' KV-event wire shapes, built in code for the normalizer tests.
+rmpv.workspace = true
 portpicker = "0.1"
 tempfile = "3"
 
+[[bench]]
+name = "kv_relay_apply"
+harness = false
+
 [lints]
 workspace = true