feat(router): add cache_aware_length policy with long/short pool split

bindings/golang/src/policy.rs

@@ -24,8 +24,8 @@ use openai_protocol::{
 };
 use smg::{
     policies::{
-        BucketPolicy, CacheAwarePolicy, LoadBalancingPolicy, PowerOfTwoPolicy, RandomPolicy,
-        RoundRobinPolicy, SelectWorkerInfo,
+        BucketPolicy, CacheAwarePolicy, LoadBalancingPolicy, PowerOfTwoPolicy,
+        RandomPolicy, RoundRobinPolicy, SelectWorkerInfo,
     },
     routers::grpc::{backend_client::BackendClient, utils::process_chat_messages},
     worker::{

bindings/python/src/lib.rs

@@ -20,6 +20,7 @@ pub enum PolicyType {
     RoundRobin,
     Passthrough,
     CacheAware,
+    CacheAwareLength,
     PowerOfTwo,
     LeastLoad,
     Bucket,
@@ -526,6 +527,11 @@ struct Router {
     worker_overload_token_usage: Option<f64>,
     worker_overload_protection: bool,
     disable_load_monitoring: bool,
+    chars_per_token: usize,
+    long_prefill_threshold: usize,
+    long_pool_max_load: usize,
+    short_pool_max_load: usize,
+    long_prefill_indices: Vec<usize>,
 }
 
 impl Router {
@@ -629,6 +635,25 @@ impl Router {
                     cache_ttl_secs: self.cache_ttl_secs,
                     cache_boundaries: self.cache_boundaries.clone(),
                 },
+                PolicyType::CacheAwareLength => ConfigPolicyConfig::CacheAwareLength {
+                    cache_threshold: self.cache_threshold,
+                    balance_abs_threshold: self.balance_abs_threshold,
+                    balance_rel_threshold: self.balance_rel_threshold,
+                    eviction_interval_secs: self.eviction_interval_secs,
+                    max_tree_size: self.max_tree_size,
+                    block_size: self.block_size,
+                    balance_token_usage_threshold: self.balance_token_usage_threshold,
+                    overload_token_usage_threshold: self.overload_token_usage_threshold,
+                    overlap_decay: self.overlap_decay,
+                    selection_temperature: self.selection_temperature,
+                    cache_index: self.parse_cache_index()?,
+                    cache_ttl_secs: self.cache_ttl_secs,
+                    cache_boundaries: self.cache_boundaries.clone(),
+                    chars_per_token: self.chars_per_token,
+                    long_prefill_threshold: self.long_prefill_threshold,
+                    long_pool_max_load: self.long_pool_max_load,
+                    short_pool_max_load: self.short_pool_max_load,
+                },
                 PolicyType::PowerOfTwo => ConfigPolicyConfig::PowerOfTwo {
                     load_check_interval_secs: self.load_monitor_interval,
                 },
@@ -931,6 +956,7 @@ impl Router {
                 self.server_key_path.as_ref(),
             )
             .dp_minimum_tokens_scheduler(self.dp_minimum_tokens_scheduler)
+            .long_prefill_indices(self.long_prefill_indices.clone())
             .build()
     }
 }
@@ -1083,6 +1109,11 @@ impl Router {
         worker_overload_token_usage = None,
         worker_overload_protection = false,
         disable_load_monitoring = false,
+        chars_per_token = 4,
+        long_prefill_threshold = 100_000,
+        long_pool_max_load = 4,
+        short_pool_max_load = 32,
+        long_prefill_indices = vec![],
     ))]
     #[expect(clippy::too_many_arguments)]
     #[expect(
@@ -1233,6 +1264,11 @@ impl Router {
         worker_overload_token_usage: Option<f64>,
         worker_overload_protection: bool,
         disable_load_monitoring: bool,
+        chars_per_token: usize,
+        long_prefill_threshold: usize,
+        long_pool_max_load: usize,
+        short_pool_max_load: usize,
+        long_prefill_indices: Vec<usize>,
     ) -> PyResult<Self> {
         let mut all_urls = worker_urls.clone();
 
@@ -1397,6 +1433,11 @@ impl Router {
             worker_overload_token_usage,
             worker_overload_protection,
             disable_load_monitoring,
+            chars_per_token,
+            long_prefill_threshold,
+            long_pool_max_load,
+            short_pool_max_load,
+            long_prefill_indices,
         })
     }
 

bindings/python/src/smg/router.py

@@ -25,6 +25,7 @@ def policy_from_str(policy_str: str | None) -> PolicyType:
         "round_robin": PolicyType.RoundRobin,
         "passthrough": PolicyType.Passthrough,
         "cache_aware": PolicyType.CacheAware,
+        "cache_aware_length": PolicyType.CacheAwareLength,
         "power_of_two": PolicyType.PowerOfTwo,
         "least_load": PolicyType.LeastLoad,
         "bucket": PolicyType.Bucket,

bindings/python/src/smg/router_args.py

@@ -15,6 +15,7 @@
     "round_robin",
     "passthrough",
     "cache_aware",
+    "cache_aware_length",
     "power_of_two",
     "least_load",
     "manual",
@@ -242,6 +243,12 @@ class RouterArgs:
     worker_overload_protection: bool = False
     # Restore the conditional load-monitor poll gate (default: poll always)
     disable_load_monitoring: bool = False
+    # cache_aware_length policy: long/short pool split
+    chars_per_token: int = 4
+    long_prefill_threshold: int = 100_000
+    long_pool_max_load: int = 4
+    short_pool_max_load: int = 32
+    long_prefill_indices: list[int] = dataclasses.field(default_factory=list)
 
     @staticmethod
     def add_cli_args(
@@ -627,6 +634,7 @@ def add_cli_args(
         )
         routing_group.add_argument(
             f"--{prefix}eviction-interval-secs",
+            f"--{prefix}eviction-interval",
             type=int,
             default=RouterArgs.eviction_interval_secs,
             help="Interval in seconds between cache eviction operations",
@@ -677,6 +685,42 @@ def add_cli_args(
                 " approximate serving-engine cache retention. Defaults to 180."
             ),
         )
+        # cache_aware_length policy parameters
+        routing_group.add_argument(
+            f"--{prefix}chars-per-token",
+            type=int,
+            default=RouterArgs.chars_per_token,
+            help="Divisor for char-level token estimation when X-Prompt-Tokens"
+            " is absent (cache_aware_length policy). Default 4.",
+        )
+        routing_group.add_argument(
+            f"--{prefix}long-prefill-threshold",
+            type=int,
+            default=RouterArgs.long_prefill_threshold,
+            help="Uncached-prefill-token boundary between long and short"
+            " requests (cache_aware_length policy). Default 100000.",
+        )
+        routing_group.add_argument(
+            f"--{prefix}long-pool-max-load",
+            type=int,
+            default=RouterArgs.long_pool_max_load,
+            help="Load ceiling for the long pool (pool=long workers)"
+            " (cache_aware_length policy). Default 4.",
+        )
+        routing_group.add_argument(
+            f"--{prefix}short-pool-max-load",
+            type=int,
+            default=RouterArgs.short_pool_max_load,
+            help="Load ceiling for the short pool (remaining workers)"
+            " (cache_aware_length policy). Default 32.",
+        )
+        routing_group.add_argument(
+            f"--{prefix}long-prefill-indices",
+            type=_parse_int_csv,
+            default=[],
+            help="Comma-separated 0-based indices of --prefill URLs that belong"
+            " to the long pool (get pool=long label for cache_aware_length).",
+        )
         routing_group.add_argument(
             f"--{prefix}max-idle-secs",
             f"--{prefix}sticky-key-idle-secs",
@@ -1681,6 +1725,25 @@ def from_cli_args(cls, args: argparse.Namespace, use_router_prefix: bool = False
         return cls(**args_dict)
 
     def _validate_router_args(self):
+        if (self.prefill_urls or self.decode_urls) and not (
+            self.pd_disaggregation or self.epd_disaggregation
+        ):
+            raise ValueError(
+                "--prefill/--decode require --pd-disaggregation or --epd-disaggregation"
+            )
+
+        if len(set(self.long_prefill_indices)) != len(self.long_prefill_indices):
+            raise ValueError("--long-prefill-indices must not contain duplicate values")
+
+        if self.long_prefill_indices:
+            if min(self.long_prefill_indices) < 0:
+                raise ValueError("--long-prefill-indices values must be non-negative")
+            if max(self.long_prefill_indices) >= len(self.prefill_urls):
+                raise ValueError(
+                    "--long-prefill-indices value out of range for "
+                    f"{len(self.prefill_urls)} configured prefill workers"
+                )
+
         # Validate configuration based on mode
         if self.epd_disaggregation:
             if self.encode_policy: