From da5a0e5a0e22cd536080734c3435a008d28cd18f Mon Sep 17 00:00:00 2001 From: carlosmolina0615 Date: Sat, 4 Jul 2026 14:17:14 +0200 Subject: [PATCH 1/2] [Bugfix][Router] Seed the prefix trie on the below-threshold QPS fallback path With prefix_min_match_length > 0, PrefixAwareRouter starts with an empty HashTrie: every request matches below the threshold, falls back to QPS routing, and returns without inserting the prompt. The trie never seeds, so prefix affinity never activates and the router permanently degrades to pure QPS routing. Insert the prompt into the trie on the fallback path, attributed to the QPS-selected endpoint. Short or new prefixes keep load-balancing across endpoints, and once a request's prefix history exceeds the threshold it pins to the endpoint that already holds its KV cache, which is the intended semantics of #959. Signed-off-by: carlosmolina0615 --- src/tests/test_prefixaware_router.py | 57 +++++++++++++++++++++++- src/vllm_router/routers/routing_logic.py | 9 +++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/tests/test_prefixaware_router.py b/src/tests/test_prefixaware_router.py index 6266645aa..36691f5c7 100644 --- a/src/tests/test_prefixaware_router.py +++ b/src/tests/test_prefixaware_router.py @@ -38,6 +38,11 @@ async def test_route_falls_back_to_qps_when_match_below_threshold(): When the longest prefix match is shorter than prefix_min_match_length, the request should NOT use the matched endpoint. It should fall back to QPS-based routing and pick the engine with the lowest QPS. + + The prompt must still be inserted into the trie, attributed to the + QPS-selected endpoint. Otherwise a router that starts with an empty trie + never seeds it (every request matches below the threshold), and prefix + affinity never activates. """ endpoints = [ @@ -66,7 +71,57 @@ async def test_route_falls_back_to_qps_when_match_below_threshold(): assert url == "http://engine2.com" - fake_hashtrie.insert.assert_not_awaited() + fake_hashtrie.insert.assert_awaited_once_with( + "some prompt text", "http://engine2.com" + ) + + +@pytest.mark.asyncio +async def test_below_threshold_seeding_enables_later_pinning(): + """ + Regression test for the trie never seeding when + prefix_min_match_length > 0. + + Uses the real HashTrie. The first request has no prefix match, so it is + routed by QPS — but it must seed the trie. A follow-up request extending + the same prompt then matches above the threshold and must pin to the + endpoint that served the first request, even when QPS stats would prefer + the other endpoint. + """ + + endpoints = [ + EndpointInfo(url="http://engine1.com"), + EndpointInfo(url="http://engine2.com"), + ] + request = Request(headers={}) + + # 128 = one HashTrie chunk; the first chunk of both prompts is identical. + router = PrefixAwareRouter(prefix_min_match_length=128) + + first_prompt = "x" * 128 + "y" * 72 + followup_prompt = first_prompt + "z" * 40 + + # First request: cold trie, match below threshold -> QPS picks engine1. + request_stats = { + "http://engine1.com": RequestStats(qps=5), + "http://engine2.com": RequestStats(qps=10), + } + url = await router.route_request( + endpoints, None, request_stats, request, {"prompt": first_prompt} + ) + assert url == "http://engine1.com" + + # Follow-up: shares the first 128-char chunk, so it matches at the + # threshold and must pin to engine1 even though engine2 now has the + # lower QPS. + request_stats = { + "http://engine1.com": RequestStats(qps=10), + "http://engine2.com": RequestStats(qps=5), + } + url = await router.route_request( + endpoints, None, request_stats, request, {"prompt": followup_prompt} + ) + assert url == "http://engine1.com" @pytest.mark.asyncio diff --git a/src/vllm_router/routers/routing_logic.py b/src/vllm_router/routers/routing_logic.py index e7dcd0a48..9ac2f037d 100644 --- a/src/vllm_router/routers/routing_logic.py +++ b/src/vllm_router/routers/routing_logic.py @@ -505,7 +505,14 @@ async def route_request( ) if match_length < self.prefix_min_match_length: - return self._qps_routing(endpoints, request_stats) + # Fall back to QPS routing, but still record the prompt in the + # trie. Without this, a router configured with + # prefix_min_match_length > 0 starts with an empty trie, every + # request matches below the threshold, nothing is ever inserted, + # and prefix affinity never activates. + selected_endpoint = self._qps_routing(endpoints, request_stats) + await self.hashtrie.insert(prompt, selected_endpoint) + return selected_endpoint selected_endpoint = random.choice(list(matched_endpoint)) From b105ae85fdacb599242c0dd21d9e55d4aadf059c Mon Sep 17 00:00:00 2001 From: carlosmolina0615 Date: Sat, 4 Jul 2026 14:27:06 +0200 Subject: [PATCH 2/2] [Bugfix][Router] Do not insert None into the trie when no endpoints are available Signed-off-by: carlosmolina0615 --- src/vllm_router/routers/routing_logic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vllm_router/routers/routing_logic.py b/src/vllm_router/routers/routing_logic.py index 9ac2f037d..d6692fdfa 100644 --- a/src/vllm_router/routers/routing_logic.py +++ b/src/vllm_router/routers/routing_logic.py @@ -511,7 +511,8 @@ async def route_request( # request matches below the threshold, nothing is ever inserted, # and prefix affinity never activates. selected_endpoint = self._qps_routing(endpoints, request_stats) - await self.hashtrie.insert(prompt, selected_endpoint) + if selected_endpoint is not None: + await self.hashtrie.insert(prompt, selected_endpoint) return selected_endpoint selected_endpoint = random.choice(list(matched_endpoint))