Keep non-English query terms searchable (#964)

Graph queries filtered every token with len > 2, which dropped common two-character Chinese search terms while trying to suppress short English noise. Centralize query token selection and apply the length gate only to pure-English tokens so mixed or non-English terms remain searchable.

Constraint: Issue #962 reports space-separated Chinese query terms such as 前端, 依赖, and 安装 are lost by graphify query.

Rejected: Add Chinese segmentation now | the reported failure is fixed by preserving existing space-separated non-English tokens without expanding query behavior.

Confidence: high

Scope-risk: narrow

Directive: Keep CLI, MCP query, and benchmark query tokenization on one helper when changing query-term rules.

Tested: uv run --with pytest pytest tests/test_serve.py tests/test_query_cli.py tests/test_benchmark.py

Tested: graphify update .

Not-tested: Full test suite.

Co-authored-by: OmX <omx@oh-my-codex.dev>

Author: candy-Tong

graphify/benchmark.py

@@ -7,6 +7,7 @@
 from networkx.readwrite import json_graph
 
 from graphify.build import edge_data
+from graphify.serve import _query_terms
 
 
 _CHARS_PER_TOKEN = 4  # standard approximation
@@ -37,7 +38,7 @@ def _estimate_tokens(text: str) -> int:
 
 def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int:
     """Run BFS from best-matching nodes and return estimated tokens in the subgraph context."""
-    terms = [t.lower() for t in question.split() if len(t) > 2]
+    terms = _query_terms(question)
     scored = []
     for nid, data in G.nodes(data=True):
         label = data.get("label", "").lower()

graphify/serve.py

@@ -50,6 +50,19 @@ def _strip_diacritics(text: str) -> str:
     return "".join(c for c in nfkd if not unicodedata.combining(c))
 
 
+def _query_terms(question: str) -> list[str]:
+    """Split a query into searchable terms, filtering only short English terms."""
+    terms: list[str] = []
+    for raw in question.split():
+        term = raw.lower().strip()
+        if not term:
+            continue
+        is_english_only = all("a" <= ch <= "z" for ch in term)
+        if not is_english_only or len(term) > 2:
+            terms.append(term)
+    return terms
+
+
 _EXACT_MATCH_BONUS = 1000.0
 _PREFIX_MATCH_BONUS = 100.0
 _SUBSTRING_MATCH_BONUS = 1.0
@@ -306,7 +319,7 @@ def _query_graph_text(
     token_budget: int = 2000,
     context_filters: list[str] | None = None,
 ) -> str:
-    terms = [t.lower() for t in question.split() if len(t) > 2]
+    terms = _query_terms(question)
     scored = _score_nodes(G, terms)
     start_nodes = _pick_seeds(scored)
     if not start_nodes:

tests/test_benchmark.py

@@ -47,6 +47,13 @@ def test_query_bfs_expands_neighbors():
     assert tokens_deep >= tokens_shallow
 
 
+def test_query_keeps_short_non_english_terms():
+    G = nx.Graph()
+    G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0)
+    tokens = _query_subgraph_tokens(G, "前端", depth=1)
+    assert tokens > 0
+
+
 # --- run_benchmark ---
 
 def test_run_benchmark_returns_reduction(tmp_path):

tests/test_serve.py

@@ -13,6 +13,7 @@
     _dfs,
     _filter_graph_by_context,
     _infer_context_filters,
+    _query_terms,
     _query_graph_text,
     _resolve_context_filters,
     _subgraph_to_text,
@@ -79,6 +80,19 @@ def test_score_nodes_source_file_partial():
     assert "n2" in nids
 
 
+def test_query_terms_filters_only_short_english_terms():
+    terms = _query_terms("前端 dependency 依赖 install 安装 to of 包管理器 项目约定 a前")
+    assert terms == ["前端", "dependency", "依赖", "install", "安装", "包管理器", "项目约定", "a前"]
+
+
+def test_query_graph_text_keeps_short_non_english_terms():
+    G = nx.Graph()
+    G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0)
+    text = _query_graph_text(G, "前端", mode="bfs", depth=1)
+    assert "No matching nodes found." not in text
+    assert "NODE 前端" in text
+
+
 def test_infer_context_filters_for_calls_question():
     assert _infer_context_filters("who calls extract") == ["call"]