diff --git a/CHANGELOG.md b/CHANGELOG.md index 1df12b35..014b3a66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,15 @@ - 修复 offload local-llm 模式下每次 LLM 调用都重新创建 fetch wrapper 的性能问题(现在在 `LocalLlmClient` 构造函数中创建一次并缓存)。 - 注入逻辑抽取到 `src/utils/no-think-fetch.ts` 共享,新增 vitest 单测覆盖全部策略 / 跳过 embedding / 非 JSON 容错。 +### 🐛 修复 / 安全性 + +- **FTS5 MATCH 表达式查询语义注入** ([#160](https://github.com/TencentDB/TencentDB-Agent-Memory/issues/160)):将 `buildFtsQuery()` 收敛为字面量 token 构造器,用户输入永远不能贡献 FTS5 查询结构: + - `sanitizeFtsInput()` 先对原始文本做 NFKC 归一化,再只提取 Unicode 字母、数字和下划线;FTS5 operators、列过滤、引号、通配符、括号和 `NEAR/5` 这类语法片段都会变成普通 token 分隔。 + - jieba 分词或 regex 回退后,`sanitizeFtsToken()` 会再次对每个 tokenizer 输出做字面量提取,防止 tokenizer stub/native binding 返回带标点或查询片段的 token。 + - 所有输出 token 都由代码包成 FTS5 phrase literal,再用代码生成的 `OR` 连接;下游 `searchL1Fts()` / `searchL0Fts()` 继续通过 `MATCH ?` 参数绑定执行。 + - 与 #178 的窄修不同,本修复不会删除用户实际输入的 `AND` / `OR` / `NOT` / `NEAR` 等词,而是把它们转成 `"AND"` 等可搜索字面量,避免安全修复误伤召回。 + - 新增 46 个测试(31 个单元测试 + 15 个真实 `node:sqlite` FTS5 fixture 集成测试覆盖 tokenizer 二次净化、true MATCH 执行、reserved-word 字面量召回、recall 对比与 200 次 fuzz)。 + ### ⚠️ 升级注意(仅在显式配置 `timezone` 时生效) 如果你**显式**设置了 IANA 时区(如 `"Asia/Shanghai"`): diff --git a/src/core/store/sqlite.recall.test.ts b/src/core/store/sqlite.recall.test.ts new file mode 100644 index 00000000..44d87446 --- /dev/null +++ b/src/core/store/sqlite.recall.test.ts @@ -0,0 +1,280 @@ +/** + * Recall & true-FTS5 integration tests for the sanitization in + * `src/core/store/sqlite.ts`. + * + * Two angles are exercised here: + * + * 1. **True FTS5 execution** — we instantiate a real `node:sqlite` + * database with an FTS5 virtual table, index a fixture, and run the + * queries that `buildFtsQuery()` produces. This proves the generated + * MATCH expressions are syntactically valid AND that malicious inputs + * cannot poison result semantics. + * + * 2. **Recall comparison** — we compare the result set size and top-K + * contents between the new sanitized `buildFtsQuery()` and the old + * raw-token form on a fixed 25-row corpus. A regression on ordinary + * keyword queries would surface here. + * + * The tests intentionally avoid `@node-rs/jieba` so they remain + * deterministic in CI environments without the optional native binding. + */ + +import { describe, expect, it, beforeAll, afterAll } from "vitest"; +import { DatabaseSync } from "node:sqlite"; + +import { _setJiebaForTest, buildFtsQuery } from "./sqlite.js"; + +interface IndexedDoc { + record_id: string; + content: string; +} + +const FIXTURE: IndexedDoc[] = [ + { record_id: "r1", content: "TypeScript builds scalable web APIs" }, + { record_id: "r2", content: "TencentDB memory plugin uses SQLite FTS5" }, + { record_id: "r3", content: "Vector search with sqlite-vec extension" }, + { record_id: "r4", content: "用户偏好简洁的 TypeScript 示例" }, + { record_id: "r5", content: "Travel plan for Tokyo in May" }, + { record_id: "r6", content: "Plan a trip to Japan and visit Tokyo Tower" }, + { record_id: "r7", content: "OpenClaw integration guide for Hermes" }, + { record_id: "r8", content: "ANDROID 12 release notes for programmers" }, + { record_id: "r9", content: "ORACLE database administrator tutorial" }, + { record_id: "r10", content: "NEARBY restaurants open after midnight" }, + { record_id: "r11", content: "FTS5 reserved operators in attack samples" }, + { record_id: "r12", content: "Content column filter syntax content:foo" }, + { record_id: "r13", content: "Prefix wildcard example alpha*" }, + { record_id: "r14", content: "Parenthesized group (alpha) OR (beta)" }, + { record_id: "r15", content: "Phrase-quote injection alpha\" OR \"beta" }, + { record_id: "r16", content: "Single quote escape alpha'beta" }, + { record_id: "r17", content: "中文测试 分词 用户 编程 TypeScript" }, + { record_id: "r18", content: "BM25 ranking comparison baseline" }, + { record_id: "r19", content: "Plugin memory pipeline L0 L1 L2 L3" }, + { record_id: "r20", content: "Empty match NULL semantics check" }, + { record_id: "r21", content: "Sanity check fixture record 21" }, + { record_id: "r22", content: "Sanity check fixture record 22" }, + { record_id: "r23", content: "Sanity check fixture record 23" }, + { record_id: "r24", content: "Sanity check fixture record 24" }, + { record_id: "r25", content: "Sanity check fixture record 25" }, + { record_id: "r26", content: "Boolean operator words AND OR NOT NEAR" }, +]; + +/** Build the OLD pre-sanitization MATCH expression for comparison only. */ +function legacyBuildFtsQuery(raw: string): string | null { + const tokens = raw.match(/[\p{L}\p{N}_]+/gu) ?? []; + if (tokens.length === 0) return null; + return tokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR "); +} + +/** + * Insert the fixture into a fresh in-memory FTS5 table. Each describe + * block creates its own db (and closes it in `afterAll`) so tests are + * isolated across forks. + */ +function setupFts(): DatabaseSync { + const database = new DatabaseSync(":memory:"); + database.exec(` + CREATE VIRTUAL TABLE docs USING fts5( + record_id UNINDEXED, + content, + tokenize = 'unicode61 remove_diacritics 2' + ) + `); + const insert = database.prepare( + "INSERT INTO docs (record_id, content) VALUES (?, ?)", + ); + for (const doc of FIXTURE) insert.run(doc.record_id, doc.content); + return database; +} + +describe("buildFtsQuery — real FTS5 execution", () => { + let db: DatabaseSync; + + beforeAll(() => { + _setJiebaForTest(null); + db = setupFts(); + }); + + afterAll(() => { + try { + db?.close(); + } catch { + /* best effort */ + } + }); + + /** Run a MATCH query and return the record_ids returned (BM25 order). */ + function queryIds(matchExpr: string): string[] { + const rows = db + .prepare( + "SELECT record_id FROM docs WHERE docs MATCH ? ORDER BY bm25(docs)", + ) + .all(matchExpr) as Array<{ record_id: string }>; + return rows.map((r) => r.record_id); + } + + it("returns a syntactically valid MATCH for the happy path", () => { + const match = buildFtsQuery("TypeScript memory"); + expect(match).not.toBeNull(); + const ids = queryIds(match!); + expect(ids.length).toBeGreaterThan(0); + }); + + it("does not throw on column-filter abuse", () => { + const match = buildFtsQuery("content:foo"); + expect(match).not.toBeNull(); + expect(() => queryIds(match!)).not.toThrow(); + }); + + it("does not throw on parenthesized group expression", () => { + const match = buildFtsQuery("(alpha) OR (beta)"); + expect(match).not.toBeNull(); + expect(() => queryIds(match!)).not.toThrow(); + }); + + it("does not throw on prefix-wildcard syntax", () => { + const match = buildFtsQuery("alpha*"); + expect(match).not.toBeNull(); + expect(() => queryIds(match!)).not.toThrow(); + }); + + it("does not throw on phrase-quote injection", () => { + const match = buildFtsQuery('alpha" OR "beta'); + expect(match).not.toBeNull(); + expect(() => queryIds(match!)).not.toThrow(); + }); + + it("keeps operator words but returns null for pure syntax", () => { + expect(buildFtsQuery("AND OR NOT NEAR")).toBe( + '"AND" OR "OR" OR "NOT" OR "NEAR"', + ); + expect(buildFtsQuery("***(((")).toBeNull(); + }); + + it("executes AND NOT beta cleanly as OR-of-tokens", () => { + const match = buildFtsQuery("alpha AND NOT beta"); + expect(match).not.toBeNull(); + // Should be a safe OR-of-terms query; must execute without throwing. + expect(() => queryIds(match!)).not.toThrow(); + }); + + it("keeps reserved words searchable as literal document text", () => { + const match = buildFtsQuery("AND OR NOT NEAR"); + expect(match).toBe('"AND" OR "OR" OR "NOT" OR "NEAR"'); + expect(queryIds(match!)).toContain("r26"); + }); +}); + +describe("buildFtsQuery — recall comparison vs. legacy", () => { + let db: DatabaseSync; + + beforeAll(() => { + _setJiebaForTest(null); + db = setupFts(); + }); + + afterAll(() => { + try { + db?.close(); + } catch { + /* best effort */ + } + }); + + function queryIds(matchExpr: string): string[] { + const rows = db + .prepare( + "SELECT record_id FROM docs WHERE docs MATCH ? ORDER BY bm25(docs)", + ) + .all(matchExpr) as Array<{ record_id: string }>; + return rows.map((r) => r.record_id); + } + + /** + * Sanitized recall must be a SUPERSET of legacy recall for benign queries: + * we drop only reserved words / syntax chars, never alphanumeric tokens. + */ + const queries = [ + "TypeScript memory", + "users prefer concise", + "TencentDB FTS5", + "Travel Tokyo", + "OpenClaw Hermes", + "Sanity check", + ]; + + for (const q of queries) { + it(`recall parity — "${q}"`, () => { + const legacy = legacyBuildFtsQuery(q); + const safe = buildFtsQuery(q); + expect(safe, `safe query for ${q}`).not.toBeNull(); + if (safe === null) return; + const legacyIds = legacy ? new Set(queryIds(legacy)) : new Set(); + const safeIds = new Set(queryIds(safe)); + for (const id of legacyIds) { + expect( + safeIds.has(id), + `${q}: legacy hit ${id} missing from safe`, + ).toBe(true); + } + }); + } +}); + +describe("buildFtsQuery — fuzz: random adversarial inputs never throw", () => { + let db: DatabaseSync; + + beforeAll(() => { + _setJiebaForTest(null); + db = setupFts(); + }); + + afterAll(() => { + try { + db?.close(); + } catch { + /* best effort */ + } + }); + + function queryIds(matchExpr: string): string[] { + const rows = db + .prepare( + "SELECT record_id FROM docs WHERE docs MATCH ? ORDER BY bm25(docs)", + ) + .all(matchExpr) as Array<{ record_id: string }>; + return rows.map((r) => r.record_id); + } + + it("survives 200 randomized attack-style queries without throwing", () => { + const corpus = [ + "alpha beta", + 'alpha" OR "beta', + "alpha AND NOT beta", + "(NEAR/5 alpha beta)", + "alpha* OR beta", + "content:-foo", + "message:bar", + "AND OR NOT NEAR", + "*()*()", + "OR AND OR AND OR", + '"a" AND "b" OR "c" NOT "d"', + "user:-something", + "and AND aND AnD", + ]; + let totalQueries = 0; + for (let i = 0; i < 200; i++) { + const n = 1 + (i % 3); + const fragments: string[] = []; + for (let j = 0; j < n; j++) { + fragments.push(corpus[(i * 7 + j * 13) % corpus.length]); + } + const input = fragments.join(" "); + const match = buildFtsQuery(input); + totalQueries++; + if (match !== null) { + expect(() => queryIds(match)).not.toThrow(); + } + } + expect(totalQueries).toBe(200); + }); +}); diff --git a/src/core/store/sqlite.test.ts b/src/core/store/sqlite.test.ts new file mode 100644 index 00000000..498834c6 --- /dev/null +++ b/src/core/store/sqlite.test.ts @@ -0,0 +1,273 @@ +/** + * Unit tests for FTS5 sanitization in `src/core/store/sqlite.ts`. + * + * These tests exercise `sanitizeFtsInput()`, `sanitizeFtsToken()` and + * `buildFtsQuery()` (in fallback mode — jieba stubbed to null). + * + * The behavior under test corresponds to the four acceptance levels of + * issue #160: + * 1. Basic escape — special characters are neutralized. + * 2. Complete coverage — every FTS5 operator / syntax character / colset + * abuse has an explicit test, including NFKC full-width variants and + * word-boundary safe substrings (ANDROID, ORACLE, NEARBY, SCANNER). + * 3. Recall — exercised in `sqlite.recall.test.ts` with a real FTS5 table. + * 4. Whitelist / parameterization — `sanitizeFtsToken()` is the per-token + * unicode-letter whitelist; the recall suite executes generated MATCH + * expressions through the same prepared `MATCH ?` shape as production. + * + * Expected outputs here specify the public MATCH expression that callers rely + * on: user text may become literal search terms, but never query syntax. + */ + +import { describe, expect, it, beforeEach } from "vitest"; + +import { + _setJiebaForTest, + buildFtsQuery, + sanitizeFtsInput, + sanitizeFtsToken, +} from "./sqlite.js"; + +beforeEach(() => { + // Force fallback (regex) tokenizer for determinism in this suite. + _setJiebaForTest(null); +}); + +describe("sanitizeFtsInput (sanitization layer 1)", () => { + it("returns empty string for empty input", () => { + expect(sanitizeFtsInput("")).toBe(""); + }); + + it("preserves benign Chinese / English / digits", () => { + expect(sanitizeFtsInput("用户 编程 TypeScript 2026")).toBe( + "用户 编程 TypeScript 2026", + ); + }); + + it("preserves FTS5 reserved words as literal search text", () => { + expect(sanitizeFtsInput("alpha \uFF21\uFF2E\uFF24 beta")).toBe( + "alpha AND beta", + ); + expect(sanitizeFtsInput("alpha Or beta")).toBe("alpha Or beta"); + expect(sanitizeFtsInput("alpha not beta")).toBe("alpha not beta"); + expect(sanitizeFtsInput("alpha Near beta")).toBe("alpha Near beta"); + }); + + it("does NOT touch substrings that contain reserved words", () => { + expect(sanitizeFtsInput("ANDROID")).toBe("ANDROID"); + expect(sanitizeFtsInput("SCANNER")).toBe("SCANNER"); + expect(sanitizeFtsInput("ORACLE")).toBe("ORACLE"); + expect(sanitizeFtsInput("NEARBY")).toBe("NEARBY"); + expect(sanitizeFtsInput("android 12")).toBe("android 12"); + }); + + it("strips FTS5 syntax chars: \" ' * ( )", () => { + // Syntax characters become separators and are collapsed by token extraction. + expect(sanitizeFtsInput('alpha "beta" gamma')).toBe("alpha beta gamma"); + expect(sanitizeFtsInput("alpha'beta")).toBe("alpha beta"); + expect(sanitizeFtsInput("alpha*")).toBe("alpha"); + expect(sanitizeFtsInput("(alpha) (beta)")).toBe("alpha beta"); + }); + + it("turns column-filter syntax into literal tokens", () => { + expect(sanitizeFtsInput("content:foo")).toBe("content foo"); + expect(sanitizeFtsInput("-content:foo")).toBe("content foo"); + expect(sanitizeFtsInput("message:hello world")).toBe("message hello world"); + expect(sanitizeFtsInput("-session:abc")).toBe("session abc"); + expect(sanitizeFtsInput("actor:user1 topic:food")).toBe( + "actor user1 topic food", + ); + }); + + it("normalizes full-width Unicode variants via NFKC", () => { + expect(sanitizeFtsInput("alpha \uFF21\uFF2E\uFF24 beta")).toBe( + "alpha AND beta", + ); + expect(sanitizeFtsInput("alpha \uFF21\uFF2E\uFF24ROID beta")).toBe( + "alpha ANDROID beta", + ); + }); + + it("handles tricky mixed inputs", () => { + expect(sanitizeFtsInput('"alpha" OR "beta"')).toBe("alpha OR beta"); + expect(sanitizeFtsInput("(content:foo) OR (message:bar)")).toBe( + "content foo OR message bar", + ); + }); + + it("collapses separators into single spaces", () => { + expect(sanitizeFtsInput("hello world")).toBe("hello world"); + }); +}); + +describe("sanitizeFtsToken (sanitization layer 2 — per-token)", () => { + it("returns null for empty input", () => { + expect(sanitizeFtsToken("")).toBeNull(); + }); + + it("wraps ASCII letters in phrase quotes", () => { + expect(sanitizeFtsToken("alpha")).toBe('"alpha"'); + }); + + it("preserves unicode letters and digits", () => { + expect(sanitizeFtsToken("用户2026")).toBe('"用户2026"'); + expect(sanitizeFtsToken("東京_タワー")).toBe('"東京_タワー"'); + }); + + it("drops pure-punctuation tokens", () => { + expect(sanitizeFtsToken("***")).toBeNull(); + expect(sanitizeFtsToken("...")).toBeNull(); + expect(sanitizeFtsToken("()")).toBeNull(); + }); + + it("quotes FTS5 reserved words as ordinary phrase tokens", () => { + expect(sanitizeFtsToken("AND")).toBe('"AND"'); + expect(sanitizeFtsToken("or")).toBe('"or"'); + expect(sanitizeFtsToken("Not")).toBe('"Not"'); + expect(sanitizeFtsToken("near")).toBe('"near"'); + }); + + it("strips embedded punctuation but keeps word fragments", () => { + expect(sanitizeFtsToken("foo:bar")).toBe('"foo" OR "bar"'); + expect(sanitizeFtsToken("C++")).toBe('"C"'); + }); + + it("filters embedded double-quotes via the unicode whitelist", () => { + // Quotes are filtered out by the unicode-letter/number whitelist; only + // the letters around them survive. This is intentional — quotes do not + // survive into the index token stream because they are FTS5 syntax. + expect(sanitizeFtsToken('a"b')).toBe('"a" OR "b"'); + expect(sanitizeFtsToken('he said "hi"')).toBe('"he" OR "said" OR "hi"'); + }); +}); + +describe("buildFtsQuery (fallback mode)", () => { + it("returns null for empty / null / undefined input", () => { + expect(buildFtsQuery("")).toBeNull(); + expect(buildFtsQuery(null)).toBeNull(); + expect(buildFtsQuery(undefined)).toBeNull(); + }); + + it("returns null when nothing survives sanitization", () => { + expect(buildFtsQuery("***")).toBeNull(); + expect(buildFtsQuery("(())")).toBeNull(); + }); + + it("searches reserved operators as literal user terms", () => { + expect(buildFtsQuery("AND")).toBe('"AND"'); + expect(buildFtsQuery("AND OR NOT NEAR")).toBe( + '"AND" OR "OR" OR "NOT" OR "NEAR"', + ); + }); + + it("quotes reserved operators instead of letting them control MATCH", () => { + expect(buildFtsQuery("alpha AND NOT beta")).toBe( + '"alpha" OR "AND" OR "NOT" OR "beta"', + ); + expect(buildFtsQuery("alpha Or beta")).toBe('"alpha" OR "Or" OR "beta"'); + }); + + it("escapes phrase-quote injection", () => { + expect(buildFtsQuery('alpha" OR "beta')).toBe( + '"alpha" OR "OR" OR "beta"', + ); + }); + + it("neutralizes the prefix-wildcard abuse path", () => { + expect(buildFtsQuery("alpha*")).toBe('"alpha"'); + }); + + it("neutralizes parenthesized group expressions", () => { + expect(buildFtsQuery("(alpha) OR (beta)")).toBe( + '"alpha" OR "OR" OR "beta"', + ); + }); + + it("neutralizes column-filter syntax", () => { + // `content:` is stripped — its letters also leak in via the regex via + // the whitespace separator, but the unicode-letter whitelist inside + // sanitizeFtsToken filters `content` back out ONLY when it is a + // stand-alone token. In this query `content:` is collapsed to a space + // by `sanitizeFtsInput` so we only ever see `foo` and `bar`. + expect(buildFtsQuery("content:foo bar")).toBe( + '"content" OR "foo" OR "bar"', + ); + expect(buildFtsQuery("-content:foo")).toBe('"content" OR "foo"'); + }); + + it("handles NFKC full-width operator variants", () => { + expect(buildFtsQuery("alpha \uFF21\uFF2E\uFF24 beta")).toBe( + '"alpha" OR "AND" OR "beta"', + ); + }); + + it("does NOT over-strip benign substrings (word-boundary safe)", () => { + expect(buildFtsQuery("ANDROID")).toBe('"ANDROID"'); + expect(buildFtsQuery("ORACLE")).toBe('"ORACLE"'); + expect(buildFtsQuery("NEARBY")).toBe('"NEARBY"'); + }); + + it("returns Chinese / mixed language query verbatim (only stripped syntax)", () => { + expect(buildFtsQuery("用户 TypeScript 编程")).toBe( + '"用户" OR "TypeScript" OR "编程"', + ); + }); + + it("deduplicates tokens after sanitization", () => { + expect(buildFtsQuery("alpha alpha alpha")).toBe('"alpha"'); + }); + + it("preserves reserved words even when surrounded by other tokens", () => { + expect(buildFtsQuery("foo AND bar AND baz")).toBe( + '"foo" OR "AND" OR "bar" OR "baz"', + ); + }); + + it("literalizes a long chain of mixed operators / syntax / junk", () => { + // All user text becomes quoted literal tokens; no FTS5 operator survives. + expect(buildFtsQuery('alpha AND NOT "beta" OR (gamma) NEAR hello')).toBe( + [ + '"alpha"', + '"AND"', + '"NOT"', + '"beta"', + '"OR"', + '"gamma"', + '"NEAR"', + '"hello"', + ].join(" OR "), + ); + }); + + it("re-sanitizes tokenizer output before building MATCH", () => { + _setJiebaForTest({ + cutForSearch(text: string): string[] { + expect(text).toBe("用户 AND TypeScript alpha OR beta foo bar"); + return [ + "用户", + "AND", + "TypeScript", + 'alpha" OR "beta', + "foo:bar", + "***", + "用户", + ]; + }, + }); + + expect( + buildFtsQuery('用户 AND TypeScript alpha" OR "beta foo:bar'), + ).toBe( + [ + '"用户"', + '"AND"', + '"TypeScript"', + '"alpha"', + '"OR"', + '"beta"', + '"foo"', + '"bar"', + ].join(" OR "), + ); + }); +}); diff --git a/src/core/store/sqlite.ts b/src/core/store/sqlite.ts index 02816e50..f24e64c8 100644 --- a/src/core/store/sqlite.ts +++ b/src/core/store/sqlite.ts @@ -195,15 +195,87 @@ const ZH_STOP_WORDS = new Set([ * Example (fallback): * "旅行计划 API" → '"旅行计划" OR "API"' */ -export function buildFtsQuery(raw: string): string | null { +const FTS_LITERAL_TOKEN_RE = /[\p{L}\p{N}_]+/gu; + +function extractFtsLiteralTokens(raw: string): string[] { + return raw.normalize("NFKC").match(FTS_LITERAL_TOKEN_RE) ?? []; +} + +function quoteFtsLiteralToken(token: string): string { + return '"' + token.replaceAll('"', '""') + '"'; +} + +function toQuotedFtsLiteralTerms(raw: string): string[] { + return extractFtsLiteralTokens(raw).map(quoteFtsLiteralToken); +} + +/** + * Normalize raw user text into a plain literal-token stream for FTS5. + * + * This helper deliberately does not parse or preserve FTS5 syntax. Operators + * such as AND / OR / NOT / NEAR, column filters such as content:foo, grouping, + * wildcards, quotes, and NEAR distance syntax are all treated as ordinary + * user text. The only characters that survive are Unicode letters, numbers, + * and underscores; every other character becomes a token separator. + * + * @internal + */ +export function sanitizeFtsInput(raw: string): string { + if (!raw) return ""; + return extractFtsLiteralTokens(raw).join(" "); +} + +/** + * Convert a tokenizer token into one or more quoted FTS5 literal terms. + * + * The return value is safe to splice into a generated MATCH expression because + * every emitted token is wrapped as a phrase literal. If a token still contains + * punctuation (for example from a tokenizer stub), it is split into literal + * sub-tokens instead of being concatenated into a new word. + * + * Returns `null` to drop tokens that contain no literal text. + * + * @internal + */ +export function sanitizeFtsToken(tok: string): string | null { + if (!tok) return null; + const terms = toQuotedFtsLiteralTerms(tok); + if (terms.length === 0) return null; + return terms.join(" OR "); +} + +/** + * Build an FTS5 MATCH query from raw text. + * + * User input is always treated as plain text, never as FTS5 query syntax: + * + * 1. `sanitizeFtsInput()` normalizes Unicode and turns punctuation / FTS5 + * syntax into token separators before tokenization. + * 2. Tokenization remains unchanged from upstream: + * - `@node-rs/jieba`'s `cutForSearch(text, true)` for Chinese (with + * stop-word filter & dedup), or + * - Unicode literal-token fallback. + * 3. `sanitizeFtsToken()` runs on every emitted token as a second guard. + * 4. Tokens are joined with ` OR ` inside FTS5 phrase quotes. + * + * The function returns `null` for input that contains no usable tokens so + * downstream FTS5 prepared statements can skip MATCH safely. + * + * @param raw User-supplied query text. `null` / `undefined` / `""` returns `null`. + */ +export function buildFtsQuery(raw: string | null | undefined): string | null { + if (raw == null) return null; + const cleaned = sanitizeFtsInput(raw); + if (!cleaned) return null; + const jieba = getJieba(); - let tokens: string[]; + let rawTokens: string[]; if (jieba) { // jieba cutForSearch: splits long words further for better recall - // e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"] - tokens = jieba - .cutForSearch(raw, true) + // e.g. a long Chinese term yields both sub-words and the full term. + rawTokens = jieba + .cutForSearch(cleaned, true) .map((t) => t.trim()) .filter((t) => { if (!t) return false; @@ -214,19 +286,18 @@ export function buildFtsQuery(raw: string): string | null { return true; }); // Deduplicate (cutForSearch may produce duplicates for sub-words) - tokens = [...new Set(tokens)]; + rawTokens = [...new Set(rawTokens)]; } else { - // Fallback: simple Unicode regex split - tokens = - raw - .match(/[\p{L}\p{N}_]+/gu) - ?.map((t) => t.trim()) - .filter(Boolean) ?? []; + rawTokens = cleaned.match(FTS_LITERAL_TOKEN_RE) ?? []; } - if (tokens.length === 0) return null; - const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`); - return quoted.join(" OR "); + const safeTokens: string[] = []; + for (const t of rawTokens) { + safeTokens.push(...toQuotedFtsLiteralTerms(t)); + } + const dedup = [...new Set(safeTokens)]; + if (dedup.length === 0) return null; + return dedup.join(" OR "); } /**