fix: preserve Unicode marks in FTS queries

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Add shared SQL LIKE literal escaping for crawler search queries.
+- Preserve Unicode combining marks in tokenized FTS5 queries.
 - Add a safe tokenized FTS5 query helper for arbitrary user search input.
 - Preserve unrelated tracked mirror edits across write synchronization rebases.
 - Retry atomic branch-and-tag snapshot pushes after rebasing and retargeting unpublished tags.

store/fts.go

@@ -36,6 +36,7 @@ func FTS5Terms(value, operator string) (string, error) {
 
 // FTS5TokenQuery converts arbitrary user input into quoted FTS5 tokens joined
 // by implicit AND, discarding punctuation that could be parsed as operators.
+// It returns an empty string when no searchable tokens remain.
 func FTS5TokenQuery(value string) string {
 	terms := make([]string, 0)
 	var token strings.Builder
@@ -47,7 +48,7 @@ func FTS5TokenQuery(value string) string {
 		token.Reset()
 	}
 	for _, r := range value {
-		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) || r == '_' {
 			token.WriteRune(r)
 			continue
 		}

store/store_test.go

@@ -166,21 +166,61 @@ func TestFTS5Helpers(t *testing.T) {
 	if query := FTS5TokenQuery(`scope-upgrade OR café_2`); query != `"scope" "upgrade" "OR" "café_2"` {
 		t.Fatalf("token query = %q", query)
 	}
-	if query := FTS5TokenQuery(`-- ""`); query != "" {
-		t.Fatalf("punctuation-only token query = %q", query)
+	if query := FTS5TokenQuery("e\u0301clair"); query != "\"e\u0301clair\"" {
+		t.Fatalf("decomposed Unicode token query = %q", query)
+	}
+	for _, input := range []string{"", `-- ""`, `*:^()`} {
+		if query := FTS5TokenQuery(input); query != "" {
+			t.Fatalf("empty/punctuation-only token query = %q", query)
+		}
 	}
 
 	ctx := context.Background()
 	st, err := Open(ctx, Options{
 		Path:   filepath.Join(t.TempDir(), "fts.db"),
-		Schema: `create virtual table docs using fts5(body);`,
+		Schema: `create virtual table docs using fts5(id unindexed, body);`,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer st.Close()
-	if _, err := st.DB().ExecContext(ctx, `insert into docs(body) values('hello world')`); err != nil {
-		t.Fatal(err)
+	docs := []struct {
+		id   string
+		body string
+	}{
+		{id: "plain", body: "scope upgrade"},
+		{id: "spaced", body: "scope filler upgrade"},
+		{id: "operator", body: "foo OR bar"},
+		{id: "unicode", body: "café_2 東京 мир"},
+		{id: "decomposed", body: "e\u0301clair"},
+	}
+	for _, doc := range docs {
+		if _, err := st.DB().ExecContext(ctx, `insert into docs(id, body) values (?, ?)`, doc.id, doc.body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	queries := []struct {
+		input string
+		want  string
+	}{
+		{input: `scope-upgrade`, want: "plain,spaced"},
+		{input: `foo" OR bar*`, want: "operator"},
+		{input: `café_2 東京 мир`, want: "unicode"},
+		{input: "e\u0301clair", want: "decomposed"},
+	}
+	for _, tt := range queries {
+		var got string
+		query := FTS5TokenQuery(tt.input)
+		err := st.DB().QueryRowContext(ctx, `
+			select coalesce(group_concat(id, ','), '')
+			from (select id from docs where docs match ? order by id)
+		`, query).Scan(&got)
+		if err != nil {
+			t.Fatalf("query %q: %v", tt.input, err)
+		}
+		if got != tt.want {
+			t.Fatalf("query %q matches %q, want %q", tt.input, got, tt.want)
+		}
 	}
 	if err := OptimizeFTS5(ctx, st.DB(), "docs"); err != nil {
 		t.Fatal(err)