[codex] Add tokenized FTS5 query helper

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add a safe tokenized FTS5 query helper for arbitrary user search input.
 - Preserve unrelated tracked mirror edits across write synchronization rebases.
 - Retry atomic branch-and-tag snapshot pushes after rebasing and retargeting unpublished tags.
 - Update Go to 1.26.4 for current standard-library security fixes and add race and vulnerability CI gates.

store/fts.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"unicode"
 )
 
 func FTS5Phrase(value string) string {
@@ -33,6 +34,29 @@ func FTS5Terms(value, operator string) (string, error) {
 	return strings.Join(quoted, separator), nil
 }
 
+// FTS5TokenQuery converts arbitrary user input into quoted FTS5 tokens joined
+// by implicit AND, discarding punctuation that could be parsed as operators.
+func FTS5TokenQuery(value string) string {
+	terms := make([]string, 0)
+	var token strings.Builder
+	flush := func() {
+		if token.Len() == 0 {
+			return
+		}
+		terms = append(terms, FTS5Phrase(token.String()))
+		token.Reset()
+	}
+	for _, r := range value {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
+			token.WriteRune(r)
+			continue
+		}
+		flush()
+	}
+	flush()
+	return strings.Join(terms, " ")
+}
+
 type contextExecer interface {
 	ExecContext(context.Context, string, ...any) (sql.Result, error)
 }

store/store_test.go

@@ -163,6 +163,12 @@ func TestFTS5Helpers(t *testing.T) {
 	if _, err := FTS5Terms("hello", "NEAR"); err == nil {
 		t.Fatal("unsupported operator should fail")
 	}
+	if query := FTS5TokenQuery(`scope-upgrade OR café_2`); query != `"scope" "upgrade" "OR" "café_2"` {
+		t.Fatalf("token query = %q", query)
+	}
+	if query := FTS5TokenQuery(`-- ""`); query != "" {
+		t.Fatalf("punctuation-only token query = %q", query)
+	}
 
 	ctx := context.Background()
 	st, err := Open(ctx, Options{