diff --git a/CHANGELOG.md b/CHANGELOG.md index e90425d..860b701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add a safe tokenized FTS5 query helper for arbitrary user search input. - Preserve unrelated tracked mirror edits across write synchronization rebases. - Retry atomic branch-and-tag snapshot pushes after rebasing and retargeting unpublished tags. - Update Go to 1.26.4 for current standard-library security fixes and add race and vulnerability CI gates. diff --git a/store/fts.go b/store/fts.go index d4891c0..5f7c912 100644 --- a/store/fts.go +++ b/store/fts.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "strings" + "unicode" ) func FTS5Phrase(value string) string { @@ -33,6 +34,29 @@ func FTS5Terms(value, operator string) (string, error) { return strings.Join(quoted, separator), nil } +// FTS5TokenQuery converts arbitrary user input into quoted FTS5 tokens joined +// by implicit AND, discarding punctuation that could be parsed as operators. +func FTS5TokenQuery(value string) string { + terms := make([]string, 0) + var token strings.Builder + flush := func() { + if token.Len() == 0 { + return + } + terms = append(terms, FTS5Phrase(token.String())) + token.Reset() + } + for _, r := range value { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' { + token.WriteRune(r) + continue + } + flush() + } + flush() + return strings.Join(terms, " ") +} + type contextExecer interface { ExecContext(context.Context, string, ...any) (sql.Result, error) } diff --git a/store/store_test.go b/store/store_test.go index 74ca959..bbaf5ec 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -163,6 +163,12 @@ func TestFTS5Helpers(t *testing.T) { if _, err := FTS5Terms("hello", "NEAR"); err == nil { t.Fatal("unsupported operator should fail") } + if query := FTS5TokenQuery(`scope-upgrade OR café_2`); query != `"scope" "upgrade" "OR" "café_2"` { + t.Fatalf("token query = %q", query) + } + if query := FTS5TokenQuery(`-- ""`); query != "" { + t.Fatalf("punctuation-only token query = %q", query) + } ctx := context.Background() st, err := Open(ctx, Options{