Change search subcommand to show results based on distance

This works better as all relevant hits are returned and no noise is offered
when less than 20 pages match.

Author: calpaterson

README.md

@@ -27,7 +27,7 @@ uv run memoryfield-tool --help
 ```
 
 Requires Python 3.12+.  Semantic search needs a local
-[ollama](https://ollama.dev) with `nomic-embed-text` pulled; everything else
+[ollama](https://ollama.com) with `nomic-embed-text` pulled; everything else
 works without it (search falls back to substring matching).
 
 ## Commands
@@ -196,8 +196,12 @@ s3 fields index to `~/.cache/memoryfield-tool/indexes/<field>.sqlite3` on the
 local machine.  Both use the spec's `pages` schema.  Indexes are derived data
 and can be regenerated with `index` at any time — the cache-dir index is
 machine-local, so it is not shared across machines (run `index` on each one) and
-is not included in exports.  `search` returns the top 20 nearest pages (cosine
-distance shown in the output; no hard distance cutoff).  After each `write`,
+is not included in exports.  `search` returns every page whose cosine distance is
+at most `--max-distance` (default 0.45); cosine distance is shown in the output.
+It exits with status 1 and a stderr message when no pages match.  When nothing
+meets the cut-off, no results are returned (the substring fallback only applies
+when the index or embedder is unavailable, and is capped at 20 results).  After
+each `write`,
 the index is rebuilt in the background (a detached process; its stderr goes to
 `~/.cache/memoryfield-tool/reindex.log`).
 

src/memoryfield_tool/cli.py

@@ -962,21 +962,36 @@ def _format_result(result: search.SearchResult) -> str:
 @click.argument("query")
 @click.option("--field", "field_name", default=None, help="Limit to a single memoryfield")
 @click.option("--json", "as_json", is_flag=True, help="Output as JSON array")
-def search_cmd(query: str, field_name: str | None, as_json: bool) -> None:
+@click.option(
+    "--max-distance",
+    type=click.FloatRange(min=0.0, max=2.0),
+    default=search.DEFAULT_MAX_DISTANCE,
+    show_default=True,
+    help="Maximum cosine distance for semantic results",
+)
+def search_cmd(query: str, field_name: str | None, as_json: bool, max_distance: float) -> None:
     """Search pages across connected memoryfields.
 
     Examples:
 
     \b
         memoryfield-tool search 'carbon fibre'
         memoryfield-tool search --json 'carbon fibre'
+        memoryfield-tool search --max-distance 0.3 'carbon fibre'
+
+    Exits with status 1 when no pages match.
     """
     cfg = config.load_config()
     field_list = fields.connected_fields(cfg, field_name)
 
-    results, errors = search.search_all(field_list, query)
+    results, errors = search.search_all(field_list, query, max_distance)
     for name, msg in errors:
         click.echo(f"error: memoryfield {name}: {msg}", err=True)
+    if not results:
+        click.echo("No matching results found.", err=True)
+        if as_json:
+            click.echo("[]")
+        sys.exit(1)
     if as_json:
         payload = [
             {
@@ -990,9 +1005,6 @@ def search_cmd(query: str, field_name: str | None, as_json: bool) -> None:
         click.echo(json.dumps(payload, indent=2, default=str))
         return
 
-    if not results:
-        click.echo("No matching results found.")
-        return
     if len(field_list) == 1:
         for _fname, r in results:
             click.echo(_format_result(r))

src/memoryfield_tool/search.py

@@ -12,6 +12,9 @@
 
 RESULT_LIMIT = 20
 
+# this is a good threshold for nomic-embed-text-v1
+DEFAULT_MAX_DISTANCE: float = 0.45
+
 
 @dataclass(frozen=True)
 class SearchResult:
@@ -40,7 +43,7 @@ def _summary_from_json(fm_json: str) -> str:
     return str(summary) if summary is not None else ""
 
 
-def _vector_search(index_loc: Path, query: str) -> list[SearchResult] | None:
+def _vector_search(index_loc: Path, query: str, max_distance: float) -> list[SearchResult] | None:
     if not index_loc.is_file():
         return None
     result = embed.embed_texts([f"search_query: {query}"])
@@ -51,9 +54,12 @@ def _vector_search(index_loc: Path, query: str) -> list[SearchResult] | None:
 
     with contextlib.closing(_open_index(index_loc)) as db:
         cur = db.execute(
-            "SELECT filename, vec_distance_cosine(embedding, ?) AS distance "
-            f"FROM pages ORDER BY distance LIMIT {RESULT_LIMIT}",
-            (qblob,),
+            """
+            SELECT filename, vec_distance_cosine(embedding, ?) AS distance
+            FROM pages
+            WHERE vec_distance_cosine(embedding, ?) <= ?
+            ORDER BY distance""",
+            (qblob, qblob, max_distance),
         )
         rows = cur.fetchall()
 
@@ -109,15 +115,22 @@ def _substring_search(t: Transport, query: str) -> list[SearchResult]:
     return results[:RESULT_LIMIT]
 
 
-def search_field(t: Transport, index_loc: Path, query: str) -> list[SearchResult]:
-    vector = _vector_search(index_loc, query)
-    if vector:
+def search_field(
+    t: Transport,
+    index_loc: Path,
+    query: str,
+    max_distance: float = DEFAULT_MAX_DISTANCE,
+) -> list[SearchResult]:
+    vector = _vector_search(index_loc, query, max_distance)
+    if vector is not None:
         return vector
     return _substring_search(t, query)
 
 
 def search_all(
-    field_list: list[config.Field], query: str
+    field_list: list[config.Field],
+    query: str,
+    max_distance: float = DEFAULT_MAX_DISTANCE,
 ) -> tuple[list[tuple[str, SearchResult]], list[tuple[str, str]]]:
     """Search all fields, skipping unreachable ones.
 
@@ -130,7 +143,7 @@ def search_all(
         try:
             t = fields.get_transport(field)
             loc = fields.index_location(field)
-            for result in search_field(t, loc, query):
+            for result in search_field(t, loc, query, max_distance):
                 results.append((field.name, result))
         except (TransportError, click.ClickException) as e:
             errors.append((field.name, str(e)))

tests/test_cli.py

@@ -826,6 +826,9 @@ def test_schema_output(cli_runner):
     catalog = next(c for c in data["commands"] if c["name"] == "catalog")
     sort_by = next(p for p in catalog["params"] if p["name"] == "sort_by")
     assert set(sort_by["choices"]) == {"path", "title", "created", "updated"}
+    search_entry = next(c for c in data["commands"] if c["name"] == "search")
+    max_distance = next(p for p in search_entry["params"] if p["name"] == "max_distance")
+    assert max_distance["default"] == 0.45
 
 
 def test_schema_meta_covers_every_command():
@@ -926,6 +929,44 @@ def test_search_json(cli_runner, connected):
     assert any(r["filename"] == "gamma.md" and r["field"] == "notes" for r in data)
 
 
+def test_search_max_distance_excludes_all(cli_runner, connected, fake_embed):
+    _cfg_path, _field_path = connected
+    cli_runner.invoke(cli.cli, ["index", "--field", "notes"])
+    result = cli_runner.invoke(
+        cli.cli, ["search", "--field", "notes", "--max-distance", "0.01", "beta"]
+    )
+    assert result.exit_code == 1
+    assert result.stdout == ""
+    assert "No matching results found." in result.stderr
+
+
+def test_search_max_distance_returns_all(cli_runner, connected, fake_embed):
+    _cfg_path, _field_path = connected
+    cli_runner.invoke(cli.cli, ["index", "--field", "notes"])
+    result = cli_runner.invoke(
+        cli.cli, ["search", "--field", "notes", "--max-distance", "1.0", "beta"]
+    )
+    assert result.exit_code == 0
+    for name in ("index.md", "alpha.md", "beta.md", "gamma.md"):
+        assert name in result.output
+
+
+def test_search_no_results_exit_code(cli_runner, connected):
+    _cfg_path, _field_path = connected
+    result = cli_runner.invoke(cli.cli, ["search", "--field", "notes", "zzzz-not-there"])
+    assert result.exit_code == 1
+    assert result.stdout == ""
+    assert "No matching results found." in result.stderr
+
+
+def test_search_json_no_results_exit_code(cli_runner, connected):
+    _cfg_path, _field_path = connected
+    result = cli_runner.invoke(cli.cli, ["search", "--json", "--field", "notes", "zzzz-not-there"])
+    assert result.exit_code == 1
+    assert json.loads(result.stdout) == []
+    assert "No matching results found." in result.stderr
+
+
 def test_search_multi_field_prefix(cli_runner, config_env, field_dir, tmp_path):
     field2 = tmp_path / "field2"
     field2.mkdir()

tests/test_search.py

@@ -20,35 +20,45 @@ def test_vector_ranking_exact_match(field_dir, fake_embed, monkeypatch):
     }
 
 
-def test_vector_no_threshold_returns_ranked(field_dir, monkeypatch):
-    def controlled(texts):
-        out = []
-        for t in texts:
-            if t.startswith("search_document:"):
-                if "Alpha" in t:
-                    out.append([1.0] + [0.0] * 767)
-                else:
-                    out.append([0.0, 1.0] + [0.0] * 766)
-            else:
-                out.append([1.0, 0.1] + [0.0] * 766)
-        return out
-
-    monkeypatch.setattr("memoryfield_tool.embed.embed_texts", controlled)
+def _polarised_embed(texts: list[str]) -> list[list[float]]:
+    """Docs mentioning Alpha sit at similarity 0.995 to the query; all others at 0.0995."""
+    out = []
+    for t in texts:
+        if t.startswith("search_document:") and "Alpha" in t:
+            out.append([1.0] + [0.0] * 767)
+        elif t.startswith("search_document:"):
+            out.append([0.0, 1.0] + [0.0] * 766)
+        else:
+            out.append([1.0, 0.1] + [0.0] * 766)
+    return out
+
+
+def test_vector_threshold_filters_below_default(field_dir, monkeypatch):
+    monkeypatch.setattr("memoryfield_tool.embed.embed_texts", _polarised_embed)
     index.build_index(transport.local(field_dir), index.index_path(field_dir), progress=False)
-
     results = search.search_field(
         transport.local(field_dir), index.index_path(field_dir), "some query"
     )
+    assert [r.filename for r in results] == ["alpha.md"]
+
+
+def test_vector_large_max_distance_returns_ranked(field_dir, monkeypatch):
+    monkeypatch.setattr("memoryfield_tool.embed.embed_texts", _polarised_embed)
+    index.build_index(transport.local(field_dir), index.index_path(field_dir), progress=False)
+    results = search.search_field(
+        transport.local(field_dir),
+        index.index_path(field_dir),
+        "some query",
+        max_distance=1.0,
+    )
     filenames = [r.filename for r in results]
+    assert set(filenames) == {"alpha.md", "beta.md", "gamma.md", "index.md"}
     assert filenames[0] == "alpha.md"
-    assert "beta.md" in filenames
-    assert "gamma.md" in filenames
-    assert "index.md" in filenames
     assert all(r.distance is not None for r in results)
     assert results[0].distance < results[1].distance
 
 
-def test_vector_results_limited_to_20(field_dir, fake_embed):
+def test_vector_returns_all_above_default_threshold(field_dir, fake_embed):
     for i in range(25):
         (field_dir / f"page-{i:02d}.md").write_text(
             f"---\ntitle: Page {i}\n---\n\nbody {i}\n", encoding="utf-8"
@@ -57,7 +67,7 @@ def test_vector_results_limited_to_20(field_dir, fake_embed):
     results = search.search_field(
         transport.local(field_dir), index.index_path(field_dir), "anything"
     )
-    assert len(results) == 20
+    assert len(results) == 29
 
 
 def test_vector_result_without_frontmatter(field_dir, fake_embed):