Feature/knowledge box

Cargo.toml

@@ -27,10 +27,22 @@ exclude = [
 [[bin]]
 name = "octobrain"
 path = "src/main.rs"
+
+# Local retrieval-quality benchmark (BEIR). Gated behind the `bench` feature so
+# it never compiles into normal/release builds. Run with:
+#   cargo run --release --features bench --bin beir_bench -- <dataset_dir> [label]
+[[bin]]
+name = "beir_bench"
+path = "src/bin/beir_bench.rs"
+required-features = ["bench"]
+
 [features]
 default = ["fastembed", "huggingface"]
 fastembed = ["octolib/fastembed"]
 huggingface = ["octolib/huggingface"]
+# Compiles the beir_bench binary. No extra deps — uses serde_json + octolib,
+# already pulled in by the library.
+bench = []
 
 [dependencies]
 lancedb = { version = "0.26.2", default-features = false }
@@ -75,6 +87,19 @@ regex = "1"
 # which conflicts with tantivy-common 0.9.0's blanket `From<B> for FileSlice`, causing E0119.
 time = { version = "=0.3.47", default-features = false }
 
+# Hold the transitive brotli compression chain (via reqwest / tower-http / lance) on
+# alloc-no-stdlib 2.x. brotli 8.0.3 uses alloc-no-stdlib "2.0" directly but takes its
+# `StandardAlloc` from alloc-stdlib; alloc-stdlib 0.2.3 and brotli-decompressor 5.0.2
+# both moved to alloc-no-stdlib 3.0, so a re-resolve hands StandardAlloc the 3.0
+# `Allocator` trait while brotli expects 2.0.4's — breaking `impl BrotliAlloc for
+# StandardAlloc` (E0277, "multiple versions of alloc_no_stdlib"). Pinning these two
+# leaf crates to their last 2.0-only releases collapses the chain to a single
+# alloc-no-stdlib 2.0.4. NB: pinning alloc-no-stdlib itself does NOT help — cargo lets
+# 2.x and 3.x coexist; the crates that SELECT the version must be pinned, and both
+# (not just one) — leaving either unpinned lets it re-introduce the 3.0 line.
+alloc-stdlib = "=0.2.2"
+brotli-decompressor = "=5.0.1"
+
 # Local dep for testing
 # octolib = { path = "../octolib", default-features = false }
 octolib = { version = "0.23.0", default-features = false }

README.md

@@ -225,6 +225,29 @@ See [MCP Integration](#mcp-integration) for Claude Desktop setup.
 - **Query Expansion (HyDE-lite)** — Pseudo-relevance feedback for +10-30% recall on long-tail queries
 - **MCP Protocol** — Full MCP 2025-03-26 compliance for AI tool integration
 
+## Benchmarks
+
+Retrieval quality of octobrain's knowledge system on standard [BEIR](https://github.com/beir-cellar/beir) datasets — **nDCG@10**, fully local, no LLM judge, using the default local embedder `bge-small-en-v1.5` (384-dim, 33M params). Each corpus passage is indexed through octobrain's real retrieval path and scored against the official qrels (metrics reproduce `pytrec_eval`).
+
+| Dataset | octobrain vector | octobrain hybrid | BM25¹ | bge-small-en-v1.5² |
+|---|---|---|---|---|
+| **SciFact** (5.2K docs, 300 q) | 0.722 | **0.742** | 0.665 | 0.713 |
+| **NFCorpus** (3.6K docs, 323 q) | 0.341 | **0.363** | 0.325 | 0.343 |
+
+- **vector** = dense-only retrieval; reproduces the embedder's published BEIR numbers (validates the harness).
+- **hybrid** = BM25 + vector fused with Reciprocal Rank Fusion (k=60) — octobrain's default. Adds **+2 nDCG@10** over the bare embedding and beats classic BM25 on both datasets.
+
+¹ Canonical BM25 from the [BEIR paper](https://arxiv.org/abs/2104.08663) (Anserini/Lucene, k1=0.9 b=0.4).
+² From the [bge-small-en-v1.5 model card](https://huggingface.co/BAAI/bge-small-en-v1.5) (MTEB).
+
+> Scope: this measures the **ranking** layer (embedding + BM25 fusion + reranking). BEIR passages are pre-chunked, so octobrain's chunking strategy is not exercised here.
+
+Reproduce (downloads the datasets, builds a release binary, runs fully offline):
+
+```bash
+cd benches && bash scripts/run_retrieval.sh
+```
+
 ## Configuration
 
 Configuration is stored in `~/.local/share/octobrain/config.toml`. All options have sensible defaults.

benches/.gitignore

@@ -4,3 +4,5 @@ results/*
 !results/README.md
 __pycache__/
 *.pyc
+# Downloaded BEIR datasets (fetched on demand by scripts/run_retrieval.sh)
+data/

benches/README.md

@@ -1,17 +1,64 @@
 # Octobrain Benchmarks
 
-Reproducible quality benchmarks for octobrain's memory architecture. Everything
-runs in Docker — no host install, no Python on your machine. Bring your own
-LLM (any OpenAI-compatible endpoint: Ollama Cloud, OpenAI, Together, Groq, etc).
+Reproducible quality benchmarks for octobrain. Two kinds live here:
+**retrieval-quality** (knowledge system) is native Rust, fully local, metric-only
+— no Docker, no LLM (see [below](#retrieval-quality-knowledge-system)). The
+**memory** benchmarks (LongMemEval etc.) are LLM-judged and run in Docker —
+bring your own LLM (any OpenAI-compatible endpoint).
 
 ## Status
 
 | Benchmark | License | Status | Notes |
 |---|---|---|---|
+| [BEIR retrieval](https://github.com/beir-cellar/beir) | Apache-2.0 | wired | Knowledge-system nDCG@10; local, no LLM, no Docker |
 | [LongMemEval](https://github.com/xiaowu0162/longmemeval) | MIT | wired | 500 questions across 6 memory abilities |
 | [Memora](https://github.com/geniesinc/Memora) | tbd | planned | "From Recall to Forgetting" — FAMA metric |
 | [LoCoMo](https://github.com/snap-research/locomo) | research | planned | 35-session multi-modal benchmark |
 
+## Retrieval quality (knowledge system)
+
+Measures how well octobrain's knowledge retrieval ranks relevant passages on
+standard [BEIR](https://github.com/beir-cellar/beir) datasets, reported as
+**nDCG@10 / Recall@10 / Recall@100 / MRR@10** against the official qrels (metrics
+reproduce `pytrec_eval`: linear gain, log2 discount, score-desc / docid-desc
+ordering, graded relevance for NFCorpus). No LLM judge — just the embedder + the
+real `KnowledgeStore` retrieval path. The same index is queried three ways:
+`vector` (dense only), `hybrid` (BM25 + vector RRF), `hybrid+rerank` (cross-encoder).
+
+```bash
+cd benches
+bash scripts/run_retrieval.sh                              # scifact + nfcorpus, bge-small
+DATASETS="scifact" bash scripts/run_retrieval.sh          # one dataset
+EMBED_MODEL="fastembed:nomic-ai/nomic-embed-text-v1.5" bash scripts/run_retrieval.sh
+BEIR_MAX_QUERIES=50 bash scripts/run_retrieval.sh         # quick smoke
+```
+
+The harness builds `cargo run --release --features bench --bin beir_bench`,
+downloads each BEIR zip into `benches/data/`, indexes the corpus **once** per
+`(dataset, embedding model)` (cached under the system temp dir; `BEIR_FRESH=1`
+to rebuild), and writes `results.jsonl` + per-scenario logs under
+`benches/results/retrieval-<ts>/`. Env knobs: `BEIR_SCENARIOS` (subset of
+`vector,hybrid,hybrid+rerank`), `BEIR_RERANK_DEPTH` (default 50), `RERANK_MODEL`.
+
+Latest numbers (bge-small-en-v1.5, default config) are in the top-level
+[README Benchmarks section](../README.md#benchmarks).
+
+### Findings
+
+- **vector / hybrid validated**: dense-only nDCG@10 reproduces the embedder's
+  published BEIR numbers (SciFact 0.722 vs 0.713, NFCorpus 0.341 vs 0.343), and
+  hybrid (BM25 + vector RRF, the default) adds a consistent ~+2 nDCG, beating
+  classic BM25 on both. The harness is trustworthy.
+- **⚠️ reranker is a no-op (needs investigation)**: `hybrid+rerank` returned
+  *byte-identical* metrics to `hybrid` for **two different** fastembed
+  cross-encoders (`jina-reranker-v2-base-multilingual` and `bge-reranker-base`),
+  with zero errors. octolib sorts results by score, so identical output across
+  two models means the fastembed reranker path yields degenerate scores that
+  never reorder. The default config **enables this reranker for memory and
+  knowledge search**, so it is silently doing nothing — confirm with a focused
+  octolib/fastembed-rs repro and fix (or switch reranker provider) before
+  relying on reranking. Rerank is therefore *excluded* from the headline numbers.
+
 ## Quick start
 
 ```bash

benches/scripts/run_retrieval.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#
+# Local, no-LLM retrieval-quality benchmark for octobrain's knowledge system.
+# Downloads small BEIR datasets, then runs the REAL KnowledgeStore retrieval
+# path (vector / hybrid / hybrid+rerank) over them and reports nDCG@10 /
+# Recall@k / MRR@10 against qrels. Each scenario is just a config file fed via
+# OCTOBRAIN_CONFIG_PATH, generated from config-templates/default.toml so there
+# is a single source of truth for the non-varying knobs.
+#
+# Usage:
+#   benches/scripts/run_retrieval.sh                 # default: scifact + nfcorpus, bge-small trio
+#   DATASETS="scifact" benches/scripts/run_retrieval.sh
+#   EMBED_MODEL="fastembed:nomic-ai/nomic-embed-text-v1.5" benches/scripts/run_retrieval.sh
+#   BEIR_MAX_QUERIES=50 benches/scripts/run_retrieval.sh   # quick smoke
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BENCH_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO_DIR="$(cd "$BENCH_DIR/.." && pwd)"
+TEMPLATE="$REPO_DIR/config-templates/default.toml"
+DATA_DIR="$BENCH_DIR/data"
+
+DATASETS="${DATASETS:-scifact nfcorpus}"
+EMBED_MODEL="${EMBED_MODEL:-fastembed:BAAI/bge-small-en-v1.5}"
+RERANK_MODEL="${RERANK_MODEL:-fastembed:jina-reranker-v2-base-multilingual}"
+BEIR_BASE_URL="https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets"
+
+BIN="$REPO_DIR/target/release/beir_bench"
+TS="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
+OUT_DIR="$BENCH_DIR/results/retrieval-$TS"
+mkdir -p "$OUT_DIR" "$DATA_DIR"
+RESULTS="$OUT_DIR/results.jsonl"
+
+# Build once if needed.
+if [ ! -x "$BIN" ]; then
+  echo "[run] building beir_bench (release)…" >&2
+  ( cd "$REPO_DIR" && cargo build --release --features bench --bin beir_bench )
+fi
+
+# Fetch a BEIR dataset zip if not already present.
+fetch() {
+  local ds="$1"
+  if [ ! -d "$DATA_DIR/$ds" ]; then
+    echo "[run] downloading $ds…" >&2
+    curl -sSL "$BEIR_BASE_URL/$ds.zip" -o "$DATA_DIR/$ds.zip"
+    unzip -q -o "$DATA_DIR/$ds.zip" -d "$DATA_DIR"
+    rm -f "$DATA_DIR/$ds.zip"
+  fi
+}
+
+# Generate a scenario config from the template, overriding only the knobs that
+# vary. awk tracks the current [section] so it edits the right `enabled`/`model`.
+gen_config() {
+  local out="$1" embed="$2" hybrid="$3" rerank="$4" rerank_model="$5"
+  awk -v emb="$embed" -v hyb="$hybrid" -v rrk="$rerank" -v rrkm="$rerank_model" '
+    /^\[/ { sect=$0 }
+    sect=="[embedding]" && /^model =/ { print "model = \"" emb "\""; next }
+    sect=="[search.hybrid]" && /^enabled =/ { print "enabled = " hyb; next }
+    sect=="[search.reranker]" && /^enabled =/ { print "enabled = " rrk; next }
+    sect=="[search.reranker]" && /^model =/ { print "model = \"" rrkm "\""; next }
+    { print }
+  ' "$TEMPLATE" > "$out"
+}
+
+echo "[run] embedding: $EMBED_MODEL" >&2
+echo "[run] datasets:  $DATASETS" >&2
+: > "$RESULTS"
+
+# One config per embedding model — the bench indexes each corpus once and sweeps
+# vector / hybrid / hybrid+rerank internally, emitting one JSON line per scenario.
+cfg="$OUT_DIR/config.toml"
+gen_config "$cfg" "$EMBED_MODEL" "true" "true" "$RERANK_MODEL"
+
+for ds in $DATASETS; do
+  fetch "$ds"
+  echo "[run] === $ds ===" >&2
+  OCTOBRAIN_CONFIG_PATH="$cfg" "$BIN" "$DATA_DIR/$ds" "$ds" \
+    | tee "$OUT_DIR/log-$ds.txt" \
+    | grep '^JSON ' | sed 's/^JSON //' >> "$RESULTS" || true
+done
+
+echo >&2
+echo "[run] results → $RESULTS" >&2
+# Compact table to stdout.
+python3 - "$RESULTS" <<'PY'
+import json, sys
+rows = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
+if not rows:
+    print("no results"); sys.exit(0)
+w = max(len(r["label"]) if "label" in r else 0 for r in rows)
+hdr = f'{"label".ljust(w)}  {"nDCG@10":>8} {"R@10":>7} {"R@100":>7} {"MRR@10":>7}  scenario'
+print(hdr); print("-"*len(hdr))
+for r in rows:
+    print(f'{r["label"].ljust(w)}  {r["ndcg@10"]:>8.4f} {r["recall@10"]:>7.4f} '
+          f'{r["recall@100"]:>7.4f} {r["mrr@10"]:>7.4f}  hybrid={r["hybrid"]} rerank={r["reranker"]}')
+PY

config-templates/default.toml

@@ -35,12 +35,16 @@ max_tokens_per_batch = 100000
 timeout_secs = 30
 
 [search]
-# Similarity threshold for memory search (0.0 to 1.0)
-# Lower values = more results, higher values = fewer but more relevant
+# Default minimum relevance for SEMANTIC (text) queries — a memory must score at
+# least this to be returned. Filter-only listings (recent, by-type, by-tags) are
+# unaffected. A query may override this per-call via its own min_relevance.
+# Lower = more results, higher = fewer but more relevant. Range 0.0-1.0.
 # Default: 0.3
 similarity_threshold = 0.3
 
-# Maximum number of results to return from search
+# Hard ceiling on results returned from any search — the search layer never
+# returns more than this, even if a caller requests a larger limit. The default
+# page size (when no limit is given) is memory.max_search_results below.
 # Default: 50
 max_results = 50
 
@@ -128,7 +132,8 @@ auto_cleanup_days = 365
 # Default: 0.1
 cleanup_min_importance = 0.1
 
-# Maximum memories returned in search
+# Default page size for search — how many results to return when a caller gives
+# no explicit limit. The absolute hard ceiling is search.max_results above.
 # Default: 50
 max_search_results = 50