From dec0a1d1dfcd0d486510c0b895f961f3ac1ffe3f Mon Sep 17 00:00:00 2001 From: Aidas Agejevas Date: Mon, 20 Jul 2026 15:30:33 -0700 Subject: [PATCH] docs: plain voice pass over comments, docstrings, and docs Rewrite generated prose in plain declarative sentences: 635 em dashes and 22 double-hyphen dashes removed across Python, Rust, Swift, and markdown, each a real rewrite (sentence splits, comma and parenthetical folds, label colons), never a punctuation swap. String literals, table cells, code fences, and the canonical-JSON fixtures keep their dashes. Also drop two roadmap references from public doc-comments. --- CHANGELOG.md | 72 ++++++++--------- README.md | 22 +++--- benchmarks/README.md | 4 +- benchmarks/batched_retrieval/README.md | 14 ++-- benchmarks/batched_retrieval/diagrams.py | 2 +- benchmarks/batched_retrieval/run.py | 6 +- benchmarks/direct_gpu_sweep/modal_bench.py | 2 +- benchmarks/filtered_batch/README.md | 4 +- benchmarks/filtered_batch/modal_bench.py | 4 +- benchmarks/govreport_scale/README.md | 42 +++++----- benchmarks/govreport_scale/modal_bench.py | 2 +- .../turbovec_govreport_scale.py | 8 +- benchmarks/gpu_vanilla_vs_augmented/README.md | 62 +++++++-------- .../gpu_vanilla_vs_augmented/diagrams.py | 14 ++-- .../gpu_vanilla_vs_augmented/modal_bench.py | 10 +-- .../gpu_vanilla_vs_augmented/run_bench.py | 2 +- .../turbovec_vva_bench.py | 28 +++---- .../turbovec_vva_runner.py | 4 +- benchmarks/graph_memory/graph_memory_bench.py | 8 +- benchmarks/hybrid/hybrid_recall.py | 2 +- benchmarks/laptop/README.md | 14 ++-- benchmarks/laptop/diagrams.py | 2 +- benchmarks/laptop/run.py | 2 +- benchmarks/late_interaction/README.md | 6 +- benchmarks/memory_integrations/README.md | 14 ++-- benchmarks/mps_vs_neon/README.md | 14 ++-- benchmarks/mps_vs_neon/diagrams.py | 2 +- benchmarks/mps_vs_neon/run.py | 8 +- .../lodedb-cloud-core/src/artifact_store.rs | 16 ++-- crates/lodedb-cloud-core/src/blob_layout.rs | 10 +-- crates/lodedb-cloud-core/src/client_ops.rs | 52 ++++++------- crates/lodedb-cloud-core/src/digest.rs | 4 +- crates/lodedb-cloud-core/src/error.rs | 6 +- .../src/generation_inventory.rs | 42 +++++----- crates/lodedb-cloud-core/src/lib.rs | 2 +- .../src/local_artifact_store.rs | 14 ++-- crates/lodedb-cloud-core/src/managed.rs | 28 +++---- .../src/manifest_transfer.rs | 24 +++--- .../src/object_artifact_store.rs | 32 ++++---- crates/lodedb-cloud-core/src/paths.rs | 6 +- .../src/snapshot_identity.rs | 14 ++-- crates/lodedb-cloud-core/src/status.rs | 18 ++--- crates/lodedb-cloud-core/src/store_target.rs | 6 +- crates/lodedb-cloud-core/src/sync_plan.rs | 22 +++--- crates/lodedb-cloud-core/src/sync_state.rs | 21 ++--- .../lodedb-cloud-core/src/transfer_policy.rs | 18 ++--- crates/lodedb-cloud-core/src/verify.rs | 10 +-- .../lodedb-cloud-core/tests/artifact_store.rs | 8 +- crates/lodedb-cloud-core/tests/client_ops.rs | 4 +- crates/lodedb-cloud-core/tests/common/mod.rs | 6 +- .../tests/generation_inventory.rs | 14 ++-- crates/lodedb-cloud-core/tests/managed.rs | 4 +- .../tests/object_artifact_store.rs | 4 +- crates/lodedb-cloud-core/tests/restore.rs | 4 +- .../tests/snapshot_identity.rs | 4 +- crates/lodedb-cloud-core/tests/sync_ops.rs | 22 +++--- crates/lodedb-cloud-core/tests/sync_plan.rs | 8 +- .../lodedb-cloud-core/tests/verify_status.rs | 2 +- crates/lodedb-core/src/engine/mod.rs | 14 ++-- .../src/storage/commit_manifest.rs | 10 +-- crates/lodedb-core/src/storage/mod.rs | 2 +- crates/lodedb-core/src/storage/util.rs | 4 +- crates/lodedb-core/src/storage/wal.rs | 6 +- crates/lodedb-core/src/vector/turbovec.rs | 2 +- crates/lodedb-core/tests/core_engine.rs | 4 +- crates/lodedb-ffi/src/lib.rs | 4 +- crates/lodedb-gpu/src/lib.rs | 8 +- docs/architecture.md | 40 +++++----- docs/benchmarks.md | 32 ++++---- docs/graph.md | 4 +- docs/integrations.md | 18 ++--- docs/late-interaction.md | 22 +++--- docs/native_core_wal_payloads.md | 4 +- src/lodedb/__init__.py | 2 +- src/lodedb/cloud/__init__.py | 14 ++-- src/lodedb/cloud/_agents_scaffold.py | 10 +-- src/lodedb/cloud/_config.py | 10 +-- src/lodedb/cloud/_env_file.py | 2 +- src/lodedb/cloud/cli.py | 78 +++++++++---------- src/lodedb/engine/_atomic_io.py | 4 +- src/lodedb/engine/_commit_manifest.py | 12 +-- src/lodedb/engine/_filelock.py | 8 +- src/lodedb/engine/_lexical.py | 6 +- src/lodedb/engine/_predicate.py | 8 +- src/lodedb/engine/gpu_turbovec.py | 6 +- src/lodedb/engine/route_registry.py | 2 +- src/lodedb/graph/__init__.py | 2 +- src/lodedb/graph/_store.py | 4 +- src/lodedb/graph/knowledge_graph.py | 8 +- src/lodedb/local/__init__.py | 10 +-- src/lodedb/local/backends.py | 2 +- src/lodedb/local/benchmark.py | 4 +- src/lodedb/local/cli.py | 6 +- src/lodedb/local/db.py | 56 ++++++------- src/lodedb/local/integrations/__init__.py | 18 ++--- src/lodedb/local/integrations/kotaemon.py | 10 +-- src/lodedb/local/integrations/langchain.py | 4 +- src/lodedb/local/integrations/llama_index.py | 2 +- .../local/integrations/llama_index_graph.py | 6 +- src/lodedb/local/integrations/privategpt.py | 10 +-- src/lodedb/local/mcp_install.py | 10 +-- src/lodedb/local/mcp_server.py | 4 +- src/lodedb/local/migrate/detect.py | 8 +- src/lodedb/local/migrate/report.py | 4 +- src/lodedb/local/migrate/runner.py | 2 +- src/lodedb/local/migrate/sources/__init__.py | 8 +- src/lodedb/local/migrate/sources/base.py | 4 +- src/lodedb/local/onnx_artifacts.py | 6 +- swift/LodeDBCore/Package.swift | 2 +- .../Sources/LodeDBCore/Appender.swift | 2 +- .../LodeDBCore/MetalVectorScanner.swift | 2 +- .../Sources/LodeDBCore/NativeCoreBridge.swift | 2 +- tests/test_cloud_agents_scaffold.py | 4 +- tests/test_cloud_client_api.py | 2 +- tests/test_cloud_connect_targets.py | 8 +- tests/test_cloud_credentials.py | 4 +- tests/test_cloud_env_file.py | 2 +- tests/test_cloud_import_root.py | 6 +- tests/test_cloud_routing_hint.py | 6 +- tests/test_cloud_store_verbs.py | 6 +- tests/test_cloud_surface_parity.py | 2 +- tests/test_cloud_sync.py | 6 +- tests/test_cloud_target_dispatch.py | 12 +-- tests/test_cloud_trampoline.py | 6 +- tests/test_import_boundary.py | 10 +-- tests/test_kotaemon_adapter.py | 4 +- tests/test_local_backends.py | 2 +- tests/test_local_cli_and_server.py | 2 +- tests/test_local_enumeration.py | 2 +- tests/test_local_persistence_recovery.py | 2 +- tests/test_local_raw_text.py | 2 +- tests/test_local_vector_in.py | 2 +- tests/test_segments.py | 2 +- tests/test_wal_commit_mode.py | 4 +- 134 files changed, 714 insertions(+), 713 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce7d47d..98305231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 user's store over HTTPS, duck-typing the local verb surface), the transfer verbs (`push`/`pull`/`sync`/`status`/`verify`/`keys`) on a native transfer core bundled into the extension as `lodedb._turbovec.cloud` (one - commit-format implementation shared with the engine — the new + commit-format implementation shared with the engine, the new `crates/lodedb-cloud-core`, which also learns the 1.3.2 `tvvf` rescore sidecar so rescore stores transfer completely), and the full `lodedb cloud` CLI (login, tokens, store/environment/org management, transfer, @@ -30,8 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `lodedb cloud login`), and the `"org/environment/store"` triple and full `orecloud://` URLs are accepted too. The call returns the cloud store handle (same `add`/`search`/`get`/`remove` verbs over HTTPS). For config-driven code, - the plain constructor also dispatches the explicit URL form — - `LodeDB("orecloud://org/environment/store")` — through the same funnel. + the plain constructor also dispatches the explicit URL form + `LodeDB("orecloud://org/environment/store")` through the same funnel. Local-only construction options (`model=`, `read_only=`, ...) are rejected on cloud targets with a targeted error, and a cloud target without the `[cloud]` extra's dependencies raises the install hint. @@ -62,7 +62,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`open_vector_store` / `LodeDB(vector_dim=...)`) that retains text (`store_text=True`, the default, or `index_text=True`): they rank the text carried on the stored vectors with LodeDB's Okapi BM25, no embedder needed. `mode="vector"`/`"hybrid"` still raise - `VectorOnlyIndexError` on such a store (they must embed the query — use `search_by_vector`), + `VectorOnlyIndexError` on such a store (they must embed the query; use `search_by_vector`), and an unset `mode` resolves to `"lexical"` there instead of `"hybrid"`. This lets vector-in integrations (mem0, Haystack, any external embedder) offer keyword search through the public API instead of reaching into engine internals. No on-disk format change. @@ -75,15 +75,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **WAL segment primitives for out-of-band ingest (`lodedb.local.segments`).** Store-free planning (`plan_documents`), record building (`build_embedded_documents_record`, `delete_documents_record`), immutable WAL-format segment encode/decode - (`encode_segment`/`decode_segment`, strict), and `fold_segment` — fold a downloaded - segment into a warm writable `commit_mode="generation"` handle, stamping LSNs at fold - time and publishing one O(changed) generation delta per batch via `LodeDB.persist()`. + (`encode_segment`/`decode_segment`, strict), and `fold_segment`, which folds a + downloaded segment into a warm writable `commit_mode="generation"` handle, stamps LSNs + at fold time, and publishes one O(changed) generation delta per batch via `LodeDB.persist()`. Core: `plan_documents`, `build_embedded_documents_payload`, `is_native_replayable_op`, and `CoreEngine::apply_wal_records` in `lodedb-core`; segments reuse the WAL frame format - byte-for-byte and carry raw text only under `store_text=True`. Advanced API — not + byte-for-byte and carry raw text only under `store_text=True`. Advanced API, not re-exported from the package root. -- **`LodeDB.discard()` — close without persisting.** Releases the handle and its writer +- **`LodeDB.discard()`: close without persisting.** Releases the handle and its writer lock while dropping un-persisted in-memory state, leaving the store at its last committed generation. The abort path after a failed `fold_segment` batch, where the in-memory state may be partially applied and a graceful `close()` would persist it. @@ -95,14 +95,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A document added and removed between two persists no longer bricks the store.** The vector delta recorded the never-committed row's removal, and the strict delta replay - ("removed-id count mismatch") then rejected the store on every fresh open — the warm + ("removed-id count mismatch") then rejected the store on every fresh open. The warm handle kept serving while every new reader failed. The add+remove now cancel out of the delta; a *committed* row replaced or removed in the same window still records its removal. Reachable from any multi-record fold (`Checkpointer` over appended WAL records, `fold_segment` batches); found by a randomized concurrency test in the cloud companion. - **Duplicate document ids within one ingest batch are refused.** A repeated id in one `plan_documents` batch built a single `apply_embedded_documents` record whose earlier - occurrence's vector row survived the fold with no owner — after reopen, any query + occurrence's vector row survived the fold with no owner. After reopen, any query touching the orphan row failed with "TurboVec returned an unknown stable id". The planner, the payload builder, and the replay boundary now all reject the shape; callers that mean "replace" should keep the last occurrence or split the batch. @@ -227,11 +227,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **The native Rust core (TurboVec) is now the sole engine.** The Python `LodeEngine` and - `LodeIndex` are removed; every read and write — text and vector ingest, search + `LodeIndex` are removed; every read and write runs through the in-process Rust + `CoreEngine`, with no Python fallback. That covers text and vector ingest, search (vector / hybrid / lexical), filters, batch, late-interaction, persistence, and the - WAL/generation commit — runs through the in-process Rust `CoreEngine`, with no Python - fallback. The public Python API is unchanged but for one additive change: `list_documents` - gains `after`/`limit` keyset paging. MPS vector scanning was dropped (MPS embedding via + WAL/generation commit. The public Python API is unchanged but for one additive change: + `list_documents` gains `after`/`limit` keyset paging. MPS vector scanning was dropped (MPS embedding via PyTorch is unaffected). ### Added @@ -367,7 +367,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **LlamaIndex `VectorStore` adapter** (`lodedb[llama-index]`). `LodeDBVectorStore` in `lodedb.local.integrations.llama_index` wraps the LodeDB SDK as a LlamaIndex - `BasePydanticVectorStore`, joining the existing LangChain adapter. It is *text-path* — + `BasePydanticVectorStore`, joining the existing LangChain adapter. It is *text-path*: LodeDB embeds text internally (`is_embedding_query=False`), so LlamaIndex's own `embed_model` is not used. Query modes map onto the SDK's retrieval modes: `VectorStoreQueryMode.DEFAULT` to vector search, `HYBRID`/`SEMANTIC_HYBRID` to the BM25 + RRF @@ -460,19 +460,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 exact-match, so existing filters are unchanged. Ordered comparisons (`$gt`/`$gte`/`$lt`/`$lte`) are numeric when both the stored value and the operand parse as numbers, otherwise lexicographic; equality and membership (`$eq`/`$ne`/`$in`/`$nin`) always compare as strings, so - `{"n": {"$eq": 9.9}}` does not match a stored `9.90`. No storage-format change — the predicates - evaluate at query time over the existing string metadata. + `{"n": {"$eq": 9.9}}` does not match a stored `9.90`. There is no storage-format change; + the predicates evaluate at query time over the existing string metadata. - **Single-writer concurrency safety.** A LodeDB handle now holds an exclusive OS advisory lock (`/.lodedb.lock`) for its lifetime, so concurrent processes can no longer corrupt - the on-disk store. A second open of the same path waits for the first to close — then loads - the accumulated state and composes — and fails fast with `ConcurrentWriterError` once + the on-disk store. A second open of the same path waits for the first to close (then + loads the accumulated state and composes) and fails fast with `ConcurrentWriterError` once `LODEDB_PERSIST_LOCK_TIMEOUT` (default 30s) elapses, the model SQLite uses with a busy timeout. The kernel releases the lock on process exit, so a crash never wedges the path. Local filesystems only (advisory locks are unreliable on NFS/SMB). Live cross-process refresh (a reader auto-seeing another live process's writes) remains out of scope. - **Read-only handles (single writer, many readers).** `LodeDB.open_readonly(path)` (or `read_only=True`) opens a non-mutating snapshot that takes **no** writer lock, so it can - read a path while a writer holds it — `lodedb query` and `lodedb get` now use it, so they + read a path while a writer holds it. `lodedb query` and `lodedb get` now use it, so they work alongside a running `lodedb serve`/`mcp`. Mutating calls raise `ReadOnlyError`; the path must already exist. A read-only open loads the single consistent generation named by the atomic commit manifest (below), so it never observes a torn cross-file mix. @@ -487,15 +487,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (JSON state base + `.jsd` journal, `.tvim` vector base + `.tvd` journal, and the opt-in `.tvtext` raw-text base + `.txd` journal); they are now written as generation-addressed artifacts under a per-index `.gen/` directory and sealed by atomically swapping a single - `.commit.json` root pointer — that swap is the only commit point. Because every + `.commit.json` root pointer; that swap is the only commit point. Because every artifact (including raw text, on by default) is generation-addressed and pinned by the root, none is overwritten in place, so a crash (or `kill -9`) mid-commit leaves the previously committed generation fully intact and the next open rolls back to it (dropping the uncommitted artifacts) instead of failing closed and stuck. Lock-free readers load exactly - the generation the root manifest names — consistent snapshot isolation (text included), no - torn cross-file reads. Stores written by v0.1.x load via a legacy fallback and migrate to - the new layout on their next write; superseded generations are garbage-collected (the most - recent few are retained for in-flight readers). + the generation the root manifest names, so they get consistent snapshot isolation (text + included) and never a torn cross-file read. Stores written by v0.1.x load via a legacy + fallback and migrate to the new layout on their next write; superseded generations are + garbage-collected (the most recent few are retained for in-flight readers). - **First-class opt-in Apple-GPU (MPS) exact scan.** The Metal/MPS resident scan is now a selectable route at CUDA-level capability, off by default: in-place `patch()` on small mutations (O(changed) swap-remove + batched upsert), an `MpsDirectTurboVecPolicy` @@ -503,7 +503,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 an optional `LODEDB_MPS_MEMORY_BUDGET_BYTES` guard, and honest `lodedb doctor` reporting. MPS uses shared resident-scan helper code for deterministic top-k ordering and tile sizing; CUDA extraction onto those helpers remains a hardware-verified follow-up. **NEON remains the - default on Apple Silicon** — the MPS scan was slower than NEON at every batch size on the + default on Apple Silicon.** The MPS scan was slower than NEON at every batch size on the measured M1, so it is opt-in and off by default; any future default flip is gated on per-chip `benchmarks/mps_vs_neon` crossover data, especially for newer Apple GPUs such as M5. @@ -546,8 +546,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **GPU (`[gpu]`) resident copy now patches in place on small mutations** instead of rebuilding the whole dequantized array. Adds and removes apply in O(changed) rows (swap-remove + batched upsert) with a fail-closed rebuild fallback, so syncing a small - delta into a large GPU-resident index is dramatically cheaper — e.g. ~560× faster at - 1,000 changed rows over a 1M-row corpus on an A10 — with identical top-k results. + delta into a large GPU-resident index is dramatically cheaper (e.g. ~560× faster at + 1,000 changed rows over a 1M-row corpus on an A10), with identical top-k results. ### Fixed @@ -570,15 +570,15 @@ First public release. - **Delta persistence**: on-disk index (`.tvim` / `.tvd`) plus a journal (`.jsd`) for fast incremental updates, and an optional `.tvtext` raw-text sidecar gated by `store_text` (default `True`; set `store_text=False` to keep no document text on disk). -- **Python API** — `LodeDB` with `add`, `search`, `search_many`, `get`, and `persist`; +- **Python API**: `LodeDB` with `add`, `search`, `search_many`, `get`, and `persist`; results returned as `LodeSearchHit`. -- **`lodedb` CLI** — `doctor`, `index`, `query`, `get`, `benchmark`, `serve`, `mcp`. -- **Local HTTP dev server** (`lodedb serve`) — loopback-only, no auth, metrics-only +- **`lodedb` CLI**: `doctor`, `index`, `query`, `get`, `benchmark`, `serve`, `mcp`. +- **Local HTTP dev server** (`lodedb serve`): loopback-only, no auth, metrics-only telemetry; exposes `/healthz`, `/stats`, `/search`, and `/get`. - **Optional extras**: - - `[gpu]` — opt-in CUDA-resident exact scan (cupy; Linux/CUDA only). - - `[mcp]` — stdio MCP server so coding agents can use LodeDB as local memory. - - `[langchain]` — LangChain `VectorStore` adapter. + - `[gpu]`: opt-in CUDA-resident exact scan (cupy; Linux/CUDA only). + - `[mcp]`: stdio MCP server so coding agents can use LodeDB as local memory. + - `[langchain]`: LangChain `VectorStore` adapter. ### Notes @@ -586,7 +586,7 @@ First public release. `pyyaml`. Heavier research dependencies are not imported on the core path; this is enforced by `tests/test_import_boundary.py`. - LodeDB is licensed under Apache-2.0. The vendored TurboVec core - (`third_party/turbovec/`) is MIT — see [`NOTICE`](NOTICE). + (`third_party/turbovec/`) is MIT; see [`NOTICE`](NOTICE). [Unreleased]: https://github.com/Egoist-Machines/LodeDB/compare/v0.1.2...HEAD [0.1.2]: https://github.com/Egoist-Machines/LodeDB/compare/v0.1.1...v0.1.2 diff --git a/README.md b/README.md index 298b9c45..e83f67f4 100644 --- a/README.md +++ b/README.md @@ -632,23 +632,23 @@ db.add("the quick brown fox") # embedded server-side db.search("fox", k=5) ``` -A bare store id is enough — the org/environment half resolves from the +A bare store id is enough. The org/environment half resolves from the credential (`token=`, the `ORECLOUD_TOKEN` environment variable, or `lodedb cloud login`); pass the full `"org/environment/store"` triple in cross-environment scripts. For config-driven code, where one string field (an env var, a YAML value) must express either a local path or a cloud store, the -plain constructor accepts the explicit URL form — -`LodeDB("orecloud://org/environment/store")` — and returns the same handle. +plain constructor accepts the explicit URL form +`LodeDB("orecloud://org/environment/store")` and returns the same handle. An application serving many end users should hold one `Client` and open -per-user handles from it (`Client().store(user_id)` — one credential -resolution, one shared HTTP pool); `from lodedb.cloud import Client` works as -the import root. +per-user handles from it (`Client().store(user_id)`); the credential resolves +once and the handles share one HTTP pool. `from lodedb.cloud import Client` +works as the import root. Work locally with `lodedb` exactly as before. The client is first-party code -in this package — `lodedb.cloud` and the `lodedb cloud` CLI, with push/pull/ -sync running on the same bundled native core as the engine — and the extra -installs only its dependencies (`httpx`, `pynacl`). Everything cloud loads -lazily, so a plain `import lodedb` stays network-free. +in this package (`lodedb.cloud` and the `lodedb cloud` CLI), and push/pull/ +sync run on the same bundled native core as the engine. The extra installs +only its dependencies (`httpx`, `pynacl`). Everything cloud loads lazily, so +a plain `import lodedb` stays network-free. ## Concurrency & durability @@ -693,7 +693,7 @@ lazily, so a plain `import lodedb` stays network-free. read-your-writes against an append's returned LSN). On Windows the shared lock degrades to an exclusive hold, so appenders serialize there rather than coexisting. A record is a precomputed vector plus metadata (with an optional caption, e.g. for an image, retained only when the appender - opts into `store_text` -- off by default, so no raw text reaches the WAL); with an embedder + opts into `store_text`; it is off by default, so no raw text reaches the WAL); with an embedder configured, the appender also ingests full text (chunked by the core, embedded in the binding layer, then logged as a post-embedding record) so text writes are multi-producer too, without a captured base generation. It requires WAL commit mode. Exposed as the native `CoreAppender`, over diff --git a/benchmarks/README.md b/benchmarks/README.md index 6ec2a42e..e167d373 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,7 +1,7 @@ # Benchmarks Self-contained benchmarks. All artifacts are **metrics-only** (counts, bytes, latency, -ids, backend labels — never raw documents, queries, or embeddings), and each folder has its +ids, backend labels; never raw documents, queries, or embeddings), and each folder has its own README with the exact reproduction command, hardware, sample sizes, and results. Provenance is tagged inline in [`../docs/benchmarks.md`](../docs/benchmarks.md): `measured` = timed on the stated machine, `recorded` = read from the environment. No estimates. @@ -22,5 +22,5 @@ Headline launch and laptop numbers (with figures) are collected in [`../docs/benchmarks.md`](../docs/benchmarks.md); per-benchmark figures live in each `*/docs/`. -> Figures are regenerated by each benchmark's `diagrams.py`, which needs `matplotlib` — a +> Figures are regenerated by each benchmark's `diagrams.py`, which needs `matplotlib`, a > dev-only tool, not a LodeDB runtime dependency: `uv pip install matplotlib`. diff --git a/benchmarks/batched_retrieval/README.md b/benchmarks/batched_retrieval/README.md index 198ff8cc..9148562a 100644 --- a/benchmarks/batched_retrieval/README.md +++ b/benchmarks/batched_retrieval/README.md @@ -1,4 +1,4 @@ -# Batched-retrieval throughput — `search_many` queries/sec +# Batched-retrieval throughput: `search_many` queries/sec A metrics-only benchmark of the **public SDK path** `LodeDB.search_many(queries, k=...)`: how many queries/sec it serves as the query batch grows. `search_many` is the batched entry @@ -9,7 +9,7 @@ throughput gains. It runs **anywhere**: on a non-CUDA host it reports the CPU-kernel curve; with a CUDA GPU and the `[gpu]` extra it adds the GPU-resident line, so the batch crossover is visible end -to end. No document or query text is ever logged — only counts, batch sizes, timings, +to end. No document or query text is ever logged, only counts, batch sizes, timings, throughput, and backend labels. This complements [`../direct_gpu_sweep/`](../direct_gpu_sweep) (the launch proof, which @@ -31,19 +31,19 @@ not a LodeDB runtime dependency (`uv pip install matplotlib`). ## What it measures -- **Throughput** (`queries_per_second`) per batch size — `repeats` timed `search_many` calls +- **Throughput** (`queries_per_second`) per batch size: `repeats` timed `search_many` calls on a batch of exactly `batch_size` queries; `q/s = batch_size × repeats / elapsed`. - **Per-query latency** (`per_query_ms`) and per-call p50 (`per_call_ms_p50`). - On CUDA, each GPU row carries `speedup_vs_cpu` and the `gpu_stage_one_status` audit label so you can confirm the GPU path actually engaged (vs. silent CPU fallback). **Embedding is excluded by design.** Queries are embedded by a trivial local hash backend, so -the number isolates **retrieval** — the stage batching and the GPU accelerate. (End-to-end +the number isolates **retrieval**, the stage batching and the GPU accelerate. (End-to-end `search_many` also pays query-embedding cost, which is independent of the store.) ## Results -Single-run, Apple M1 (CPU kernel only — no CUDA on this host), `minilm` (dim 384), 50,000 +Single-run, Apple M1 (CPU kernel only; no CUDA on this host), `minilm` (dim 384), 50,000 docs / chunks, `k=10`, 5 repeats: | batch | queries/sec | per-query ms | @@ -54,7 +54,7 @@ docs / chunks, `k=10`, 5 repeats: | 256 | 3,714 | 0.269 | | 1024 | 3,882 | 0.258 | -Batching alone is ~**2.9×** over single-query serving on CPU here — and that is *before* the +Batching alone is ~**2.9×** over single-query serving on CPU here, and that is *before* the GPU-resident path, which (per the [GPU benchmarks](../gpu_vanilla_vs_augmented)) pulls ahead of the CPU ceiling at batch ≥ 2 and scales with GPU class. Re-run on your own hardware for variance. @@ -66,5 +66,5 @@ variance. The LangChain and LlamaIndex adapters call single-query `search` (their retriever contracts are one-query-at-a-time), so they do **not** exercise `search_many` and leave this throughput on the table. Batched retrieval is not part of those framework contracts, so this is a -standalone benchmark rather than an adapter change — but it quantifies the headroom available +standalone benchmark rather than an adapter change, but it quantifies the headroom available to any offline / batch / multi-query workload that adopts `search_many` directly. diff --git a/benchmarks/batched_retrieval/diagrams.py b/benchmarks/batched_retrieval/diagrams.py index 948b277f..ffd6c6fb 100644 --- a/benchmarks/batched_retrieval/diagrams.py +++ b/benchmarks/batched_retrieval/diagrams.py @@ -5,7 +5,7 @@ --results benchmarks/batched_retrieval/results/laptop_m1.json \ --out docs -Plots ``search_many`` queries/sec vs query batch size — one line for the CPU kernel and, +Plots ``search_many`` queries/sec vs query batch size: one line for the CPU kernel and, when the results include GPU rows, one for the GPU-resident path. matplotlib is a dev-only dependency (not part of the lodedb runtime set); install it separately to render charts. """ diff --git a/benchmarks/batched_retrieval/run.py b/benchmarks/batched_retrieval/run.py index 2f2178e6..a40ff1f9 100644 --- a/benchmarks/batched_retrieval/run.py +++ b/benchmarks/batched_retrieval/run.py @@ -1,18 +1,18 @@ #!/usr/bin/env python3 """Batched-retrieval throughput: ``LodeDB.search_many`` queries/sec vs batch size. -This benchmarks the **public SDK path** ``LodeDB.search_many(queries, k=...)`` — the +This benchmarks the **public SDK path** ``LodeDB.search_many(queries, k=...)``, the batched entry point that lets CUDA hosts serve a query batch from the GPU-resident exact scan. Single-query ``search`` never takes that path, so batched ``search_many`` is the only way the GPU-batch story shows up through the supported API. It reports **queries/sec** across batch sizes for the native CPU kernel and, when a CUDA driver is present, the native -GPU-resident path — so the batch crossover is visible end to end. The native scan reads +GPU-resident path, so the batch crossover is visible end to end. The native scan reads ``LODEDB_GPU_DIRECT_TURBOVEC`` per scan (``off`` forces the CPU baseline, any other value leaves the GPU path eligible), toggled here without rebuilding the index. Embedding is intentionally excluded: queries are embedded by a trivial local hash backend so the number isolates **retrieval** (the stage the GPU accelerates), the same property a -batched-retrieval integration would lean on. Metrics only — counts, batch sizes, timings, +batched-retrieval integration would lean on. Metrics only: counts, batch sizes, timings, throughput, and a CPU-vs-GPU overlap; never documents, queries, or embeddings. uv run python benchmarks/batched_retrieval/run.py --docs 50000 --queries 1024 diff --git a/benchmarks/direct_gpu_sweep/modal_bench.py b/benchmarks/direct_gpu_sweep/modal_bench.py index c0f242ef..061a2198 100644 --- a/benchmarks/direct_gpu_sweep/modal_bench.py +++ b/benchmarks/direct_gpu_sweep/modal_bench.py @@ -62,7 +62,7 @@ def _build_image() -> modal.Image: # third_party/turbovec into the lodedb._turbovec extension, and # turbovec-python depends on the sibling turbovec core via path = "../turbovec". # So the full workspace must live under the build dir (/root/lodedb-src), - # exactly as `uv sync` sees it locally and in CI — otherwise maturin errors + # exactly as `uv sync` sees it locally and in CI; otherwise maturin errors # with "manifest path third_party/turbovec/turbovec-python/Cargo.toml does not exist". .add_local_dir( str(repo_root / "third_party" / "turbovec"), diff --git a/benchmarks/filtered_batch/README.md b/benchmarks/filtered_batch/README.md index 5c1c9f00..a94c5fe0 100644 --- a/benchmarks/filtered_batch/README.md +++ b/benchmarks/filtered_batch/README.md @@ -5,7 +5,7 @@ allowlist-pushdown fix for it. Before the fix, an unfiltered `search_many` rode the GPU-resident scan, but a *filtered* one widened the effective `top_k` to the corpus size and -post-filtered — which tripped the resident `top_k` cap +post-filtered, which tripped the resident `top_k` cap (`GPU_DIRECT_TURBOVEC_MAX_TOP_K = 4096`) and silently bypassed the GPU to the CPU kernel, scaling O(corpus) per query. After the fix, the filter is pushed into the scan as a shared allowlist (in-kernel on CPU, an `-inf` score mask on @@ -13,7 +13,7 @@ GPU/MPS), so `top_k` stays `k` and filtered batches stay on the fast path. The harness sweeps `(gpu_policy × batch_size × {unfiltered, selective, non-selective})`, capturing latency and the redacted `query_batch_completed` -telemetry — `gpu_stage_one_status` / `gpu_fallback_reason` — so the cliff (and +telemetry (`gpu_stage_one_status` / `gpu_fallback_reason`), so the cliff (and its closure) is *proven*, not just timed. It also records the host CPU ISA (AVX2 vs AVX-512, which Modal varies per run) and the kernel's own backend label. diff --git a/benchmarks/filtered_batch/modal_bench.py b/benchmarks/filtered_batch/modal_bench.py index 73e8dbd0..fd48abe7 100644 --- a/benchmarks/filtered_batch/modal_bench.py +++ b/benchmarks/filtered_batch/modal_bench.py @@ -37,8 +37,8 @@ def _build_image() -> modal.Image: """CUDA image with LodeDB compiled from local src (maturin + vendored crate). - ``/root/lodedb-src`` must mirror the repo layout — pyproject + readme/license - + ``src/`` + the full ``third_party/turbovec/`` workspace — so the single + ``/root/lodedb-src`` must mirror the repo layout (pyproject + readme/license + + ``src/`` + the full ``third_party/turbovec/`` workspace) so the single maturin ``pip install`` can compile ``lodedb._turbovec`` against the sibling crate, exactly as ``uv sync`` sees it locally and in CI. """ diff --git a/benchmarks/govreport_scale/README.md b/benchmarks/govreport_scale/README.md index da511365..ed618907 100644 --- a/benchmarks/govreport_scale/README.md +++ b/benchmarks/govreport_scale/README.md @@ -1,24 +1,24 @@ -# GovReport at scale — vanilla vs. augmented TurboVec, 100K → 1M +# GovReport at scale: vanilla vs. augmented TurboVec, 100K → 1M Scale + correctness evidence for the TurboVec scan on **real GovReport embeddings** (MiniLM, 384-d, cosine) across a 100K → 1M corpus-size sweep: does the scan stay correct at 1M vectors, and what is the CPU scan's practical throughput ceiling? -Both paths scan the **same 4-bit index** — the difference is the *scoring* step, not the bit +Both paths scan the **same 4-bit index**; the difference is the *scoring* step, not the bit width: -- **vanilla** — TurboVec's native CPU SIMD scan (`index.search`), the default. It sums a +- **vanilla**: TurboVec's native CPU SIMD scan (`index.search`), the default. It sums a **uint8 lookup table** (ADC-style) over the 4-bit codes; measured single-thread and all-threads (the CPU ceiling). -- **augmented** — the GPU-resident **fp16-reconstruction** scan +- **augmented**: the GPU-resident **fp16-reconstruction** scan (`lodedb.engine.gpu_turbovec.GpuDirectTurboVecSession`, the opt-in `[gpu]` path): each 4-bit code is reconstructed to fp16 and scored with a full GEMM dot product. It is "exact" only - as *exact arithmetic over the 4-bit reconstruction* (no uint8-LUT rounding) — not exact vs + as *exact arithmetic over the 4-bit reconstruction* (no uint8-LUT rounding), not exact vs the original fp32. Recall is R@1-within-top-k vs **exact fp32 brute-force** ground truth on the same embeddings -(that ground truth is the genuine fp32 exact). GPU-only — the augmented path needs CUDA and -1M-vector brute force is not laptop-scale. +(that ground truth is the genuine fp32 exact). This sweep is GPU-only because the augmented +path needs CUDA and 1M-vector brute force is not laptop-scale. ## Run @@ -36,7 +36,7 @@ modal run benchmarks/govreport_scale/modal_bench.py::main python benchmarks/govreport_scale/diagrams.py ``` -## Results — Modal L40S (`measured`) +## Results: Modal L40S (`measured`) **Corpus** (`recorded`): GovReport (`ccdv/govreport-summarization`) embedded with MiniLM (`all-MiniLM-L6-v2`, 384-d), chunked at 480 chars to reach a full **1,000,000** vectors; 1,000 @@ -45,7 +45,7 @@ host CPU kernel (Modal assigns it), so the speed charts overlay **both hosts mea **AVX2** host (this repo's run, `results.json`) and an **AVX-512** host (`results_avx512.json`). Recall is host-independent. Embedding 1M chunks took ~276s. -### Recall — preserved at 1M, and the reconstruction scan widens its lead +### Recall: preserved at 1M, and the reconstruction scan widens its lead R@1-in-top-k vs fp32 brute force: @@ -58,14 +58,14 @@ R@1-in-top-k vs fp32 brute force: Both scans recover the true nearest neighbour within the top-8 at every scale (R@8 = 1.000). At top-1 the vanilla uint8-LUT scan loses recall as the corpus grows (0.943 → 0.913); the augmented fp16-reconstruction scan holds (0.950 → 0.929), so its edge **widens with scale** -(+0.007 at 100K → +0.016 at 1M) — the exact GEMM avoids the uint8-LUT rounding error the LUT +(+0.007 at 100K → +0.016 at 1M). The exact GEMM avoids the uint8-LUT rounding error the LUT scan accumulates as more near-neighbours crowd in. (Both carry the same irreducible 4-bit -code-quantization loss — which is why R@8 = 1.0 for both. Recall is CPU-arch-independent, so +code-quantization loss, which is why R@8 = 1.0 for both. Recall is CPU-arch-independent, so these numbers are stable across hosts.) ![Recall vs scale](docs/recall_vs_scale.png) -### Speed — the CPU scan's ceiling (per host), and where the GPU pulls ahead +### Speed: the CPU scan's ceiling (per host), and where the GPU pulls ahead CPU all-threads throughput (the ceiling) for each host, vs the host-independent GPU scan (queries/sec, batch 64 for the GPU): @@ -79,14 +79,14 @@ CPU all-threads throughput (the ceiling) for each host, vs the host-independent ¹ Host-independent (it's the L40S either way); the AVX-512 run measured the GPU within ~6% (53,800 / 15,351 / 7,990 q/s). Single-thread is ~150–2,000 q/s, similar across hosts. -The CPU scan is O(N) (throughput ~1/N), and its ceiling depends on the host CPU kernel — at -1M, **~4,050 q/s on AVX-512 vs ~2,840 on AVX2** (all-threads). The augmented GPU reconstruction +The CPU scan is O(N) (throughput ~1/N), and its ceiling depends on the host CPU kernel; at +1M it is **~4,050 q/s on AVX-512 vs ~2,840 on AVX2** (all-threads). The augmented GPU reconstruction scan (batch 64) is **2.0× the AVX-512 ceiling / 2.8× the AVX2 ceiling** at 1M, and the multiple -grows with corpus size — the GPU's edge is naturally larger over a weaker CPU. +grows with corpus size; the GPU's edge is naturally larger over a weaker CPU. ![Throughput vs scale](docs/speed_vs_scale.png) -### Batch sweep at 1M — the GPU win is batched +### Batch sweep at 1M: the GPU win is batched Augmented GPU throughput vs batch (L40S), against each host's CPU all-threads ceiling: @@ -105,18 +105,18 @@ scan wins, so single queries route to the CPU. ## Reading -- **Correctness holds at 1M** — both scans find the true nearest neighbour within top-8 at +- **Correctness holds at 1M.** Both scans find the true nearest neighbour within top-8 at every scale; the reconstruction scan is at least as good at top-1 and its lead widens with corpus size. -- **The CPU scan ceiling is real and known** — at 1M, ~4,050 q/s all-threads on AVX-512 and +- **The CPU scan ceiling is real and known.** At 1M, ~4,050 q/s all-threads on AVX-512 and ~2,840 on AVX2 (single-thread ~150–2,000). Beyond it, the batched GPU reconstruction scan is the path: ≈2–3× the ceiling at batch 64, ≈5–8× at batch 1024 (the lower multiple is against the faster AVX-512 host). - Caveats: **single run per host.** The CPU ceiling depends on the host CPU kernel, so the - speed numbers carry host variance — both an AVX2 and an AVX-512 host are measured here, and + speed numbers carry host variance; both an AVX2 and an AVX-512 host are measured here, and the **ratios are the portable signal** (recall is host-independent and matches across hosts). - Chunks are 480 chars, chosen only to reach a full 1M vectors — chunk size sets the vector - count, not scan correctness. This is **not** a concurrency/thread-safety test — that's a + Chunks are 480 chars, chosen only to reach a full 1M vectors; chunk size sets the vector + count, not scan correctness. This is **not** a concurrency/thread-safety test; that's a separate follow-up. Files: the measurement core is `turbovec_govreport_scale.py` (GovReport loader + embedding + diff --git a/benchmarks/govreport_scale/modal_bench.py b/benchmarks/govreport_scale/modal_bench.py index ef6dfd33..cd879c67 100644 --- a/benchmarks/govreport_scale/modal_bench.py +++ b/benchmarks/govreport_scale/modal_bench.py @@ -2,7 +2,7 @@ Embeds GovReport chunks (MiniLM) up to ~1M vectors on the GPU, then measures recall@k vs fp32 brute force (the vanilla **uint8-LUT** scan + the augmented **fp16-reconstruction** -scan — both over the same 4-bit index) and the CPU scan's throughput ceiling (vanilla +scan, both over the same 4-bit index) and the CPU scan's throughput ceiling (vanilla single-/all-threads vs the augmented GPU path) across a 100K -> 1M corpus-size sweep, plus a batch sweep at 1M. diff --git a/benchmarks/govreport_scale/turbovec_govreport_scale.py b/benchmarks/govreport_scale/turbovec_govreport_scale.py index 4c2b623e..8a29b16b 100644 --- a/benchmarks/govreport_scale/turbovec_govreport_scale.py +++ b/benchmarks/govreport_scale/turbovec_govreport_scale.py @@ -1,19 +1,19 @@ """GovReport at scale: vanilla vs augmented TurboVec recall + speed, 100K -> 1M. Embeds GovReport chunks (MiniLM, cosine) up to ~1M vectors on the GPU, then runs the -vanilla-vs-augmented cells across a corpus-size sweep — the scale + correctness evidence a +vanilla-vs-augmented cells across a corpus-size sweep, the scale + correctness evidence a launch review flagged as missing for the CPU scan. Both paths read the SAME 4-bit index: - **recall@k vs fp32 brute force** for the vanilla **uint8-LUT** scan and the augmented - **fp16-reconstruction** scan, at each corpus size — does the scan still find the true + **fp16-reconstruction** scan, at each corpus size. Does the scan still find the true nearest neighbour at 1M? -- **the CPU scan's practical ceiling** — vanilla single-thread and all-threads throughput +- **the CPU scan's practical ceiling**: vanilla single-thread and all-threads throughput (fresh ``RAYON_NUM_THREADS``-pinned subprocesses) vs the augmented GPU throughput, per corpus size and a batch sweep at the top size. This is a dev-only benchmark (not part of the shipped ``lodedb`` package). It reuses the vanilla-vs-augmented cell runner from the sibling ``gpu_vanilla_vs_augmented`` benchmark -(only the data source is new). GPU-only — the augmented path needs CUDA, and 1M-vector +(only the data source is new). GPU-only. The augmented path needs CUDA, and 1M-vector brute-force ground truth is not laptop-scale. """ diff --git a/benchmarks/gpu_vanilla_vs_augmented/README.md b/benchmarks/gpu_vanilla_vs_augmented/README.md index c0565014..0610a357 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/README.md +++ b/benchmarks/gpu_vanilla_vs_augmented/README.md @@ -5,24 +5,24 @@ Benchmarks comparing the **vanilla TurboVec uint8-LUT scan** (the upstream LodeDB's **augmented GPU-resident fp16-reconstruction scan** (`lodedb.engine.gpu_turbovec`), built on the CPU + GPU patches in `third_party/turbovec/LOCAL_PATCHES.md`. It reproduces the kind of comparison the -upstream repo plots — recall, search speed, compression — but swaps the FAISS baseline +upstream repo plots (recall, search speed, compression) but swaps the FAISS baseline for the augmented GPU path. **No FAISS.** -Both series run against the **same** vendored 4-bit TurboVec index — the difference is +Both series run against the **same** vendored 4-bit TurboVec index; the difference is the *scoring step*. Both scan the same 4-bit index; the CPU sums a uint8 LUT (ADC), the GPU does a full fp16 GEMM dot product. So "exact" here means exact over the 4-bit reconstruction, not fp32: -- **vanilla** — `index.search(queries, k)`, the native NEON/AVX SIMD scan that sums a +- **vanilla**: `index.search(queries, k)`, the native NEON/AVX SIMD scan that sums a **uint8 lookup table (ADC-style)** over the compact 2/4-bit codes. The local patches *add* APIs (reconstruction, upsert, encoded-row export); they do not change this kernel, so it is the faithful "vanilla" baseline. -- **augmented** — the GPU path that reconstructs each 4-bit code to **fp16** and scores +- **augmented**: the GPU path that reconstructs each 4-bit code to **fp16** and scores with a GEMM dot product (`lodedb.engine.gpu_turbovec.GpuDirectTurboVecSession`): every row is reconstructed once to fp16 on the GPU, and query batches are scored with a rotated-query GEMM + streaming device top-k. Because it scores the reconstructed vectors with full fp16 arithmetic rather than the uint8 LUT, it avoids the uint8-LUT rounding - error the LUT scan accumulates (recall ≥ vanilla) — at the cost of fp16-resident GPU + error the LUT scan accumulates (recall ≥ vanilla), at the cost of fp16-resident GPU memory. It is "exact" only as exact arithmetic over the 4-bit reconstruction, not exact vs fp32; both scans carry the same irreducible 4-bit code-quantization loss. @@ -48,10 +48,10 @@ on **incremental persistence**, not scan speed. queries; ground truth is exact fp32 inner-product top-k. - **Scaling**: a synthetic corpus-size sweep (100K → 1M, d=1536, 4-bit) for the speed axis, where the GPU path's throughput scaling shows. Synthetic vectors are unit-norm - Gaussian — the data-oblivious regime TurboVec's random-rotation quantization targets. + Gaussian, the data-oblivious regime TurboVec's random-rotation quantization targets. - **Single- vs multi-threaded** CPU speed is measured in fresh subprocesses with `RAYON_NUM_THREADS` pinned (rayon reads it once per process). -- All metrics are counts / bytes / latency only — no documents, queries, or embeddings +- All metrics are counts / bytes / latency only; no documents, queries, or embeddings are logged or persisted. ## Reproduce @@ -102,7 +102,7 @@ modal run benchmarks/gpu_vanilla_vs_augmented/modal_bench.py::ceiling \ ### CPU-only smoke (any machine) -The CPU axes (speed ST/MT, recall, memory, update) run anywhere — the GPU series records +The CPU axes (speed ST/MT, recall, memory, update) run anywhere; the GPU series records `skipped` where CUDA is absent: ```bash @@ -111,20 +111,20 @@ python benchmarks/gpu_vanilla_vs_augmented/run_bench.py --smoke # tiny synthet ### Files -All files in `benchmarks/gpu_vanilla_vs_augmented/` are dev-only — **not** part of the +All files in `benchmarks/gpu_vanilla_vs_augmented/` are dev-only, **not** part of the shipped `lodedb` package: -- `turbovec_vva_bench.py` — measurement library + single-cell `python -m` runner. -- `turbovec_vva_runner.py` — orchestrator across all axes + `full`/`smoke` specs. -- `modal_bench.py` — Modal A10/L40S entry + dataset prep + self-contained image. -- `run_bench.py` — local CPU-only smoke wrapper. -- `diagrams.py` — results JSON → SVG/PNG. -- `results/` — committed result bundles (`results_a10.json`, `results_l40s.json`). -- `docs/` — rendered diagrams. +- `turbovec_vva_bench.py`: measurement library + single-cell `python -m` runner. +- `turbovec_vva_runner.py`: orchestrator across all axes + `full`/`smoke` specs. +- `modal_bench.py`: Modal A10/L40S entry + dataset prep + self-contained image. +- `run_bench.py`: local CPU-only smoke wrapper. +- `diagrams.py`: results JSON → SVG/PNG. +- `results/`: committed result bundles (`results_a10.json`, `results_l40s.json`). +- `docs/`: rendered diagrams. ## Results -> **Provenance — honest.** The GPU numbers below were **measured on Modal** (A10 and +> **Honest provenance.** The GPU numbers below were **measured on Modal** (A10 and > L40S); they are not reproducible without a CUDA host. The `machine` block in each > results bundle records the run: NVIDIA A10 / L40S GPU; x86 host, 32 vCPU; TurboVec > native kernel **AVX-512BW** for both committed runs. Modal assigns the host CPU, so the @@ -132,7 +132,7 @@ shipped `lodedb` package: > fixed per run. 100K corpus, 1,000 queries, k=64; recall on real OpenAI-DBpedia > (d=1536/3072) + GloVe-200. Charts in `docs/` are rendered from `results/results_a10.json`. -### Search speed — the augmented GPU win is *batched* (`docs/speed_batch.svg`) +### Search speed: the augmented GPU win is *batched* (`docs/speed_batch.svg`) The augmented fp16-reconstruction path scores via a batched GEMM, so throughput climbs with batch while the vanilla uint8-LUT scan is batch-insensitive. d=1536, 4-bit, 100K, **A10**: @@ -151,14 +151,14 @@ batches to the GPU. ![GPU throughput vs batch size](docs/speed_batch.png) **Bigger GPU (L40S, `results/results_l40s.json`):** the same sweep on an L40S (48 GB, -AVX-512 host) climbs to **50,326 q/s at batch 1024 = 4.8×** the CPU baseline (≈10,420 q/s) — +AVX-512 host) climbs to **50,326 q/s at batch 1024 = 4.8×** the CPU baseline (≈10,420 q/s), about 2× the A10's absolute throughput. The augmented scan's advantage scales with GPU class (A10 2.8× → L40S 4.8× at batch 1024) and would go further on an A100/H100. **Parity panel** (`docs/speed_parity.svg`, batch 64): augmented vs vanilla-all-threads on -A10 — d=3072/4-bit **5,609 vs 2,825 q/s (2.0×)**, d=1536/4-bit **11,486 vs 8,422 (1.4×)**; -at 2-bit the compact CPU SIMD scan is competitive (d=1536/2-bit **0.8×**) — the GPU does -full-precision fp16 work regardless of bit width while the 2-bit AVX-512 scan is very +A10, d=3072/4-bit **5,609 vs 2,825 q/s (2.0×)**, d=1536/4-bit **11,486 vs 8,422 (1.4×)**; +at 2-bit the compact CPU SIMD scan is competitive (d=1536/2-bit **0.8×**) because the GPU +does full-precision fp16 work regardless of bit width while the 2-bit AVX-512 scan is very cheap. Vanilla single-thread: 195–739 q/s. **Corpus scaling** (`docs/speed_scaling.svg`, d=1536/4-bit, batch 64): augmented stays @@ -168,9 +168,9 @@ cheap. Vanilla single-thread: 195–739 q/s. ![Search throughput vs corpus size](docs/speed_scaling.png) -### Recall — preserved (`docs/recall.svg`) +### Recall: preserved (`docs/recall.svg`) -R@1-within-top-k — vanilla uint8-LUT scan vs augmented fp16-reconstruction scan: +R@1-within-top-k, vanilla uint8-LUT scan vs augmented fp16-reconstruction scan: | dataset | 2-bit (van / aug) | 4-bit (van / aug) | |---|---|---| @@ -178,9 +178,9 @@ R@1-within-top-k — vanilla uint8-LUT scan vs augmented fp16-reconstruction sca | openai-3072 | 0.911 / 0.910 | 0.978 / 0.972 | | glove-200 (low dim) | 0.556 / 0.559 | 0.834 / **0.844** | -All reach ~1.0 by k=8 (GloVe by k=64 — the harder regime: 2-bit @1 is only ~0.56). At +All reach ~1.0 by k=8 (GloVe by k=64, the harder regime: 2-bit @1 is only ~0.56). At d=1536/3072 the uint8-LUT rounding error is negligible, so fp16 reconstruction doesn't -move recall. At **GloVe d=200** — where the LUT rounding error is largest — fp16 +move recall. At **GloVe d=200**, where the LUT rounding error is largest, fp16 reconstruction does help, but only a little: **+0.010 at 4-bit k=1** (0.844 vs 0.834), +0.002–0.004 at 2-bit. Both scans carry the same irreducible 4-bit code-quantization loss (that is why recall tops out and R@8 ≈ 1.0 for both); the augmented scan's value is @@ -188,15 +188,15 @@ throughput, not a recall jump. ![Recall vanilla vs augmented](docs/recall.png) -### Memory — the GPU's cost (`docs/memory.svg`) +### Memory: the GPU's cost (`docs/memory.svg`) -Bytes/vector — vanilla compact codes (RAM) vs augmented fp16 resident (GPU): +Bytes/vector, vanilla compact codes (RAM) vs augmented fp16 resident (GPU): d=1536: 4-bit **768** / 2-bit **384** vs fp16 **3,072** (4–8×); d=3072: **1,536 / 768** vs **6,144**; fp32 reference 6,144 / 12,288. The GPU throughput is bought with ~4× memory. ![Memory per vector](docs/memory.png) -### Update / persist — the CPU-patch win (`docs/update.svg`) +### Update / persist: the CPU-patch win (`docs/update.svg`) Apply 1,000 changed rows + persist. The vanilla full `.tvim` rewrite is **O(N)**; the encoded-row delta export is **O(changed)** and sub-millisecond regardless of corpus (A10): @@ -209,14 +209,14 @@ encoded-row delta export is **O(changed)** and sub-millisecond regardless of cor Full rewrite grows O(N) while the delta stays sub-millisecond, so the speedup scales with corpus (hundreds to >1,000×; sub-ms deltas make the exact multiplier slightly noisy -run-to-run — the L40S run lands 261× → 942× over the same sweep). This is the clearest +run-to-run, and the L40S run lands 261× → 942× over the same sweep). This is the clearest augmentation win. ![Update persist: full rewrite vs delta](docs/update.png) ### Bottom line -- **CPU delta-persistence patch: a large, scaling win** (173× → 1,308× at 100K → 1M) — the +- **CPU delta-persistence patch: a large, scaling win** (173× → 1,308× at 100K → 1M), the standout. - **GPU fp16-reconstruction batched scan: ~1.4–2.8× over the AVX-512 uint8-LUT CPU scan at batch ≥64 on the A10 (4.8× on the L40S), recall preserved.** Single queries and 2-bit diff --git a/benchmarks/gpu_vanilla_vs_augmented/diagrams.py b/benchmarks/gpu_vanilla_vs_augmented/diagrams.py index 09ffec63..b5fe6347 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/diagrams.py +++ b/benchmarks/gpu_vanilla_vs_augmented/diagrams.py @@ -1,7 +1,7 @@ """Render vanilla-vs-augmented TurboVec benchmark diagrams from a results JSON. -Plots the comparison across each axis — the vanilla uint8-LUT CPU SIMD scan vs -the augmented fp16-reconstruction GPU scan (recall curves, search speed, memory, +Plots the vanilla uint8-LUT CPU SIMD scan against the augmented +fp16-reconstruction GPU scan on each axis (recall curves, search speed, memory, update). No FAISS. Both scan the same 4-bit index; the difference is the scoring step (the CPU sums a uint8 LUT (ADC), the GPU does a full fp16 GEMM dot product), so "exact" here means exact over the 4-bit reconstruction, not fp32. One figure @@ -27,10 +27,10 @@ import matplotlib.pyplot as plt # noqa: E402 from matplotlib.ticker import FuncFormatter # noqa: E402 -VANILLA_C = "#2563eb" # blue — vanilla CPU -VANILLA2_C = "#93c5fd" # light blue — vanilla single-thread -AUG_C = "#dc2626" # red — augmented GPU -FP32_C = "#94a3b8" # grey — fp32 reference +VANILLA_C = "#2563eb" # blue: vanilla CPU +VANILLA2_C = "#93c5fd" # light blue: vanilla single-thread +AUG_C = "#dc2626" # red: augmented GPU +FP32_C = "#94a3b8" # grey: fp32 reference K_GRID = (1, 2, 4, 8, 16, 32, 64) @@ -130,7 +130,7 @@ def plot_speed_scaling(rows: list[dict], out: Path) -> list[str]: return _save(fig, out, "speed_scaling") -L40S_C = "#7c3aed" # violet — augmented GPU, L40S +L40S_C = "#7c3aed" # violet: augmented GPU, L40S def plot_speed_batch( diff --git a/benchmarks/gpu_vanilla_vs_augmented/modal_bench.py b/benchmarks/gpu_vanilla_vs_augmented/modal_bench.py index c9751fbb..76b1dbd9 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/modal_bench.py +++ b/benchmarks/gpu_vanilla_vs_augmented/modal_bench.py @@ -11,7 +11,7 @@ ``cupy-cuda12x``, installs the local ``lodedb`` package, and mounts this benchmark directory so the measurement core (``turbovec_vva_bench`` / ``turbovec_vva_runner``, dev-only sibling scripts) is importable. The benchmark core also runs directly on -any CUDA host without Modal — see ``turbovec_vva_runner.py``. +any CUDA host without Modal (see ``turbovec_vva_runner.py``). Launch from the repo root (relative image paths resolve there): @@ -37,8 +37,8 @@ # Runtime libraries the benchmark core needs in the container. ``lodedb.engine`` # is imported through the top-level ``lodedb`` package, so its declared runtime -# deps must be present before lodedb is installed with ``--no-deps`` (below) — -# installing lodedb's deps via PyPI would otherwise pull a stock ``turbovec`` and +# deps must be present before lodedb is installed with ``--no-deps`` (below). +# Installing lodedb's deps via PyPI would otherwise pull a stock ``turbovec`` and # clobber the patched vendored wheel. _LODEDB_RUNTIME_DEPENDENCIES = ( "numpy>=2.0.0", @@ -230,7 +230,7 @@ def _prep_and_run(spec: dict, require_backend: str) -> dict: """Shared container body: optional host-CPU guard, prep real datasets, run matrix. ``require_backend`` (e.g. ``"avx512bw"``) makes the call **bail in seconds** if the - host's TurboVec CPU kernel is not the requested one — Modal assigns the host CPU, so + host's TurboVec CPU kernel is not the requested one. Modal assigns the host CPU, so the launcher can relaunch for a fresh host without paying for a full run on the wrong baseline. ``""`` or ``"any"`` accepts whatever host. """ @@ -273,7 +273,7 @@ def run_benchmark(spec: dict, require_backend: str = "") -> dict: @app.function(gpu="L40S", cpu=16.0, memory=65536, timeout=3600) def run_benchmark_l40s(spec: dict, require_backend: str = "") -> dict: - """GPU-ceiling variant on an L40S (48 GB) — shows where the exact-GEMM path pulls ahead.""" + """GPU-ceiling variant on an L40S (48 GB). Shows where the exact-GEMM path pulls ahead.""" return _prep_and_run(spec, require_backend) diff --git a/benchmarks/gpu_vanilla_vs_augmented/run_bench.py b/benchmarks/gpu_vanilla_vs_augmented/run_bench.py index 03180647..1fbddb54 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/run_bench.py +++ b/benchmarks/gpu_vanilla_vs_augmented/run_bench.py @@ -1,7 +1,7 @@ """Local CPU-only convenience wrapper for the vanilla-vs-augmented benchmark. The measurement core lives beside this file (``turbovec_vva_bench`` + -``turbovec_vva_runner``) as dev-only scripts — not part of the shipped ``lodedb`` +``turbovec_vva_runner``) as dev-only scripts, not part of the shipped ``lodedb`` package. This wrapper just makes those siblings importable and forwards to the runner's CLI: diff --git a/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_bench.py b/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_bench.py index 106daf22..c4ba81e0 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_bench.py +++ b/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_bench.py @@ -1,28 +1,28 @@ -"""Vanilla TurboVec vs the augmented GPU TurboVec — measurement library. +"""Measurement library for vanilla TurboVec vs the augmented GPU TurboVec. Compares the *same* vendored 4-bit TurboVec index two ways, with NO FAISS. Both -scan the same 4-bit codes; the difference is the scoring step — the CPU sums a +scan the same 4-bit codes; the difference is the scoring step. The CPU sums a uint8 LUT (ADC), the GPU reconstructs to fp16 and does a full GEMM dot product. So "exact" below means exact over the 4-bit reconstruction, not fp32: -- **vanilla** — TurboVec's native CPU SIMD scan, ``index.search(queries, k)``, +- **vanilla**: TurboVec's native CPU SIMD scan, ``index.search(queries, k)``, which sums a uint8 lookup table (ADC-style) over the codes. This is exactly the kernel the upstream repo benchmarks; the local patches add APIs (reconstruction/upsert/encoded-rows) but do not change this scan. -- **augmented** — the GPU-resident fp16-reconstruction scan +- **augmented**: the GPU-resident fp16-reconstruction scan (:class:`lodedb.engine.gpu_turbovec.GpuDirectTurboVecSession`): all rows are reconstructed once to fp16 on the GPU and batches are scored with a rotated-query GEMM + streaming device top-k. Because it scores the reconstructed vectors with full fp16 arithmetic (not the uint8 LUT), it avoids the uint8-LUT rounding error the LUT scan accumulates, so its recall is >= vanilla while - throughput scales on the GPU — at higher resident memory (fp16 rows vs compact + throughput scales on the GPU, at higher resident memory (fp16 rows vs compact 2/4-bit codes). Both scans carry the same irreducible 4-bit code-quantization loss. -Axes (all metrics only — no payloads): - speed — ms/query + queries/sec, vanilla uint8-LUT scan (ST + MT) vs augmented fp16 scan. - recall — R@1-within-top-k vs exact fp32 ground truth, vanilla vs augmented. - memory — bytes/vector compact (vanilla, in RAM) vs fp16 resident (augmented GPU). - update — incremental update + persist cost: vanilla full rewrite (O(N)) vs the +Axes (all metrics only, no payloads): + speed: ms/query + queries/sec, vanilla uint8-LUT scan (ST + MT) vs augmented fp16 scan. + recall: R@1-within-top-k vs exact fp32 ground truth, vanilla vs augmented. + memory: bytes/vector compact (vanilla, in RAM) vs fp16 resident (augmented GPU). + update: incremental update + persist cost, vanilla full rewrite (O(N)) vs the augmented delta export (O(changed rows)). Thread count for the CPU scan is controlled by ``RAYON_NUM_THREADS`` and must be set @@ -69,7 +69,7 @@ def make_vectors(n: int, dim: int, *, seed: int) -> NDArray[np.float32]: def _unit(vectors: NDArray[np.float32]) -> NDArray[np.float32]: """Returns row-wise unit-normalized vectors (TurboVec ranks by cosine). - Applied to loaded datasets so the exact ground truth is cosine top-k — the same + Applied to loaded datasets so the exact ground truth is cosine top-k, the same metric TurboVec's scan and the GPU reconstruction rank by (both strip the per-vector norm). Idempotent on already-unit data (synthetic, most OpenAI). """ @@ -101,7 +101,7 @@ def exact_topk_ids( """Returns exact top-k stable ids (1-based) by inner product, in fp32. Computed in row tiles so the full (queries x corpus) score matrix is never - materialized — keeps the CPU ground truth feasible at large corpus sizes. + materialized, which keeps the CPU ground truth feasible at large corpus sizes. """ n = int(vectors.shape[0]) @@ -253,9 +253,9 @@ def time_update_persist( Applies ``update_count`` in-place value updates to an existing index and measures the persist cost each way: - - **vanilla** — no in-place upsert in stock TurboVec: re-add the changed ids + - **vanilla**: no in-place upsert in stock TurboVec, so re-add the changed ids and write the *whole* ``.tvim`` (the only stock durability primitive). - - **augmented** — ``upsert_with_ids`` (in-place slot replace) + ``export_encoded`` + - **augmented**: ``upsert_with_ids`` (in-place slot replace) + ``export_encoded`` of just the changed rows (the delta the ``.tvd``/``.tvim`` delta store ships). """ diff --git a/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_runner.py b/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_runner.py index 802cfba7..54abfa8f 100644 --- a/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_runner.py +++ b/benchmarks/gpu_vanilla_vs_augmented/turbovec_vva_runner.py @@ -1,13 +1,13 @@ """Orchestrate the vanilla-vs-augmented TurboVec benchmark across all four axes. This module and its measurement core (``turbovec_vva_bench``) are dev-only -benchmark scripts that live beside this file — they are NOT part of the shipped +benchmark scripts that live beside this file; they are NOT part of the shipped ``lodedb`` package. Vanilla single- vs multi-threaded speed is measured in fresh subprocesses with ``RAYON_NUM_THREADS`` pinned (rayon reads it once per process, so the cell module is invoked via ``python -m`` per thread setting); the GPU path and the recall/memory/update axes run in-process. -Run directly on any host (``lodedb`` must be importable — e.g. installed, or with +Run directly on any host (``lodedb`` must be importable, e.g. installed, or with this benchmark dir on ``sys.path`` so ``import turbovec_vva_bench`` resolves; the file inserts its own directory automatically):: diff --git a/benchmarks/graph_memory/graph_memory_bench.py b/benchmarks/graph_memory/graph_memory_bench.py index 4eb03ea4..33cb53d7 100644 --- a/benchmarks/graph_memory/graph_memory_bench.py +++ b/benchmarks/graph_memory/graph_memory_bench.py @@ -1,20 +1,20 @@ """Graph-memory benchmark: vector-in, predicate filters, and graph traversal. Exercises the three capabilities added for the knowledge-graph / memory stack -and reports **metrics only** (counts, latency, throughput, recall/overlap — never +and reports **metrics only** (counts, latency, throughput, recall/overlap, never raw text, queries, or embeddings), matching the repo's benchmark provenance rules. Three sub-benchmarks, all driven from the same loaded corpus: -1. ``vector_in`` — text-in ingest (LodeDB embeds internally) vs vector-in ingest +1. ``vector_in``: text-in ingest (LodeDB embeds internally) vs vector-in ingest (caller supplies precomputed vectors via ``add_vectors_many``), plus query parity: ``search`` vs ``search_by_vector`` over byte-identical indexes should return the same hits, isolating the embedding cost vector-in removes. -2. ``filters`` — search latency across predicate selectivities: exact ``$eq`` +2. ``filters``: search latency across predicate selectivities, exact ``$eq`` (posting-allowlist pushdown) vs ``$gte`` / ``$ne`` / ``$exists`` (which today are resolved by the per-field planner). Compares the planner against a per-document scan. -3. ``graph`` — a synthetic knowledge graph over the corpus: k-hop traversal +3. ``graph``: a synthetic knowledge graph over the corpus, with k-hop traversal latency (SQLite topology) and hybrid ``search_subgraph`` latency (semantic seed + structural expansion) at scale. diff --git a/benchmarks/hybrid/hybrid_recall.py b/benchmarks/hybrid/hybrid_recall.py index 6fdff408..52f46c0b 100644 --- a/benchmarks/hybrid/hybrid_recall.py +++ b/benchmarks/hybrid/hybrid_recall.py @@ -13,7 +13,7 @@ the "embedding cannot see the literal token" failure mode reproducible without downloading a model, so the lexical contribution is isolated rather than masked by a model that happens to encode some character-level signal. Output is -raw-payload-free (counts, ratios, latency only — never tokens or terms). +raw-payload-free (counts, ratios, latency only, never tokens or terms). Run:: diff --git a/benchmarks/laptop/README.md b/benchmarks/laptop/README.md index dfab4e52..c59532da 100644 --- a/benchmarks/laptop/README.md +++ b/benchmarks/laptop/README.md @@ -1,9 +1,9 @@ -# Laptop benchmark — embedding throughput + CPU scan latency +# Laptop benchmark: embedding throughput + CPU scan latency A metrics-only benchmark of the two stages LodeDB does on a laptop: **embedding** (the -accelerated stage on Apple Silicon — the Apple GPU via MPS) and the **CPU TurboVec scan**. It +accelerated stage on Apple Silicon, the Apple GPU via MPS) and the **CPU TurboVec scan**. It runs the same measurement as `lodedb benchmark`, once per available embedding device, and -writes one combined results JSON. No document or query text is ever logged — only counts, +writes one combined results JSON. No document or query text is ever logged, only counts, bytes, and latency. There is no GPU vector search on Mac; the scan always runs on the CPU SIMD kernel (NEON). @@ -18,17 +18,17 @@ python benchmarks/laptop/diagrams.py # renders docs/* ``` `run.py` benchmarks every embedding device it finds (MPS → CPU). `diagrams.py` needs -`matplotlib`, which is a dev-only tool, not a LodeDB runtime dependency — install it +`matplotlib`, which is a dev-only tool, not a LodeDB runtime dependency; install it separately (`uv pip install matplotlib`). ## What it measures -- **Embedding throughput** (`docs/second`) — `add_many` throughput at embed batch 64, measured +- **Embedding throughput** (`docs/second`): `add_many` throughput at embed batch 64, measured with the embedding model pre-loaded before timing (`run_local_benchmark` warms the backend first). This is warm steady state, not the one-time cold model load. -- **CPU scan latency** (`search_only_ms` p50/p95) — the TurboVec scan plus result +- **CPU scan latency** (`search_only_ms` p50/p95): the TurboVec scan plus result materialization, with embedding excluded. Device-independent: it is always the CPU kernel. -- **End-to-end query latency** (`end_to_end_query_ms` p50/p95) — embed one query, then scan. +- **End-to-end query latency** (`end_to_end_query_ms` p50/p95): embed one query, then scan. This is what matters for interactive, one-query-at-a-time serving. ## Results diff --git a/benchmarks/laptop/diagrams.py b/benchmarks/laptop/diagrams.py index 910ba09e..81275719 100644 --- a/benchmarks/laptop/diagrams.py +++ b/benchmarks/laptop/diagrams.py @@ -6,7 +6,7 @@ --out docs Produces two charts (PNG + SVG): embedding throughput by device, and end-to-end -query latency (p50/p95) by device. matplotlib is a dev-only dependency — it is not +query latency (p50/p95) by device. matplotlib is a dev-only dependency; it is not part of the lodedb runtime set, so install it separately to render charts. """ diff --git a/benchmarks/laptop/run.py b/benchmarks/laptop/run.py index 54266a9d..42a6fa40 100644 --- a/benchmarks/laptop/run.py +++ b/benchmarks/laptop/run.py @@ -2,7 +2,7 @@ """Laptop benchmark: embedding throughput + CPU TurboVec scan latency, per device. Runs the same metrics-only measurement behind ``lodedb benchmark`` for each available -embedding device (mps / cpu) and writes one combined results JSON. Metrics only — +embedding device (mps / cpu) and writes one combined results JSON. Metrics only: counts, bytes, and latency; never document or query text. python benchmarks/laptop/run.py --docs 20000 --queries 200 diff --git a/benchmarks/late_interaction/README.md b/benchmarks/late_interaction/README.md index bcf0c460..b74e2ec9 100644 --- a/benchmarks/late_interaction/README.md +++ b/benchmarks/late_interaction/README.md @@ -34,7 +34,7 @@ mean-pooling cannot: Pooling a page to one vector destroys the per-patch signal MaxSim depends on, so its recall against the MaxSim ranking is near zero regardless of tuning. Late interaction holds every patch in one in-memory matrix and scores the whole corpus -with a single GEMM plus a segmented max -- no candidate-recall loss, no +with a single GEMM plus a segmented max, with no candidate-recall loss and no per-candidate read-back. ### Storage precision (`--storage`) @@ -58,8 +58,8 @@ generation (~61%) and patch read-back (~38%) dominated. Three changes removed al of it: - **Resident exact scan** (default, unfiltered, within `resident_max_bytes`): - scores the whole corpus from one in-memory matrix -- no candidate scan, no - read-back. ~110 ms -> a few ms, recall 0.73 -> 1.0. + scores the whole corpus from one in-memory matrix, with no candidate scan and + no read-back. ~110 ms -> a few ms, recall 0.73 -> 1.0. - **One row per document** (the whole patch matrix in a single row): 39-54x faster ingest and far fewer rows; filtered queries score the matching subset exhaustively, and corpora over the resident budget stream from disk (exact, diff --git a/benchmarks/memory_integrations/README.md b/benchmarks/memory_integrations/README.md index 04fa823e..0fb6cc71 100644 --- a/benchmarks/memory_integrations/README.md +++ b/benchmarks/memory_integrations/README.md @@ -136,7 +136,7 @@ then the multiplier. | **Recall@10** | 0.95 vs 1.00 | 0.95 vs 1.00 | 0.95 vs 1.00 (filtered 0.95 vs 1.00) | Every backend, LodeDB included, is fed the same precomputed vectors (LodeDB via its -vector-in SDK), so none is charged for embedding -- a store-vs-store comparison. The +vector-in SDK), so none is charged for embedding; the comparison is store against store. The in-memory defaults rewrite the whole store to persist one memory and scan in pure Python with no batch path. mem0's default Qdrant is a real DB, so its single add is fast (0.44 ms), and LodeDB now matches it (0.28 ms) while reading far faster, storing @@ -160,13 +160,13 @@ pgvector**. | pgvector | 1,947 | 35.1 | 1.00 | 2.3 ms | yes | 50 MB | 3.3x | Among the embedded local DBs, **LodeDB has the smallest footprint and the fastest -single query** -- LanceDB (10.6 ms), sqlite-vec (27 ms), and pgvector (35 ms) are 24x to +single query**. LanceDB (10.6 ms), sqlite-vec (27 ms), and pgvector (35 ms) are 24x to 78x slower per query because they scan without LodeDB's quantized SIMD kernel, trading the 5 points of recall LodeDB gives up. LanceDB is the closest on footprint (37 MB vs 15 MB). pgvector runs via an embedded `pgserver` (Postgres + pgvector, no separate service) with no ANN index, so it is an exact seq scan like LodeDB, LanceDB, and sqlite-vec. On durable single-add LodeDB's WAL-mode O(changed) commit (0.26 ms) leads -even the lazy-append stores (sqlite-vec 0.42 ms, qdrant 0.48 ms) -- see "Durable add" +even the lazy-append stores (sqlite-vec 0.42 ms, qdrant 0.48 ms). See "Durable add" under Reading. ### LlamaIndex (default `SimpleVectorStore`), RAG over ~17.5k docs @@ -185,7 +185,7 @@ under Reading. ### mem0 (default Qdrant), agent-memory workflow over ~17.5k memories This suite is vector-in (mem0 owns embeddings), so the durable-add column is -embed-free for every backend -- the fair persist-to-persist comparison. +embed-free for every backend, the fair persist-to-persist comparison. | backend | ingest docs/s | query p50 (ms) | recall@10 | filtered recall | durable add p50 | footprint | vs LodeDB | |---|---|---|---|---|---|---|---| @@ -221,7 +221,7 @@ Reading it: (see the project's GPU benchmarks). - **FAISS is the one close baseline, and it is noisy.** `*` FAISS-CPU is the only baseline that batches efficiently, but its throughput is host-dependent on Modal's - shared instances (~1,800 to 2,900 qps across suites here) -- treat it as + shared instances (~1,800 to 2,900 qps across suites here). Treat it as "comparable to several-x slower than LodeDB," not a fixed number. - **Scale.** At ~17.5k vectors the GPU is not yet compute-bound, so this understates the GPU win. The headline GPU throughput (24k qps A10, 50k qps L40S, 2.8x to 4.8x the @@ -254,7 +254,7 @@ Reading it: LodeDB's single-query range, and it is not durable (full rewrite, no payload round-trip). - **Recall.** LodeDB returns 0.95 recall@10 (4-bit quantization) versus 1.00 for the - exact/flat stores -- the deliberate trade for footprint and query speed. The scan + exact/flat stores, the deliberate trade for footprint and query speed. The scan is exact (no ANN graph), so there is no recall cliff to tune. - **mem0 filtered search.** mem0's FAISS provider has no server-side filtering: it over-fetches only `2*k` then post-filters, so within-user recall collapses to 0.04 @@ -278,7 +278,7 @@ Reading it: - LlamaIndex's `FaissVectorStore` keeps no docstore, so used standalone it returns the faiss positional index rather than the node id (and no payload); the harness maps positions back through insertion order so recall is correct. -- Qdrant and Chroma run in embedded/local mode (no server) -- the apples-to-apples +- Qdrant and Chroma run in embedded/local mode (no server), the apples-to-apples comparison against an embedded store, not their tuned server configuration. - LodeDB's cold reopen rebuilds calibration on first open, which at small corpora shows as a higher reopen time that amortizes at scale. diff --git a/benchmarks/mps_vs_neon/README.md b/benchmarks/mps_vs_neon/README.md index b6c71cd2..b213aae6 100644 --- a/benchmarks/mps_vs_neon/README.md +++ b/benchmarks/mps_vs_neon/README.md @@ -2,10 +2,10 @@ Does running the vector scan on the **Apple GPU (Metal/MPS)** beat the default **CPU NEON scan** on a Mac? This benchmark answers it on your hardware. It compares the same vendored -TurboVec index two ways — no Modal, no CUDA: +TurboVec index two ways, with no Modal and no CUDA: -- **NEON** (default) — TurboVec's native CPU SIMD scan, `index.search(queries, k)`. -- **MPS exact** (opt-in) — `lodedb.engine.mps_turbovec.MpsDirectTurboVecSession`: dequantized +- **NEON** (default): TurboVec's native CPU SIMD scan, `index.search(queries, k)`. +- **MPS exact** (opt-in): `lodedb.engine.mps_turbovec.MpsDirectTurboVecSession`, dequantized fp16 rows resident on the Apple GPU, scored with a batched matmul + `torch.topk`. It mirrors the CUDA `gpu_turbovec` path. @@ -17,7 +17,7 @@ python benchmarks/mps_vs_neon/run.py --n 100000 --dim 384 --queries 1000 python benchmarks/mps_vs_neon/diagrams.py ``` -## Result — Apple M1 (`measured`) +## Result: Apple M1 (`measured`) 100K × 384, 4-bit, k=64, median of 3 passes: @@ -35,10 +35,10 @@ python benchmarks/mps_vs_neon/diagrams.py per-batch dispatch overhead and the fp16 reconstruct, while Apple's CPU NEON scan over compact 4-bit codes is genuinely fast (and ~8× less memory). MPS does close the gap as batch grows (0.08× → 0.53×), so a **much stronger Apple GPU (M-series Pro/Max, M5+) could push the crossover -past 1.0× at high batch** — which is the open question this benchmark exists to answer. Run it +past 1.0× at high batch**. That is the open question this benchmark exists to answer. Run it on your hardware. -**Recall is preserved** — the MPS path scores dequantized rows exactly (no uint8 LUT error), so +**Recall is preserved.** The MPS path scores dequantized rows exactly (no uint8 LUT error), so its recall is at least the NEON scan's: | metric | NEON | MPS exact | @@ -48,6 +48,6 @@ its recall is at least the NEON scan's: ## Status -The MPS backend is **opt-in and not wired into device selection** — the CPU NEON scan stays the +The MPS backend is **opt-in and not wired into device selection**; the CPU NEON scan stays the default on Mac. Construct `MpsDirectTurboVecSession` explicitly to use it. It should become a default only if it beats NEON at a realistic operating point on your hardware. diff --git a/benchmarks/mps_vs_neon/diagrams.py b/benchmarks/mps_vs_neon/diagrams.py index 21604cb3..42213013 100644 --- a/benchmarks/mps_vs_neon/diagrams.py +++ b/benchmarks/mps_vs_neon/diagrams.py @@ -4,7 +4,7 @@ python benchmarks/mps_vs_neon/diagrams.py \ --results benchmarks/mps_vs_neon/results/mps_vs_neon_m1.json --out docs -matplotlib is a dev-only tool, not a LodeDB runtime dependency — install it +matplotlib is a dev-only tool, not a LodeDB runtime dependency; install it separately (``uv pip install matplotlib``). """ diff --git a/benchmarks/mps_vs_neon/run.py b/benchmarks/mps_vs_neon/run.py index e8c8abe5..55a3155e 100644 --- a/benchmarks/mps_vs_neon/run.py +++ b/benchmarks/mps_vs_neon/run.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -"""MPS exact scan vs. TurboVec NEON scan — a local Apple-Silicon benchmark. +"""MPS exact scan vs. TurboVec NEON scan, a local Apple-Silicon benchmark. -Compares the same vendored TurboVec index two ways — no Modal, no CUDA: +Compares the same vendored TurboVec index two ways (no Modal, no CUDA): -- **NEON** — TurboVec's native CPU SIMD scan (``index.search(queries, k)``), +- **NEON**: TurboVec's native CPU SIMD scan (``index.search(queries, k)``), the default on Mac. -- **MPS exact** — the opt-in +- **MPS exact**: the opt-in :class:`lodedb.engine.mps_turbovec.MpsDirectTurboVecSession`: dequantized fp16 rows resident on the Apple GPU, scored with a batched matmul + ``torch.topk``. diff --git a/crates/lodedb-cloud-core/src/artifact_store.rs b/crates/lodedb-cloud-core/src/artifact_store.rs index 1b949818..b94d9abd 100644 --- a/crates/lodedb-cloud-core/src/artifact_store.rs +++ b/crates/lodedb-cloud-core/src/artifact_store.rs @@ -24,7 +24,7 @@ use std::io::Read; /// /// The interface is deliberately small: streaming artifact I/O plus a pointer /// compare-and-swap. That is everything the generation-addressed commit format -/// needs as a cloud substrate. Streaming is the primitive — vector bases run +/// needs as a cloud substrate. Streaming is the primitive. Vector bases run /// to gigabytes, so a transfer's peak memory must be a fixed buffer, never a /// function of artifact size; the buffered methods are conveniences for the /// small payloads (pointer documents) built on top. @@ -35,7 +35,7 @@ pub trait ArtifactStore { /// [`ArtifactStoreError::NotFound`]: crate::ArtifactStoreError::NotFound fn open_read<'a>(&'a self, name: &str) -> Result>; - /// Returns one artifact's bytes, fully buffered — for small payloads + /// Returns one artifact's bytes, fully buffered, for small payloads /// only; transfers and verification stream via [`open_read`](Self::open_read). fn read_bytes(&self, name: &str) -> Result> { let mut data = Vec::new(); @@ -47,16 +47,16 @@ pub trait ArtifactStore { /// exists, hashing incrementally as it copies. /// /// `sha256` is the expected lowercase-hex digest of the streamed bytes; a - /// mismatch is an integrity error and nothing is stored — corruption can + /// mismatch is an integrity error and nothing is stored, so corruption can /// never land, even when the source is another store's live stream. A /// name already present with identical bytes is a no-op (idempotent - /// re-push); present with *different* bytes is a conflict — artifacts are + /// re-push); present with *different* bytes is a conflict. Artifacts are /// immutable and are never overwritten in place. On the already-present /// no-op path the incoming stream may be left partially (or wholly) /// unread and unvalidated: success then attests that the STORED bytes /// match `sha256`, not that the stream did. /// - /// `size_hint` is the expected byte count (0 = unknown) — advisory only: + /// `size_hint` is the expected byte count (0 = unknown), advisory only: /// backends use it to pick an upload strategy (the object store's /// conditional-claim path has a provider copy-size ceiling), never to /// trust the stream's length. Transfers pass the manifest-recorded size. @@ -105,8 +105,8 @@ pub trait ArtifactStore { /// artifacts are published first, then the pointer flips all-or-nothing. /// /// The precondition is the full committed body, not just its generation number. - /// A generation number is not a unique version token — two independent lineages - /// can share one with different content — so comparing the whole body is what + /// A generation number is not a unique version token (two independent lineages + /// can share one with different content), so comparing the whole body is what /// makes this a sound compare-and-swap rather than an ABA-prone check. /// /// [`ArtifactStoreError::PointerConflict`]: crate::ArtifactStoreError::PointerConflict @@ -121,7 +121,7 @@ pub trait ArtifactStore { /// The generation number recorded in a committed body, if present. /// /// Used only to annotate a [`PointerConflict`](crate::ArtifactStoreError::PointerConflict) -/// for readability — the swap precondition is the full body, not this number. +/// for readability; the swap precondition is the full body, not this number. pub(crate) fn body_generation(body: &Value) -> Option { body.get("generation").and_then(Value::as_u64) } diff --git a/crates/lodedb-cloud-core/src/blob_layout.rs b/crates/lodedb-cloud-core/src/blob_layout.rs index 1519c94b..05b2af15 100644 --- a/crates/lodedb-cloud-core/src/blob_layout.rs +++ b/crates/lodedb-cloud-core/src/blob_layout.rs @@ -1,7 +1,7 @@ //! Content-addressed blob naming for the managed remote layout. //! -//! A managed remote can store artifacts content-addressed — -//! `blobs/sha256/aa/` under a per-tenant prefix — instead of under +//! A managed remote can store artifacts content-addressed +//! (`blobs/sha256/aa/` under a per-tenant prefix) instead of under //! their engine path names. Content addressing is what absorbs the //! fork-collision case (two branches committing different artifacts under the //! same engine name, e.g. `idx.gen/g7.json`, coexist as two blobs), and the @@ -9,7 +9,7 @@ //! //! These helpers are pure and tested; they pin the naming contract the //! transfer plane builds on. The digest is always the lowercase-hex -//! SHA-256 of the blob's bytes — the same digest the engine records per +//! SHA-256 of the blob's bytes, the same digest the engine records per //! artifact and [`ArtifactStore::write_bytes_if_absent`] verifies. //! //! [`ArtifactStore::write_bytes_if_absent`]: crate::ArtifactStore::write_bytes_if_absent @@ -22,7 +22,7 @@ const BLOB_PREFIX: &str = "blobs/sha256"; /// Returns the store-relative blob name for a content digest: /// `blobs/sha256/aa/`, where `aa` is the digest's first two hex chars. /// -/// Rejects anything that is not exactly 64 lowercase hex characters — blob +/// Rejects anything that is not exactly 64 lowercase hex characters. Blob /// names participate in authorization paths, so a malformed digest must fail /// closed rather than mint a name outside the contract. pub fn blob_name(sha256: &str) -> Result { @@ -30,7 +30,7 @@ pub fn blob_name(sha256: &str) -> Result { Ok(format!("{BLOB_PREFIX}/{}/{sha256}", &sha256[..2])) } -/// Parses a blob name back to its content digest — the exact inverse of +/// Parses a blob name back to its content digest, the exact inverse of /// [`blob_name`]. Rejects any name that deviates from the contract (wrong /// prefix, fan-out directory disagreeing with the digest, malformed digest). pub fn parse_blob_name(name: &str) -> Result { diff --git a/crates/lodedb-cloud-core/src/client_ops.rs b/crates/lodedb-cloud-core/src/client_ops.rs index b5cd62c8..fc5ee325 100644 --- a/crates/lodedb-cloud-core/src/client_ops.rs +++ b/crates/lodedb-cloud-core/src/client_ops.rs @@ -1,7 +1,7 @@ //! Client-edge operations: the six verbs a frontend invokes by target string. //! //! Frontends (the `orecloud` CLI via the Python binding today; `lodedb cloud` / -//! `LodeDBCloud` later) name each end of an operation with one string — a local +//! `LodeDBCloud` later) name each end of an operation with one string, a local //! directory path or an `s3://bucket/prefix` URL. This module owns the //! composition from those strings to the typed primitives (`export_generation`, //! `verify_generation`, `status_for_push`, the sync classifier), so every @@ -49,7 +49,7 @@ pub struct PullOutcome { /// An explicit override of the sync classification. /// -/// A force overrides the *classification* only — never the stores' integrity +/// A force overrides the *classification* only, never the stores' integrity /// invariants. In particular, divergent lineages that reuse an artifact file /// name with different bytes (a same-epoch fork) still fail closed against /// plain directory/`s3://` remotes; see [`sync`]. @@ -110,9 +110,9 @@ pub fn push( } /// Restores `index_key`'s committed generation from `remote` into the local -/// `dir`, then proves the engine can open the restored copy read-only — -/// restore-verifies-before-accepting as one operation, so no -/// frontend can offer a pull that skips the check. +/// `dir`, then proves the engine can open the restored copy read-only. +/// Restore and verify run as one operation, so no frontend can offer a +/// pull that skips the check. /// /// The pull ships the remote verbatim ([`TransferPolicy::full`]): a remote that /// was pushed redacted already omits text, so there is nothing to filter on the @@ -159,7 +159,7 @@ pub fn pull(remote: &str, dir: &str, index_key: &str) -> Result { /// The shared local-restore tail: stage the artifacts, prove the candidate /// opens through the engine (from a scratch layout), and only then publish -/// the pointer — so every acceptance failure leaves the destination on its +/// the pointer, so every acceptance failure leaves the destination on its /// previous committed generation. `discard_wal` truncates the destination /// WAL right after the swap (force-pull semantics); callers must have /// refused a pending WAL beforehand when not forcing. The journal manifests @@ -170,8 +170,8 @@ pub fn pull(remote: &str, dir: &str, index_key: &str) -> Result { /// beside the new lineage (force-pull only), and a failure between the swap /// and the journal rebuild leaves the restored copy readable but /// fail-closed on its first O(changed) mutation. Closing them fully needs a -/// recoverable restore transaction (engine-side recovery on open) — a -/// follow-up, not this milestone. +/// recoverable restore transaction (engine-side recovery on open), a +/// follow-up to this version. /// /// Caller holds the directory writer lock. fn restore_staged( @@ -217,9 +217,9 @@ fn acquire_writer_lock(dir: &str) -> Result } /// Refuses a restore when the destination WAL still holds acknowledged -/// operations: replaying them onto the pulled lineage — or silently dropping -/// them — would corrupt or lose acknowledged writes. `hint` names the caller's -/// resolution path. +/// operations: replaying them onto the pulled lineage would corrupt +/// acknowledged writes, and silently dropping them would lose them. `hint` +/// names the caller's resolution path. pub(crate) fn ensure_no_pending_wal(dir: &str, index_key: &str, hint: &str) -> Result<()> { let ops = pending_wal_ops(dir, index_key)?; if ops > 0 { @@ -305,15 +305,15 @@ pub fn verify(target: &str, index_key: &str) -> Result { /// - `InSync` is a no-op; `LocalAhead`/`Republish` push; `RemoteAhead` pulls /// (restore + verify-open, exactly like [`pull`]). /// - `Diverged`/`Unknown` refuse with -/// [`ArtifactStoreError::SyncConflict`] — overwriting either end would +/// [`ArtifactStoreError::SyncConflict`]. Overwriting either end would /// discard a commit, a decision only the caller can make via `force`. /// - A forced push/pull skips the refusal but nothing else: the transfer still /// goes through the pointer CAS and (for pulls) the verify-open. Force also /// cannot override the stores' immutability invariant: two lineages that /// diverged from one base and reuse the same artifact file names with -/// different bytes (a same-epoch fork) fail closed with a recovery hint — -/// resolving that shape against a dumb remote needs the managed -/// content-addressed layout of a later milestone. +/// different bytes (a same-epoch fork) fail closed with a recovery hint. +/// Resolving that shape against a dumb remote needs the managed +/// content-addressed layout. /// /// Transfers are conditional on the exact remote/local state that was /// classified: a concurrent advance of the destination between classification @@ -321,7 +321,7 @@ pub fn verify(target: &str, index_key: &str) -> Result { /// that guarantee is depends on the destination backend: object stores use a /// genuinely atomic conditional write, while a directory's pointer swap is /// read-check-then-replace ([`LocalArtifactStore`](crate::LocalArtifactStore) -/// documents this) — so running sync concurrently with an *active engine +/// documents this). So running sync concurrently with an *active engine /// writer* on the same directory is out of contract, exactly as LodeDB's /// single-writer model already requires. Any successful transfer records the /// new base in the sidecar. @@ -372,7 +372,7 @@ pub fn sync( SyncForce::Pull => (false, true), SyncForce::None => match classification { SyncClassification::InSync => { - // No transfer — but if the two ends agree while the recorded + // No transfer. But if the two ends agree while the recorded // base is missing (fresh clone of an already-synced pair, a // remote switch to an identical mirror) or stale (a prior // transfer whose sidecar write failed), record the agreed @@ -436,7 +436,7 @@ pub fn sync( // Pull mirrors `pull`: take the writer lock (a live engine writer and // a restore must never interleave), refuse or discard a pending WAL, // stage + candidate-verify + publish, then record the base. The CAS - // preconditions on the raw local body read above — a concurrent + // preconditions on the raw local body read above, so a concurrent // engine commit fails as a PointerConflict rather than being // clobbered. let source_raw = remote_body.ok_or_else(|| { @@ -478,10 +478,10 @@ fn is_local_dir(target: &str) -> bool { /// /// Two lineages that diverged from one base commonly reuse the same engine /// artifact names (`.gen/g.*`) with different bytes; the store's -/// immutability invariant then refuses the overwrite — deliberately, and force -/// flags do not override it (dumb path/`s3://` remotes store artifacts by +/// immutability invariant then refuses the overwrite. That refusal is deliberate, +/// and force flags do not override it (dumb path/`s3://` remotes store artifacts by /// engine name until the managed content-addressed layout, which absorbs this, -/// arrives in a later milestone). Matching on the store's message text is +/// arrives). Matching on the store's message text is /// acceptable here because both sites live in this crate and the immutability /// tests pin the wording. pub(crate) fn explain_fork_collision(error: ArtifactStoreError) -> ArtifactStoreError { @@ -502,7 +502,7 @@ pub(crate) fn explain_fork_collision(error: ArtifactStoreError) -> ArtifactStore /// Builds the identity for a committed body: the id pair, the generation, and /// a per-store identity for each payload store (a store is carried iff its -/// sub-manifest is non-null — the same test the engine uses to load it; the +/// sub-manifest is non-null, the same test the engine uses to load it; the /// identity is a digest of that sub-manifest, which names every artifact and /// checksum the store pins). pub(crate) fn snap_ref(body: &Value) -> Result { @@ -524,17 +524,17 @@ pub(crate) fn snap_ref(body: &Value) -> Result { /// /// Local directory targets normalize to an absolute path (resolving symlinks /// and `.`/`..` like the store's own containment checks do), because a -/// relative spelling is cwd-dependent — the same string can name unrelated +/// relative spelling is cwd-dependent. The same string can name unrelated /// directories from different working directories, and trusting on the raw /// string would let one remote inherit another's history. /// /// `s3://` targets additionally carry the effective endpoint: the URL alone /// names a backend only through `AWS_ENDPOINT` (MinIO/R2 use the same /// `s3://bucket/prefix` spelling against different services), so the endpoint -/// is part of the identity. Credentials and region are deliberately not — -/// rotating keys or fixing a region points at the *same* bucket. A +/// is part of the identity. Credentials and region are deliberately not part +/// of it; rotating keys or fixing a region points at the *same* bucket. A /// normalization failure or identity mismatch can only produce a false -/// *mismatch* — failing toward force, never toward a wrong fast-forward. +/// *mismatch*, which fails toward force, never toward a wrong fast-forward. fn remote_identity(remote: &str) -> String { if is_local_dir(remote) { return crate::paths::canonical_identity(Path::new(remote)); diff --git a/crates/lodedb-cloud-core/src/digest.rs b/crates/lodedb-cloud-core/src/digest.rs index 5b4961c5..a91d1dd2 100644 --- a/crates/lodedb-cloud-core/src/digest.rs +++ b/crates/lodedb-cloud-core/src/digest.rs @@ -1,6 +1,6 @@ //! Shared artifact checksum helper. //! -//! Every artifact is addressed by the lowercase-hex SHA-256 of its bytes — the +//! Every artifact is addressed by the lowercase-hex SHA-256 of its bytes, the //! exact digest the engine records per artifact (`sha256_file_hex`). Both the //! write path (`LocalArtifactStore::write_bytes_if_absent`) and the verify path //! (`verify_generation`) re-hash bytes and compare against the manifest's recorded @@ -20,7 +20,7 @@ pub(crate) fn sha256_hex_finish(hasher: Sha256) -> String { } /// Streams `reader` to EOF and returns its lowercase-hex SHA-256 plus the byte -/// count — the bounded-memory replacement for hashing a fully buffered +/// count, the bounded-memory replacement for hashing a fully buffered /// artifact (peak memory is one copy buffer, not the artifact). pub(crate) fn sha256_hex_reader(reader: &mut dyn Read) -> std::io::Result<(String, u64)> { let mut hasher = Sha256::new(); diff --git a/crates/lodedb-cloud-core/src/error.rs b/crates/lodedb-cloud-core/src/error.rs index 19321fa0..653fcb99 100644 --- a/crates/lodedb-cloud-core/src/error.rs +++ b/crates/lodedb-cloud-core/src/error.rs @@ -21,7 +21,7 @@ pub enum ArtifactStoreError { NotFound(String), /// A root-pointer compare-and-swap precondition failed: the committed /// generation was not the one the caller expected. Kept distinct because it - /// is the *retryable* failure — a concurrent writer advanced the pointer. + /// is the *retryable* failure (a concurrent writer advanced the pointer). PointerConflict { key: String, expected: Option, @@ -53,12 +53,12 @@ pub enum ArtifactStoreError { /// destination's WAL still holds acknowledged-but-uncheckpointed /// operations. Replaying those records onto a pulled lineage (or silently /// dropping them) would corrupt or lose acknowledged writes, so the caller - /// must checkpoint the store first — or explicitly discard the records + /// must checkpoint the store first, or explicitly discard the records /// with a force-pull. Like [`SyncConflict`](Self::SyncConflict), this is a /// *decision*, not a race. PendingWal { ops: usize, hint: String }, /// A storage backend (e.g. an object store) failed for a reason that is not a - /// missing object or a pointer precondition — a network error, a permission + /// missing object or a pointer precondition: a network error, a permission /// denial, or any other transport-level failure. The message carries the /// backend's own diagnostic. Missing objects map to [`NotFound`](Self::NotFound) /// and failed conditional writes to [`PointerConflict`](Self::PointerConflict), diff --git a/crates/lodedb-cloud-core/src/generation_inventory.rs b/crates/lodedb-cloud-core/src/generation_inventory.rs index 2205d403..e8819606 100644 --- a/crates/lodedb-cloud-core/src/generation_inventory.rs +++ b/crates/lodedb-cloud-core/src/generation_inventory.rs @@ -1,7 +1,7 @@ //! Read-only inventory of the artifacts a committed generation references. //! -//! Given a committed root manifest, this enumerates every artifact it pins — -//! each per-store base plus its delta segments — as [`ArtifactRef`] records +//! Given a committed root manifest, this enumerates every artifact it pins +//! (each per-store base plus its delta segments) as [`ArtifactRef`] records //! carrying name, checksum, size, kind, base epoch, and base-vs-delta. It reads //! only the committed root (via `read_commit_manifest`); it never stats the //! files, reads the `.wal` tail, or reads the on-disk per-store `manifest.json`. @@ -9,8 +9,8 @@ //! //! [`diff_inventories`] compares a local inventory against a remote one and //! reports which artifacts are missing remotely, distinguishing "delta segments -//! onto an existing base epoch" from "a whole new base epoch" — the signal a -//! push needs to stay O(changed). +//! onto an existing base epoch" from "a whole new base epoch". A push needs +//! that distinction to stay O(changed). use crate::error::{ArtifactStoreError, Result}; use crate::paths::resolve_within; @@ -39,10 +39,10 @@ const GEN_DIR_SUFFIX: &str = ".gen"; /// suffix, e.g. `g7.json` -> `g7.json.json-delta/`). The key doubles as the base /// file extension, since the engine derives base paths as `g.`. A /// store is inventoried iff its sub-manifest is non-null; `tvmv` (multi-vector / -/// late-interaction) is included — omitting it would silently drop those artifacts -/// from a backup. `tvann` (the persisted ANN cluster partition) is base-only — -/// the engine treats a missing/corrupt `.tvann` as a cache miss and rebuilds — -/// but it still ships: a body referencing a base we never uploaded would fail +/// late-interaction) is included, since omitting it would silently drop those +/// artifacts from a backup. `tvann` (the persisted ANN cluster partition) is +/// base-only, and the engine treats a missing/corrupt `.tvann` as a cache miss +/// and rebuilds. It still ships: a body referencing a base we never uploaded would fail /// byte-verification on pull, and shipping it saves the restored/hydrated copy /// a corpus-sized re-cluster. Its delta suffix follows the engine's uniform /// `.-delta` derivation (no deltas are ever recorded today). @@ -55,7 +55,7 @@ const STORE_KINDS: &[(&str, &str)] = &[ ("tvann", ".tvann-delta"), // `tvvf` (the rescore original-vector sidecar, engine 1.3.2+) is a journaled // {base, deltas} store like tvim: vector payload, never text-gated, and the - // engine refuses to open a rescore store without it — so it ships always. + // engine refuses to open a rescore store without it, so it ships always. ("tvvf", TVVF_DELTA_DIR_SUFFIX), ]; @@ -93,8 +93,8 @@ pub struct GenerationInventory { /// What a local generation has that a remote one does not. /// -/// `ships_base` is true when the transfer must upload a base artifact — a new -/// base epoch (cold build or compaction) *or* a replacement base under an epoch +/// `ships_base` is true when the transfer must upload a base artifact, either +/// a new base epoch (cold build or compaction) or a replacement base under an epoch /// the remote already holds at a different checksum (two divergent lineages at /// the same epoch number). It is false only when every uploaded artifact is a /// delta segment onto a base the remote already holds byte-for-byte (the @@ -140,7 +140,7 @@ pub fn inventory_from_body( let base_epoch = body_u64(body, "base_epoch"); // Fail closed on a store this table does not know: a future engine that // adds a sub-manifest (as `tvann` was added) must not have its artifacts - // silently dropped from a backup — an inventory that understates what the + // silently dropped from a backup. An inventory that understates what the // body pins ships a generation whose blobs were never uploaded. Store // sub-manifests are recognizable by shape (an object carrying a journaled // `base`), which no scalar body field (`generation`, `native_dim`, …) has. @@ -264,7 +264,7 @@ pub fn diff_inventories( /// Pulls sha256 and byte size straight from the manifest entries (no file stat). /// A sub-manifest that is absent or empty yields nothing; one carrying the legacy /// pre-journal marker (`present` with no journaled `base`) fails closed rather -/// than silently omit its artifacts from a backup — rewrite the generation to +/// than silently omit its artifacts from a backup. Rewrite the generation to /// migrate it to the journaled layout first. fn refs_for_store( index_key: &str, @@ -276,7 +276,7 @@ fn refs_for_store( // Distinguish an absent store (missing key or explicit null -> no artifacts) // from a present-but-malformed one. The engine's `store_manifest` treats any // non-null value as present, so a non-null, non-object manifest is corrupt and - // must fail closed rather than be silently skipped — skipping would publish a + // must fail closed rather than be silently skipped. Skipping would publish a // body the engine then refuses to open. let sub_manifest = match sub_manifest { None | Some(Value::Null) => return Ok(Vec::new()), @@ -312,7 +312,7 @@ fn refs_for_store( // rather than copying the wrong file under a name the engine cannot find. // Most stores live at `g.`; `tvvf` keeps its own epoch // counter in the sub-manifest and lives at `vf.tvvf` - // (`tvvf_store::base_path`), so its refs must derive from that epoch — the + // (`tvvf_store::base_path`), so its refs must derive from that epoch. The // generation's base_epoch names a file the engine never writes. let (store_epoch, base_name) = if kind == "tvvf" { let vf_epoch = sub_manifest @@ -376,11 +376,11 @@ fn refs_for_store( /// /// The engine writes one journal manifest per store (in /// `.-delta/manifest.json`) whose content IS the store's -/// sub-manifest in the commit body — but the journal file itself is engine +/// sub-manifest in the commit body. The journal file itself is engine /// working state, not an artifact the body pins, so transfers never ship it. /// The read path doesn't need it (it reads the committed root), but the /// engine's O(changed) mutation path appends journal deltas through it and -/// fails closed when it is missing — which would make a restored directory +/// fails closed when it is missing, which would leave a restored directory /// readable but not writable. Restores call this to rebuild every journal /// manifest verbatim from the body, so a pulled/hydrated copy behaves /// exactly like an engine-authored one. `tvann` is excluded: the ANN sidecar @@ -440,7 +440,7 @@ pub(crate) fn write_restored_journal_manifests( /// Inventory digests cross the trust boundary twice: a managed pull uses them /// as staging *file names* (`/`) and the managed layout as /// object-key segments, so a value that is not exactly 64 lowercase hex -/// characters — an absolute path, a `../` traversal, an empty string — must +/// characters (an absolute path, a `../` traversal, an empty string) must /// fail closed here, before it can name a path anywhere downstream. fn checked_sha256(kind: &str, entry: &Map) -> Result { let digest = str_field(entry, "sha256"); @@ -457,9 +457,9 @@ fn checked_sha256(kind: &str, entry: &Map) -> Result { /// /// Engine artifact names are always single components (`g7.json`, /// `delta-00000000.jsd`). A name containing a path separator or a `.`/`..` segment -/// would, when joined onto `.gen/`, point outside that directory — still -/// inside the store root (`resolve_within` confines it there), but under another -/// index's key — letting a tampered source manifest plant a file during restore. +/// would, when joined onto `.gen/`, point outside that directory. The result +/// stays inside the store root (`resolve_within` confines it there) but lands under +/// another index's key, so a tampered source manifest could plant a file during restore. /// Fails closed; every legitimate manifest passes. fn ensure_plain_file_name(kind: &str, file_name: &str) -> Result<()> { let is_unsafe = file_name.is_empty() diff --git a/crates/lodedb-cloud-core/src/lib.rs b/crates/lodedb-cloud-core/src/lib.rs index bc20c5c1..d3aaabed 100644 --- a/crates/lodedb-cloud-core/src/lib.rs +++ b/crates/lodedb-cloud-core/src/lib.rs @@ -4,7 +4,7 @@ //! addressed commit format. Rather than reimplement (or import the hollowing-out //! Python implementation of) that format, this crate links `lodedb-core`'s //! `storage` module directly, so the cloud and the embedded engine share ONE -//! commit-format implementation — schema, on-disk layout, canonical-JSON +//! commit-format implementation: schema, on-disk layout, canonical-JSON //! checksum, and every sub-manifest (`json`/`tvim`/`tvtext`/`tvlex`/`tvmv`/`tvann`/`tvvf`). //! //! It provides an [`ArtifactStore`] abstraction with a filesystem default diff --git a/crates/lodedb-cloud-core/src/local_artifact_store.rs b/crates/lodedb-cloud-core/src/local_artifact_store.rs index 5b0a31e2..24326e0b 100644 --- a/crates/lodedb-cloud-core/src/local_artifact_store.rs +++ b/crates/lodedb-cloud-core/src/local_artifact_store.rs @@ -2,7 +2,7 @@ //! //! A committed generation already lives on disk as immutable `g.*` //! artifacts under `.gen/` pinned by `.commit.json`, so the local store -//! is a thin wrapper over `lodedb-core`'s commit-manifest primitives — there is +//! is a thin wrapper over `lodedb-core`'s commit-manifest primitives; there is //! no second format. Object-storage backends (S3/GCS/Azure) belong in a later //! milestone, not here. @@ -42,7 +42,7 @@ impl LocalArtifactStore { /// The idempotence/immutability answer for an already-present name: /// identical content (compared by a streaming re-hash) is Ok, different - /// content refuses — artifacts are immutable. + /// content refuses. Artifacts are immutable. fn refuse_unless_identical(&self, name: &str, path: &Path, sha256: &str) -> Result<()> { let (existing, _bytes) = sha256_hex_reader(&mut File::open(path)?)?; if existing == sha256 { @@ -80,7 +80,7 @@ impl ArtifactStore for LocalArtifactStore { // lineages can collide on a name. Identical bytes are an idempotent // no-op; different bytes are a genuine conflict we refuse rather than // clobber (the immutability invariant). Hash the existing file - // streaming — it can be as large as any transferred base. + // streaming; it can be as large as any transferred base. return self.refuse_unless_identical(name, &path, sha256); } if let Some(parent) = path.parent() { @@ -89,7 +89,7 @@ impl ArtifactStore for LocalArtifactStore { // Stream into a UNIQUELY-NAMED sibling scratch (two concurrent writers // of one name must never share a scratch), hashing as we copy, and // verify the digest BEFORE publication. The publish is - // `persist_noclobber` — an atomic no-replace claim, so the losing + // `persist_noclobber`, an atomic no-replace claim, so the losing // writer of a name race falls into the identical-bytes check instead // of overwriting bytes a committed pointer may already reference. // Peak memory is one copy buffer regardless of artifact size. @@ -129,13 +129,13 @@ impl ArtifactStore for LocalArtifactStore { // rename and the scratch name is gone; the hard-link+unlink // fallback some filesystems take can leave the scratch link // behind on unlink failure, so sweep it (litter, not - // correctness — the published bytes are already in place). + // correctness; the published bytes are already in place). if scratch_path.exists() { let _ = fs::remove_file(&scratch_path); } } Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { - // Lost a same-name race: never replace — compare instead. + // Lost a same-name race: never replace, compare instead. return self.refuse_unless_identical(name, &path, sha256); } Err(error) => return Err(ArtifactStoreError::Io(error.error)), @@ -147,7 +147,7 @@ impl ArtifactStore for LocalArtifactStore { } fn contains(&self, name: &str) -> Result { - // A metadata probe, not a read — `try_exists` surfaces genuine I/O + // A metadata probe, not a read. `try_exists` surfaces genuine I/O // failures (unlike `Path::exists`, which would mask them as absence). let path = resolve_within(&self.root, &self.root.join(name))?; Ok(path.try_exists()?) diff --git a/crates/lodedb-cloud-core/src/managed.rs b/crates/lodedb-cloud-core/src/managed.rs index 474ad5e7..9d7613b6 100644 --- a/crates/lodedb-cloud-core/src/managed.rs +++ b/crates/lodedb-cloud-core/src/managed.rs @@ -3,7 +3,7 @@ //! Managed transfers split the client edge in two: the Python layer speaks //! HTTP to the control plane (begin/commit push sessions, pull plans, //! presigned or proxied blob bytes), while everything that touches the commit -//! format stays here — inventories, canonical identities, the engine-written +//! format stays here: inventories, canonical identities, the engine-written //! pointer document, sidecar trust, classification, and the restore path with //! its verify-open. The seam is deliberate: Python never parses a manifest or //! re-serialises a body, so the canonical-JSON contract has exactly one @@ -14,8 +14,8 @@ //! returned (stored and served as JSON). Identities recomputed from that body //! are stable across the trip because the engine's canonical form is //! key-order-insensitive: `serde_json::Value` objects are sorted maps, so a -//! body that passed through the catalog re-canonicalises to the same digest — -//! pinned by the end-to-end tests. +//! body that passed through the catalog re-canonicalises to the same digest. +//! The end-to-end tests pin this. //! //! Unlike dumb targets, the remote identity string used for sidecar trust is //! supplied by the caller verbatim (`orecloud://org/environment/db#host=…`): the @@ -57,7 +57,7 @@ pub struct ManagedSide { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ManagedLocal { pub side: ManagedSide, - /// The policy-redacted committed body — what a push publishes. + /// The policy-redacted committed body, what a push publishes. pub body: Value, /// The engine-canonical pointer document for `body` (UTF-8 JSON bytes, /// carried as a string). Shipped verbatim at commit so the control plane @@ -76,16 +76,16 @@ pub struct ManagedPlan { pub local: Option, pub remote: Option, pub base: Option, - /// Whether the trusted base already records the remote's current snapshot - /// — when false after an `in_sync` classification, the caller should + /// Whether the trusted base already records the remote's current snapshot. + /// When false after an `in_sync` classification, the caller should /// refresh the sidecar (mirrors `client_ops::sync`'s stale-base repair). pub base_is_current: bool, /// The RAW (unredacted) local pointer's snapshot id at classification /// time; `None` when the directory holds no committed generation. A /// pull-direction materialization pins on this so a local commit landing /// between classification and materialization refuses instead of being - /// silently overwritten — the managed twin of the dumb sync carrying its - /// classified `local_raw` into the pointer CAS. + /// silently overwritten. This is the managed twin of the dumb sync carrying + /// its classified `local_raw` into the pointer CAS. pub local_raw_snapshot_id: Option, } @@ -121,7 +121,7 @@ fn managed_side(body: &Value, reference: &SnapRef) -> ManagedSide { /// when recorded against exactly `remote_id`), and classifies. /// /// `remote_body` is the branch-head body the control plane returned (`None` -/// when the remote holds nothing). Pure local I/O — no network. +/// when the remote holds nothing). Pure local I/O, no network. pub fn managed_plan( dir: &str, index_key: &str, @@ -172,7 +172,7 @@ pub fn managed_plan( let document = pointer_document(&body)?; // Cross-check the document against the recomputed identity: both // come from the same engine writer, so a mismatch means the body - // changed between the two reads — fail closed rather than begin a + // changed between the two reads. Fail closed rather than begin a // push whose commit will contradict itself. let document_id = identity_from_document(&document)?; if document_id != reference.snapshot_id { @@ -212,7 +212,7 @@ pub fn managed_plan( } /// Records `body` as the sidecar base for `remote_id` after a successful -/// managed transfer. `remote_id` is stored verbatim — the caller owns the +/// managed transfer. `remote_id` is stored verbatim; the caller owns the /// canonical spelling of managed remote identities. pub fn managed_record_base( dir: &str, @@ -345,7 +345,7 @@ impl ArtifactStore for StagingBlobStore { /// /// The restore runs under the engine's single-writer lock (a live writer and /// a restore must never interleave) and refuses when the destination WAL -/// still holds acknowledged operations — pass `discard_pending_wal` (the +/// still holds acknowledged operations. Pass `discard_pending_wal` (the /// force-pull semantics) to truncate them along with the local lineage. Every /// blob is re-hashed against the manifest checksum on write (a corrupt /// download fails the pull before any pointer moves), the candidate is @@ -407,7 +407,7 @@ pub fn managed_materialize( // addressed there), but the LOCAL directory still stores artifacts by // engine name: force-pulling a diverged lineage that reuses a name with // different bytes fails closed on the immutability invariant, exactly - // like the dumb-target verbs — recover by pulling into a fresh + // like the dumb-target verbs. Recover by pulling into a fresh // directory. The wrapper attaches that recovery hint. let staged = stage_generation_pinned( &staging, @@ -430,7 +430,7 @@ pub fn managed_materialize( )?; } // Rebuild the journal manifests (working state the body doesn't pin) so - // the restored copy is writable, not just readable — the cloud writer + // the restored copy is writable, not just readable; the cloud writer // opens hydrated copies through this path. Strictly AFTER the swap: // writing them first would, on a failed CAS, leave the candidate's // journals attached to some other writer's committed body. diff --git a/crates/lodedb-cloud-core/src/manifest_transfer.rs b/crates/lodedb-cloud-core/src/manifest_transfer.rs index 6aff5f3d..9fd7b7ff 100644 --- a/crates/lodedb-cloud-core/src/manifest_transfer.rs +++ b/crates/lodedb-cloud-core/src/manifest_transfer.rs @@ -4,17 +4,17 @@ //! then publishes the destination's root pointer. Because the source is read //! through its committed //! root pointer (never the `.wal` tail or the on-disk per-store manifests), only -//! committed state ships — an uncommitted WAL write or a torn half-commit is +//! committed state ships. An uncommitted WAL write or a torn half-commit is //! excluded by construction. //! //! The copy is O(changed): the destination's existing inventory is diffed against //! the source's, so only missing artifacts are uploaded. The destination verifies //! each artifact's checksum on write, and the pointer swap is the single commit //! point, so a crash mid-transfer leaves the destination on its previous -//! generation with some extra immutable blobs — never a torn generation. +//! generation with some extra immutable blobs, never a torn generation. //! //! Both ends are artifact stores, so a transfer is symmetric: a restore is the -//! same call with the stores swapped — the remote/backup store as `source` and +//! same call with the stores swapped, the remote/backup store as `source` and //! the local directory as `dest` (a source that was pushed redacted already omits //! text, so [`TransferPolicy::full`] on the way back restores it verbatim). Follow //! a local restore with @@ -24,7 +24,7 @@ //! Every transfer states its [`TransferPolicy`] explicitly: a redacted push //! publishes a body whose text/lexical sub-manifests are nulled and skips those //! artifacts, so only redacted state leaves the machine; a full policy ships the -//! generation verbatim. There is deliberately no default-policy convenience — +//! generation verbatim. There is deliberately no default-policy convenience; //! shipping payload-bearing stores must be a visible choice at the call site. use crate::artifact_store::ArtifactStore; @@ -58,7 +58,7 @@ pub struct TransferResult { /// [`TransferPolicy::full`] policy ships the generation verbatim. Reads the /// source's committed root, diffs it against whatever the destination already /// holds, uploads only the missing artifacts (each checksum-verified by the -/// destination), and finally swaps the destination's root pointer — the one +/// destination), and finally swaps the destination's root pointer, the one /// commit point. Idempotent: re-running when the destination already holds the /// exact body this transfer would publish uploads nothing and leaves the pointer /// untouched. @@ -98,7 +98,7 @@ pub(crate) fn export_generation_with_body( /// supplied by the caller instead of read here. /// /// The pointer swap preconditions on exactly `dest_body`, so a caller that -/// already read (and classified) the destination — the sync verb — makes its +/// already read (and classified) the destination (the sync verb) makes its /// transfer conditional on the state it decided on: a concurrent advance of /// the destination between that read and this call surfaces as a /// [`PointerConflict`](ArtifactStoreError::PointerConflict), with the @@ -124,7 +124,7 @@ pub(crate) fn export_generation_against( /// too, instead of re-read here. /// /// The sync verb classifies a specific pair of bodies; pinning the source -/// means the transfer ships exactly the classified snapshot — a concurrent +/// means the transfer ships exactly the classified snapshot, so a concurrent /// source change cannot swap in a body the classifier never saw. The /// artifact bytes are still read live by name, but every name/checksum comes /// from the pinned body and the destination re-hashes on write, so a mutated @@ -149,7 +149,7 @@ pub(crate) fn export_generation_pinned( /// Splitting staging from publication lets local restores insert their /// acceptance checks (candidate verify-open, journal reconstruction) between /// the two, so a failed check leaves the destination on its previous committed -/// generation — the pointer swap stays the single commit point. +/// generation. The pointer swap stays the single commit point. pub(crate) struct StagedExport { pub source_body: serde_json::Value, pub dest_body: Option, @@ -163,7 +163,7 @@ pub(crate) struct StagedExport { /// destination pointer. /// /// The destination re-hashes each artifact on write, so a corrupt source -/// artifact fails here — before any pointer moves. Redaction happens before +/// artifact fails here, before any pointer moves. Redaction happens before /// inventorying, exactly as in [`export_generation_pinned`]. pub(crate) fn stage_generation_pinned( source: &dyn ArtifactStore, @@ -185,7 +185,7 @@ pub(crate) fn stage_generation_pinned( for artifact in &diff.to_upload { // Streamed end to end: the destination hashes while it copies, so the // transfer's peak memory is a fixed buffer (or one multipart chunk on - // object storage) — never the artifact, which for a vector base can + // object storage), never the artifact, which for a vector base can // be gigabytes. The manifest-recorded size rides along as the // strategy hint (it is advisory; the digest gate stays authoritative). let mut reader = source.open_read(&artifact.name)?; @@ -207,10 +207,10 @@ pub(crate) fn stage_generation_pinned( }) } -/// Publishes a staged transfer's pointer — the single commit point. +/// Publishes a staged transfer's pointer, the single commit point. /// /// Skips the swap only when the destination already holds this exact committed -/// *body* — comparing the full body, not just the generation integer, because +/// *body*. It compares the full body, not just the generation integer, because /// two independent lineages can share a generation number with different /// content. The swap preconditions on the exact body staging read /// (`dest_body`), so a concurrent change between read and swap is caught as a diff --git a/crates/lodedb-cloud-core/src/object_artifact_store.rs b/crates/lodedb-cloud-core/src/object_artifact_store.rs index 7a7379b8..e4a8cdca 100644 --- a/crates/lodedb-cloud-core/src/object_artifact_store.rs +++ b/crates/lodedb-cloud-core/src/object_artifact_store.rs @@ -80,7 +80,7 @@ impl ObjectArtifactStore { } /// Claims the final artifact name from a completed scratch upload: - /// `copy_if_not_exists` where the backend supports it (atomic — a + /// `copy_if_not_exists` where the backend supports it (atomic, so a /// concurrent claimant loses cleanly and falls into the identical-bytes /// check), probe-then-copy where it does not (the residual race, now /// confined to backends without any conditional copy, e.g. plain S3 @@ -109,7 +109,7 @@ impl ObjectArtifactStore { /// The idempotence/immutability answer for an already-present name: /// identical content (compared by a streaming re-hash) is Ok, different - /// content refuses — artifacts are immutable. + /// content refuses. Artifacts are immutable. fn refuse_unless_identical(&self, name: &str, sha256: &str) -> Result<()> { let mut existing = self.open_read(name)?; let (digest, _bytes) = crate::digest::sha256_hex_reader(&mut *existing)?; @@ -157,11 +157,11 @@ struct GetBytes { /// atomic create-if-absent); anything larger streams as a sequential multipart /// upload with `MULTIPART_CHUNK_BYTES` parts, bounding memory to one part. /// Object stores offer no conditional create for multipart, so the large path -/// claims via a scratch key — see `write_stream_if_absent`. +/// claims via a scratch key (see `write_stream_if_absent`). const MULTIPART_THRESHOLD_BYTES: usize = 8 * 1024 * 1024; const MULTIPART_CHUNK_BYTES: usize = 8 * 1024 * 1024; -/// S3 caps a single copy operation (and one `UploadPartCopy` part — what the +/// S3 caps a single copy operation (and one `UploadPartCopy` part, which the /// `AWS_COPY_IF_NOT_EXISTS=multipart` claim uses) at 5 GiB, so the /// scratch-then-conditional-claim strategy only works below it. Larger /// artifacts stream their multipart directly onto the final name behind the @@ -171,8 +171,8 @@ const MULTIPART_CHUNK_BYTES: usize = 8 * 1024 * 1024; /// residual race. The size hint decides the strategy up front; an /// unknown-size stream that crosses this ceiling mid-upload fails early /// (never at claim time, after every byte moved), and a hint that overshoots -/// the ceiling merely routes a smaller object through the direct path — a -/// strategy choice, never an integrity one (the digest gate is unconditional). +/// the ceiling merely routes a smaller object through the direct path. That is +/// a strategy choice, never an integrity one (the digest gate is unconditional). const CLAIMABLE_LIMIT_BYTES: u64 = 5 * 1024 * 1024 * 1024; /// Process-local uniquifier for scratch upload keys: two same-process uploads @@ -181,7 +181,7 @@ static SCRATCH_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::Atomic /// A synchronous [`Read`] over an object's byte stream: each `read` pulls the /// next chunk through the store's own current-thread runtime, so a download's -/// peak memory is one network chunk — never the object. +/// peak memory is one network chunk, never the object. struct BlockingRead<'a> { runtime: &'a Runtime, stream: BoxStream<'static, object_store::Result>, @@ -271,8 +271,8 @@ impl ArtifactStore for ObjectArtifactStore { // Large object. Multipart completion has no conditional-create mode, // so completing directly on the final name could overwrite a - // concurrent writer's artifact AFTER that writer's pointer committed - // — silent corruption of a committed generation. Instead the parts + // concurrent writer's artifact AFTER that writer's pointer committed, + // silently corrupting a committed generation. Instead the parts // stream to a unique scratch key and the final name is claimed with // `copy_if_not_exists`, which is atomic wherever the backend supports // it (natively on the in-memory test store; via @@ -280,7 +280,7 @@ impl ArtifactStore for ObjectArtifactStore { // back to probe-then-copy where it is not. Above the provider copy // ceiling (`CLAIMABLE_LIMIT_BYTES`) no claim primitive exists at all, // so those artifacts stream directly onto the final name behind the - // probe — the documented residual race, confined to >5 GiB objects. + // probe. That is the documented residual race, confined to >5 GiB objects. if self.contains(name)? { return self.refuse_unless_identical(name, sha256); } @@ -288,7 +288,7 @@ impl ArtifactStore for ObjectArtifactStore { // The scratch key embeds the EXPECTED digest: even if two uploaders // somewhere collided on the rest of the key (pid + clock + counter // are only process-unique), they can only share a scratch when they - // are writing identical content — a substitution can never smuggle + // are writing identical content, so a substitution can never smuggle // bytes past the digest gate into the claim. let scratch = claim_via_scratch.then(|| { self.object_path(&format!( @@ -304,7 +304,7 @@ impl ArtifactStore for ObjectArtifactStore { let upload_target = scratch.as_ref().unwrap_or(&path); // One FIXED part size for the whole upload, chosen from the size // hint: R2 requires every non-final part to be the same size, and - // S3 caps an upload at 10,000 parts — so the hint scales the part + // S3 caps an upload at 10,000 parts. So the hint scales the part // size up front (with headroom for a hint that undershoots) instead // of growing parts mid-stream. let part_size = if size_hint > 0 { @@ -468,15 +468,15 @@ impl ArtifactStore for ObjectArtifactStore { } } -/// Serialises a committed body into pointer-document bytes — the engine's own -/// rendering, via [`snapshot_identity`](crate::snapshot_identity). +/// Serialises a committed body into pointer-document bytes, using the engine's +/// own rendering via [`snapshot_identity`](crate::snapshot_identity). fn serialize_pointer_document(body: &Value) -> Result> { crate::snapshot_identity::pointer_document(body) } /// Validates pointer-document bytes and returns the committed body, through -/// the engine's own schema + `body_sha256` validation (`parse_commit_manifest` -/// — no scratch file; this runs on every pointer read). +/// the engine's own schema + `body_sha256` validation (`parse_commit_manifest`, +/// with no scratch file; this runs on every pointer read). /// /// Fails closed on a garbled pointer (bad schema version or body checksum), so /// a corrupted object never loads as a valid generation. diff --git a/crates/lodedb-cloud-core/src/paths.rs b/crates/lodedb-cloud-core/src/paths.rs index 149e0a02..dd6d39e9 100644 --- a/crates/lodedb-cloud-core/src/paths.rs +++ b/crates/lodedb-cloud-core/src/paths.rs @@ -3,7 +3,7 @@ //! A committed generation's artifact *names* and pointer *keys* are joined onto a //! store root to form on-disk paths. Once a name or key can originate from CLI or //! remote/control-plane input (the cloud trust boundary), a crafted `..` segment -//! or absolute path could otherwise read or write outside the store root — a +//! or absolute path could otherwise read or write outside the store root, a //! cross-tenant disclosure in a multi-tenant store. [`resolve_within`] is the //! single primitive that confines every such join to its root. @@ -39,7 +39,7 @@ pub fn resolve_within(root: &Path, candidate: &Path) -> Result { } /// A path's canonical identity string: absolute, symlink-resolved (through the -/// existing prefix), `.`/`..`-normalised — the same resolution +/// existing prefix), `.`/`..`-normalised, the same resolution /// [`resolve_within`] uses. Two spellings of one directory (relative vs /// absolute, redundant `.` segments) map to one identity. On a resolution /// failure the raw spelling is returned, which for the sidecar-trust caller can @@ -145,7 +145,7 @@ mod tests { #[test] fn accepts_a_not_yet_created_root() { // A fresh backup destination that does not exist yet must resolve, not - // fail with ENOENT — the store creates it on first write. + // fail with ENOENT; the store creates it on first write. let root = temp_dir("fresh-parent").join("new-backup"); let resolved = resolve_within(&root, &root.join("idx.commit.json")).unwrap(); assert!(resolved.ends_with("new-backup/idx.commit.json")); diff --git a/crates/lodedb-cloud-core/src/snapshot_identity.rs b/crates/lodedb-cloud-core/src/snapshot_identity.rs index 2b5f80e7..ba9bc2bf 100644 --- a/crates/lodedb-cloud-core/src/snapshot_identity.rs +++ b/crates/lodedb-cloud-core/src/snapshot_identity.rs @@ -5,11 +5,11 @@ //! tokens (two independent lineages can share one). The answer is the engine's //! own canonical body digest: //! -//! - [`snapshot_id`] — the `body_sha256` the engine records in +//! - [`snapshot_id`]: the `body_sha256` the engine records in //! `.commit.json` for exactly this body. Two stores hold the same bytes //! iff their snapshot ids match; storage and authorization decisions key on //! it. -//! - [`logical_id`] — the snapshot id of the body's fully *redacted* form. A +//! - [`logical_id`]: the snapshot id of the body's fully *redacted* form. A //! full push and a redacted push of one engine commit share a `logical_id` //! while their `snapshot_id`s differ, so lineage comparison ("is this the //! same commit?") runs on `logical_id` and never mistakes a redaction for a @@ -18,7 +18,7 @@ //! The digest is over the engine's canonical JSON, which this crate must never //! reimplement: `lodedb-core` exports `commit_body_sha256` and //! `render_commit_manifest`, so identities and pointer documents come straight -//! from the engine's own writer — no scratch-file round trip, which matters +//! from the engine's own writer with no scratch-file round trip. That matters //! because every sync/status classifies several identities per run and the //! object-store backend validates a pointer document on every read. @@ -29,7 +29,7 @@ use lodedb_core::storage::commit_manifest::{ }; use serde_json::Value; -/// Serialises `body` into the engine's canonical pointer-document bytes — +/// Serialises `body` into the engine's canonical pointer-document bytes, /// the exact `{"body":…,"body_sha256":…,"schema_version":…}` file a /// `.commit.json` pointer carrying this body holds on disk. /// @@ -38,7 +38,7 @@ use serde_json::Value; /// are byte-identical to an engine-written pointer and `read_commit_manifest` /// validates the result the same way. The managed transfer plane ships these /// bytes to the control plane, which stores them verbatim for the -/// object-store pointer mirror — the server never re-serialises a pointer +/// object-store pointer mirror; the server never re-serialises a pointer /// itself. pub fn pointer_document(body: &Value) -> Result> { Ok(render_commit_manifest(body)?.into_bytes()) @@ -54,7 +54,7 @@ pub fn identity_from_document(document: &[u8]) -> Result { Ok(manifest.body_sha256) } -/// Returns the engine's canonical `body_sha256` for `body` — the digest a +/// Returns the engine's canonical `body_sha256` for `body`, the digest a /// `.commit.json` pointer carrying this exact body records. pub fn snapshot_id(body: &Value) -> Result { Ok(commit_body_sha256(body)?) @@ -64,7 +64,7 @@ pub fn snapshot_id(body: &Value) -> Result { /// /// Redaction ([`TransferPolicy::redacted`]) nulls the payload-bearing stores, /// and redacting an already-redacted body is a no-op, so every push of one -/// engine commit — full, redacted, or partial — maps to one `logical_id`. A +/// engine commit (full, redacted, or partial) maps to one `logical_id`. A /// corollary used by the sync classifier: a body is fully redacted iff its /// `snapshot_id` equals its `logical_id`. pub fn logical_id(body: &Value) -> Result { diff --git a/crates/lodedb-cloud-core/src/status.rs b/crates/lodedb-cloud-core/src/status.rs index 0bb08b60..27850af4 100644 --- a/crates/lodedb-cloud-core/src/status.rs +++ b/crates/lodedb-cloud-core/src/status.rs @@ -1,20 +1,20 @@ //! Compare a local generation against a remote one. //! //! [`compare_generations`] answers "what would a push move, and are the two ends -//! already in sync?" purely from two inventories — no bytes are read. It composes +//! already in sync?" purely from two inventories; no bytes are read. It composes //! [`diff_inventories`](crate::diff_inventories), so the O(changed) upload set and //! the base-vs-delta signal are computed the same way the transfer computes them. //! //! The comparison is direction-aware (push: local -> remote) and compares the two //! inventories exactly as given. A caller applying a [`TransferPolicy`] should //! build the local inventory from the *redacted* body, so `in_sync` reflects the -//! redacted push it intends to make — [`status_for_push`] does exactly that over +//! redacted push it intends to make. [`status_for_push`] does exactly that over //! two stores. `in_sync` means a push would move nothing at //! all: no artifacts to upload *and* the remote's committed body already equals the //! body the push would publish. The body check matters because a push republishes -//! the pointer whenever the bodies differ even when no bytes move — e.g. a redacted -//! push against a previously-full remote uploads nothing yet must still swap the -//! pointer to drop the text reference, so it is correctly reported as not in sync. +//! the pointer whenever the bodies differ even when no bytes move. For example, a +//! redacted push against a previously-full remote uploads nothing yet must still swap +//! the pointer to drop the text reference, so it is correctly reported as not in sync. use crate::artifact_store::ArtifactStore; use crate::error::Result; @@ -26,7 +26,7 @@ use crate::transfer_policy::TransferPolicy; /// `local_*`/`remote_*` are `None` when that side holds no committed generation. /// `artifacts_to_upload`/`bytes_to_upload`/`ships_base` describe a push from local /// to remote (the O(changed) upload set). `in_sync` is true when a push would move -/// nothing — the remote already holds every artifact the local side would ship. +/// nothing; the remote already holds every artifact the local side would ship. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusReport { pub index_key: String, @@ -42,13 +42,13 @@ pub struct StatusReport { pub in_sync: bool, /// Whether a *trusted* sync sidecar for this remote was found next to the /// local index (valid checksum, recorded against the same remote target). - /// Always false from [`compare_generations`]/[`status_for_push`] — the + /// Always false from [`compare_generations`]/[`status_for_push`]. The /// sidecar lives on the local filesystem, so only the client edge /// ([`client_ops::status`](crate::client_ops::status)) can fill lineage. pub sidecar_present: bool, /// Whether a sidecar file was found but failed validation (torn or /// tampered) and was therefore ignored. Distinct from a sidecar recorded - /// against a different remote, which is valid — just not trusted here. + /// against a different remote, which is valid, just not trusted here. pub sidecar_corrupt: bool, /// The recorded base generation from the sidecar, when one was trusted. pub base_generation: Option, @@ -119,7 +119,7 @@ pub fn compare_generations( let diff = diff_inventories(local, remote); // A push moves nothing only when there are no artifacts to upload AND the - // remote already holds the exact body the push would publish — otherwise the + // remote already holds the exact body the push would publish; otherwise the // pointer is still re-swapped (e.g. a redaction that drops a store reference // without uploading any bytes). let body_matches_remote = remote.is_some_and(|remote| remote.root_body == local.root_body); diff --git a/crates/lodedb-cloud-core/src/store_target.rs b/crates/lodedb-cloud-core/src/store_target.rs index 80e6c504..996e15de 100644 --- a/crates/lodedb-cloud-core/src/store_target.rs +++ b/crates/lodedb-cloud-core/src/store_target.rs @@ -1,7 +1,7 @@ //! Resolve a user-facing transfer target into an [`ArtifactStore`]. //! //! The client edge (CLI / Python binding) names each end of a transfer with one -//! string — a local directory path or an object-store URL — so the mapping from +//! string, a local directory path or an object-store URL, so the mapping from //! that string to a store lives here, once, rather than in every frontend: //! //! - `s3://bucket/prefix` → [`ObjectArtifactStore`] over Amazon S3 or any @@ -11,12 +11,12 @@ //! read by `AmazonS3Builder::from_env`. The URL path becomes the per-tenant key //! prefix. Setting `AWS_COPY_IF_NOT_EXISTS` (e.g. //! `header: cf-copy-destination-if-none-match: *` on R2) additionally makes the -//! large-artifact multipart claim atomic — without it, that one step falls back +//! large-artifact multipart claim atomic; without it, that one step falls back //! to a probe-then-copy on providers with no conditional copy. //! - anything without a `://` scheme → [`LocalArtifactStore`] on that directory. //! //! Other object-store schemes (`gs://`, `az://`) are rejected until there is a -//! deployment that needs them — adding one is a new match arm plus a Cargo +//! deployment that needs them. Adding one is a new match arm plus a Cargo //! feature, never a change to the transfer code. use crate::artifact_store::ArtifactStore; diff --git a/crates/lodedb-cloud-core/src/sync_plan.rs b/crates/lodedb-cloud-core/src/sync_plan.rs index a029bf72..7237eadb 100644 --- a/crates/lodedb-cloud-core/src/sync_plan.rs +++ b/crates/lodedb-cloud-core/src/sync_plan.rs @@ -1,17 +1,17 @@ -//! Pure three-pointer sync classification — no I/O. +//! Pure three-pointer sync classification, no I/O. //! //! Sync compares three snapshots of one index: the **local** committed //! generation (as the caller's transfer policy would publish it), the **base** //! recorded by the sidecar (the last state both ends agreed on), and the //! **remote** committed generation. [`classify`] reduces those to one -//! [`SyncClassification`] that says which transfer — if any — is a +//! [`SyncClassification`] that says which transfer, if any, is a //! fast-forward, and which situations require an explicit force flag. //! //! Comparison runs on [`logical_id`](crate::snapshot_identity::logical_id) //! (redaction-invariant content identity), never on generation numbers alone: //! two independent lineages can share a generation number. Generations serve //! only as a monotonicity sanity check, because single-base ancestry is -//! trust-based — the sidecar is a *claim* the local machine makes about +//! trust-based. The sidecar is a *claim* the local machine makes about //! history, so a generation that moved backwards relative to the recorded base //! is treated as rollback/tamper-suspect ([`Unknown`]) rather than //! fast-forwarded. @@ -27,7 +27,7 @@ /// absent) are what let the classifier compare two snapshots of the *same* /// commit: the two stores are independent redaction choices /// ([`TransferPolicy`]), so store-set inclusion decides between no-op and -/// republish — and because `logical_id` is computed with both stores nulled, +/// republish. And because `logical_id` is computed with both stores nulled, /// a store present on *both* ends must also match by identity, or the two /// "same" snapshots disagree about payload content (different encodings, or /// tampering) and no untransfer-free reconciliation exists. @@ -47,7 +47,7 @@ pub struct SnapRef { } impl SnapRef { - /// Whether a store present on both snapshots differs by identity — the + /// Whether a store present on both snapshots differs by identity, the /// "same commit, conflicting payload" case that must never be reconciled /// silently. fn shared_store_conflict(&self, other: &Self) -> bool { @@ -75,7 +75,7 @@ impl SnapRef { /// `base` loses nothing: the same commit, no payload store `base` lacks, /// and no shared store whose content disagrees with `base`'s. /// - /// This — not bare logical equality — is what makes a side "unchanged" + /// This, not bare logical equality, is what makes a side "unchanged" /// for fast-forward purposes: a side that enriched the base with payload /// (an unpublished `--include-text` upgrade) has moved, even though its /// logical id has not. @@ -148,7 +148,7 @@ pub fn classify( } // Generation regression against the recorded base on any present side is - // rollback/tamper-suspect: never proceed silently over it — not for ends + // rollback/tamper-suspect: never proceed silently over it, not for ends // that agree with each other (a coordinated rollback must be loud, and a // republish would publish payload onto a suspect remote), and not toward // an absent side either (auto-pushing a rolled-back generation to a fresh @@ -164,7 +164,7 @@ pub fn classify( let (local, remote) = match (local, remote) { // One side absent (and no regression above): the transfer direction - // is unambiguous — there is nothing on the other end to discard. + // is unambiguous, since there is nothing on the other end to discard. (Some(_), None) => return SyncClassification::LocalAhead, (None, Some(_)) => return SyncClassification::RemoteAhead, (Some(local), Some(remote)) => (local, remote), @@ -174,7 +174,7 @@ pub fn classify( if local.logical_id == remote.logical_id { // Same engine commit; the ends should only differ by which payload // stores they carry (the logical id nulls exactly those stores). - // First, a store present on BOTH ends must match by identity — a + // First, a store present on BOTH ends must match by identity; a // mismatch means the two "same" snapshots disagree about payload // content, which no direction of transfer reconciles losslessly. // Then compare the store sets by inclusion: @@ -182,7 +182,7 @@ pub fn classify( // republishes the pointer, upgrading the remote without discarding // anything; // - local carries no more than the remote: nothing to publish, no-op; - // - incomparable (each carries a store the other lacks — text-only vs + // - incomparable (each carries a store the other lacks, text-only vs // lexical-only): publishing either way drops a store the other end // has, so force is required. return if local.snapshot_id == remote.snapshot_id { @@ -205,7 +205,7 @@ pub fn classify( }; // A fast-forward discards the "unchanged" side, so that side must add - // nothing over the base — same commit AND no payload the base lacks. A + // nothing over the base: same commit AND no payload the base lacks. A // side that merely enriched the base with payload stores (an unpublished // `--include-text` upgrade of the last-synced commit) has moved too: // discarding it would silently drop that payload, so both-moved is diff --git a/crates/lodedb-cloud-core/src/sync_state.rs b/crates/lodedb-cloud-core/src/sync_state.rs index 46dfa2fb..8feabd9d 100644 --- a/crates/lodedb-cloud-core/src/sync_state.rs +++ b/crates/lodedb-cloud-core/src/sync_state.rs @@ -1,8 +1,8 @@ //! The sync sidecar: `/.orecloud`, the recorded base. //! //! Three-pointer sync needs a durable record of the last state local and -//! remote agreed on. That record lives *next to* the index it describes — a -//! small JSON sidecar beside `.commit.json` — because it is a claim about +//! remote agreed on. That record lives *next to* the index it describes (a +//! small JSON sidecar beside `.commit.json`) because it is a claim about //! this particular local copy's history, not about the remote (two clones of //! one remote each carry their own base). //! @@ -20,9 +20,10 @@ //! The sidecar is written atomically (temp file + rename, the same discipline //! as [`LocalArtifactStore`](crate::LocalArtifactStore)'s pointer writes) and //! carries a self-checksum over its payload. A torn, tampered, or -//! half-migrated sidecar therefore reads as *absent-but-corrupt* — which the -//! classifier maps to [`Unknown`](crate::sync_plan::SyncClassification::Unknown), -//! requiring an explicit force — rather than as a trusted base. +//! half-migrated sidecar therefore reads as *absent-but-corrupt*, never as a +//! trusted base. The classifier maps that to +//! [`Unknown`](crate::sync_plan::SyncClassification::Unknown), which requires +//! an explicit force. use crate::digest::sha256_hex; use crate::error::{ArtifactStoreError, Result}; @@ -33,7 +34,7 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -/// The sidecar file suffix (`.orecloud`). Must never end in `.json` — +/// The sidecar file suffix (`.orecloud`). Must never end in `.json`; /// see the module docs on the engine's legacy-snapshot glob. pub const SYNC_STATE_SUFFIX: &str = ".orecloud"; @@ -56,7 +57,7 @@ pub struct SyncState { /// flag distinguishing "no sidecar" from "a sidecar was present but corrupt". /// /// Both cases classify the same way (no trusted base -> `Unknown` when the -/// ends differ), but a frontend should *warn* on corruption — it usually means +/// ends differ), but a frontend should *warn* on corruption. It usually means /// a torn write or manual tampering, not a fresh directory. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SidecarRead { @@ -73,7 +74,7 @@ pub fn sync_state_path(dir: &Path, key: &str) -> PathBuf { /// /// An absent file is `{state: None, corrupt: false}`. Any parse failure, /// schema mismatch, checksum mismatch, or an `index_key` that disagrees with -/// `key` is `{state: None, corrupt: true}` — the base is untrusted, never +/// `key` is `{state: None, corrupt: true}`; the base is untrusted, never /// partially honored. Only genuine I/O failures (permissions etc.) are errors. pub fn read_sync_state(dir: &Path, key: &str) -> Result { let path = resolve_within(dir, &sync_state_path(dir, key))?; @@ -102,7 +103,7 @@ pub fn read_sync_state(dir: &Path, key: &str) -> Result { /// Writes the sidecar for `state.index_key` under `dir`, atomically. /// /// Temp file + rename in the same directory, so a crash mid-write leaves -/// either the previous sidecar or a stray `.tmp` — never a torn document (and +/// either the previous sidecar or a stray `.tmp`, never a torn document (and /// a torn document would fail its self-checksum anyway). pub fn write_sync_state(dir: &Path, state: &SyncState) -> Result<()> { let path = resolve_within(dir, &sync_state_path(dir, &state.index_key))?; @@ -124,7 +125,7 @@ pub fn write_sync_state(dir: &Path, state: &SyncState) -> Result<()> { Ok(()) } -/// The checksummed payload — everything except the `sha256` field itself. +/// The checksummed payload: everything except the `sha256` field itself. /// /// Built with one fixed construction order shared by the write path and the /// read-side verification, so the serialized bytes (and therefore the digest) diff --git a/crates/lodedb-cloud-core/src/transfer_policy.rs b/crates/lodedb-cloud-core/src/transfer_policy.rs index 8fe25c14..90d845be 100644 --- a/crates/lodedb-cloud-core/src/transfer_policy.rs +++ b/crates/lodedb-cloud-core/src/transfer_policy.rs @@ -1,19 +1,19 @@ //! Which payload-bearing stores a transfer is allowed to ship. //! -//! The redacted stores (`json`/`tvim`/`tvmv`/`tvann`/`tvvf`) carry no raw text and always ship — -//! they are the metadata and the vector/late-interaction index a restored copy +//! The redacted stores (`json`/`tvim`/`tvmv`/`tvann`/`tvvf`) carry no raw text and always ship. +//! They are the metadata and the vector/late-interaction index a restored copy //! needs to answer searches. Two stores are payload-bearing and opt-in: //! -//! - `tvtext` — the raw document text (`db.get(id)` content); -//! - `tvlex` — lexical terms, which are tokenised text and so payload-derived. +//! - `tvtext`: the raw document text (`db.get(id)` content); +//! - `tvlex`: lexical terms, which are tokenised text and so payload-derived. //! //! A [`TransferPolicy`] gates those two. `tvmv` (late-interaction patch matrices) -//! is embedding data, not text, so it ships by default like `tvim` — as does +//! is embedding data, not text, so it ships by default like `tvim`, as does //! `tvvf`, the rescore original-vector sidecar (vectors, never text). //! //! Redaction rewrites the *committed body* rather than merely skipping bytes: a //! redacted push publishes a body whose excluded sub-manifests are null, so the -//! remote generation genuinely has no text — a restore of it cannot resurrect +//! remote generation genuinely has no text and a restore of it cannot resurrect //! text that was never uploaded. use serde_json::Value; @@ -33,8 +33,8 @@ pub struct TransferPolicy { } impl TransferPolicy { - /// Ships every store, including text and lexical — a verbatim copy of the - /// committed generation. + /// Ships every store, including text and lexical, for a verbatim copy of + /// the committed generation. pub fn full() -> Self { Self { include_text: true, @@ -42,7 +42,7 @@ impl TransferPolicy { } } - /// Ships only the redacted stores (no text, no lexical) — the default posture. + /// Ships only the redacted stores (no text, no lexical), the default posture. pub fn redacted() -> Self { Self { include_text: false, diff --git a/crates/lodedb-cloud-core/src/verify.rs b/crates/lodedb-cloud-core/src/verify.rs index 03032b8c..a2386398 100644 --- a/crates/lodedb-cloud-core/src/verify.rs +++ b/crates/lodedb-cloud-core/src/verify.rs @@ -38,7 +38,7 @@ pub struct VerifyReport { /// /// Returns [`ArtifactStoreError::NotFound`] when the store holds no committed /// generation for `index_key`, and [`ArtifactStoreError::Integrity`] on the first -/// artifact whose bytes do not match — failing closed rather than reporting a +/// artifact whose bytes do not match, failing closed rather than reporting a /// partial success. Reading the pointer validates the body checksum as a /// side effect (the engine's `read_commit_manifest` fails closed on a garbled /// root), so this checks the whole chain: root body, then every referenced blob. @@ -87,7 +87,7 @@ pub struct OpenReport { /// This reuses `lodedb-core`'s `load_store` with `read_only` (which takes no writer /// lock and reads the exact committed manifest, never the `.wal` tail), so it /// proves a restored directory is loadable exactly as the embedded engine would -/// read it — the acceptance check after a restore. Returns +/// read it. This is the acceptance check after a restore. Returns /// the loaded document/chunk counts. `persistence_dir` is the local directory the /// generation was restored into. pub fn verify_local_generation_opens( @@ -118,8 +118,8 @@ pub fn verify_local_generation_opens( /// into a scratch directory alongside reconstructed journal manifests and the /// candidate pointer, and the scratch copy is verify-opened read-only. A /// checksum-consistent but semantically invalid artifact therefore fails the -/// restore while the destination still points at its previous generation — -/// without this, the pointer swap would publish the broken generation before +/// restore while the destination still points at its previous generation. +/// Without this, the pointer swap would publish the broken generation before /// the open check could reject it. pub(crate) fn verify_candidate_opens( dir: &Path, @@ -142,7 +142,7 @@ pub(crate) fn verify_candidate_opens( } if std::fs::hard_link(&source, &target).is_err() { // Filesystems without hardlinks (or cross-device edge cases): - // fall back to a byte copy — correctness over speed. + // fall back to a byte copy, correctness over speed. std::fs::copy(&source, &target)?; } } diff --git a/crates/lodedb-cloud-core/tests/artifact_store.rs b/crates/lodedb-cloud-core/tests/artifact_store.rs index 6e098b13..83496062 100644 --- a/crates/lodedb-cloud-core/tests/artifact_store.rs +++ b/crates/lodedb-cloud-core/tests/artifact_store.rs @@ -53,7 +53,7 @@ fn rewrite_different_bytes_conflicts() { .write_bytes_if_absent("idx.gen/g0.json", b"second", &sha_hex(b"second")) .unwrap_err(); assert!(matches!(err, ArtifactStoreError::Integrity(_))); - // The original bytes are preserved — never overwritten in place. + // The original bytes are preserved, never overwritten in place. assert_eq!( store.read_bytes("idx.gen/g0.json").unwrap(), b"first".to_vec() @@ -119,7 +119,7 @@ fn compare_and_swap_enforces_body_precondition() { #[test] fn compare_and_swap_rejects_same_generation_different_body() { // The ABA guard: a body sharing the committed generation *number* but carrying - // different content must NOT satisfy the precondition — a number is not a + // different content must NOT satisfy the precondition; a number is not a // version token. A generation-only check would wrongly let this swap through. let dir = tempfile::tempdir().unwrap(); let store = LocalArtifactStore::new(dir.path(), false); @@ -154,7 +154,7 @@ fn compare_and_swap_rejects_same_generation_different_body() { #[test] fn read_pointer_on_nonexistent_root_is_none() { // A store bound to a not-yet-created directory reads as empty rather than - // erroring — the precondition for exporting into a fresh backup target. + // erroring, the precondition for exporting into a fresh backup target. let parent = tempfile::tempdir().unwrap(); let root = parent.path().join("not-created-yet"); let store = LocalArtifactStore::new(&root, false); @@ -187,7 +187,7 @@ fn racing_local_writers_never_corrupt_a_published_artifact() { // Two writers streaming DIFFERENT bytes under one name: unique scratches // plus a no-replace publish mean exactly one lineage's bytes land, every // Ok return attests bytes that match one writer's digest in full, and a - // loser sees the immutability refusal — never a torn interleaving. + // loser sees the immutability refusal, never a torn interleaving. let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_path_buf(); let payload_a: Vec = std::iter::repeat_with(|| b'a').take(4 * 1024 * 1024).collect(); diff --git a/crates/lodedb-cloud-core/tests/client_ops.rs b/crates/lodedb-cloud-core/tests/client_ops.rs index ae7cca26..feb8fc78 100644 --- a/crates/lodedb-cloud-core/tests/client_ops.rs +++ b/crates/lodedb-cloud-core/tests/client_ops.rs @@ -1,6 +1,6 @@ //! Tests for the string-target client operations (`client_ops`): the facade the //! CLI/binding calls. The typed primitives are covered by their own suites; -//! these tests pin the composition — target resolution, policy pass-through, +//! these tests pin the composition: target resolution, policy pass-through, //! and pull's built-in open-verification. mod common; @@ -60,7 +60,7 @@ fn push_status_pull_round_trip_by_target_strings() { verify(remote_s, KEY).unwrap(); - // Pull restores AND proves the copy opens — one operation. + // Pull restores AND proves the copy opens, as one operation. let outcome = pull(remote_s, restored_s, KEY).unwrap(); assert!(outcome.transfer.pointer_published); assert_eq!(outcome.open.index_key, KEY); diff --git a/crates/lodedb-cloud-core/tests/common/mod.rs b/crates/lodedb-cloud-core/tests/common/mod.rs index db1b95cb..40aeb93d 100644 --- a/crates/lodedb-cloud-core/tests/common/mod.rs +++ b/crates/lodedb-cloud-core/tests/common/mod.rs @@ -23,7 +23,7 @@ pub fn sha_hex(data: &[u8]) -> String { } /// Writes a store's base file (and any delta segments) under `/.gen/`, -/// returning the `{base, deltas}` sub-manifest that names them — the exact shape +/// returning the `{base, deltas}` sub-manifest that names them, the exact shape /// the engine records and the inventory reads. pub fn store_sub( dir: &Path, @@ -115,7 +115,7 @@ pub fn write_json_commit( ) } -/// Builds a commit body without touching the filesystem — for CAS tests that only +/// Builds a commit body without touching the filesystem, for CAS tests that only /// need a well-formed pointer payload. pub fn commit_body(key: &str, generation: u64, base_epoch: u64, json_manifest: Value) -> Value { build_commit_body(CommitBodyInput { @@ -193,7 +193,7 @@ pub fn commit_engine_generation( } /// [`commit_engine_generation`], optionally also writing a lexical store -/// (`tvlex`) — for tests that need both payload-bearing stores. +/// (`tvlex`), for tests that need both payload-bearing stores. pub fn commit_engine_generation_with_lexical( dir: &Path, key: &str, diff --git a/crates/lodedb-cloud-core/tests/generation_inventory.rs b/crates/lodedb-cloud-core/tests/generation_inventory.rs index 0ef133e8..0d14cd14 100644 --- a/crates/lodedb-cloud-core/tests/generation_inventory.rs +++ b/crates/lodedb-cloud-core/tests/generation_inventory.rs @@ -95,7 +95,7 @@ fn inventory_covers_multivector_tvmv_store() { #[test] fn inventory_covers_the_ann_tvann_store() { // `tvann` (the persisted ANN cluster partition) is base-only and, to the - // engine, a rebuildable cache — but a body referencing a tvann base the + // engine, a rebuildable cache. But a body referencing a tvann base the // transfer never shipped would fail byte-verification on pull, so the // inventory must cover it like any other store. let body = json!({ @@ -121,11 +121,11 @@ fn inventory_covers_the_ann_tvann_store() { fn inventory_covers_the_rescore_tvvf_store() { // `tvvf` (the rescore original-vector sidecar, engine 1.3.2+) is a journaled // {base, deltas} store: the engine refuses to open a rescore store without - // its sidecar, so a push must ship the base AND any delta segments — a + // its sidecar, so a push must ship the base AND any delta segments; a // pulled copy missing either would be unopenable, not merely degraded. // Its base is NOT `g.tvvf`: the sidecar keeps its own epoch // counter (`vf_epoch`, here deliberately different from base_epoch) and - // lives at `vf.tvvf` (`tvvf_store::base_path`) — this body + // lives at `vf.tvvf` (`tvvf_store::base_path`). This body // mirrors exactly what `write_base_manifest` records. let body = json!({ "index_key": "idx", @@ -183,7 +183,7 @@ fn inventory_rejects_a_tvvf_manifest_without_vf_epoch() { fn inventory_rejects_a_malformed_artifact_digest() { // Digests become staging file names and object keys downstream, so a // "digest" carrying a path (or anything that is not 64 lowercase hex - // chars) must fail closed at the inventory — before it can name a path. + // chars) must fail closed at the inventory, before it can name a path. for bad in ["", "abc", "../../../etc/passwd", "/tmp/evil"] { let sub = json!({ "base": { "file_name": "g0.json", "sha256": bad, "file_bytes": 0 }, @@ -199,7 +199,7 @@ fn inventory_rejects_a_malformed_artifact_digest() { #[test] fn inventory_rejects_an_unknown_store_sub_manifest() { // A future engine store this build does not know must refuse the transfer, - // not silently drop its artifacts — an understated inventory ships a + // not silently drop its artifacts. An understated inventory ships a // generation whose referenced blobs were never uploaded. let body = json!({ "index_key": "idx", @@ -338,7 +338,7 @@ fn inventory_rejects_a_non_object_delta_entry() { #[test] fn inventory_includes_non_null_tvim_regardless_of_present_flag() { // The engine loads a non-null tvim manifest whatever `tvim_present` says, so the - // inventory must too — otherwise a restore drops the tvim base it will open. + // inventory must too; otherwise a restore drops the tvim base it will open. let body = json!({ "index_key": "idx", "generation": 1, @@ -402,7 +402,7 @@ fn inventory_rejects_a_non_object_store_manifest() { #[test] fn inventory_rejects_a_body_key_mismatch() { // The pointer file-name key ("idx") disagrees with the body's own index_key - // ("other") — only possible via tampering, since the body checksum is valid. + // ("other"), which is only possible via tampering, since the body checksum is valid. let sub = json!({ "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, "deltas": [], diff --git a/crates/lodedb-cloud-core/tests/managed.rs b/crates/lodedb-cloud-core/tests/managed.rs index 30988e2a..5c1c1d76 100644 --- a/crates/lodedb-cloud-core/tests/managed.rs +++ b/crates/lodedb-cloud-core/tests/managed.rs @@ -20,7 +20,7 @@ fn dir_str(path: &Path) -> &str { } /// Copies every artifact a committed generation pins into a staging directory -/// under its content digest — what the Python edge does with downloaded blobs. +/// under its content digest, what the Python edge does with downloaded blobs. fn stage_generation(source: &Path, body: &serde_json::Value, staging: &Path) { let inventory = lodedb_cloud_core::inventory_from_body(KEY, Some(body)) .unwrap() @@ -49,7 +49,7 @@ fn plan_for_a_fresh_local_generation_is_local_ahead_with_full_inventory() { assert_eq!(plan.report.classification.as_deref(), Some("local_ahead")); let local_part = plan.local.expect("local generation is committed"); // The redacted policy nulls tvtext/tvlex, so the plan's identity is the - // *redacted* body's — and for a text-free commit that equals the raw one. + // *redacted* body's, and for a text-free commit that equals the raw one. assert_eq!(local_part.side.snapshot_id, snapshot_id(&body).unwrap()); assert!(!local_part.side.has_text); assert!(!local_part.artifacts.is_empty()); diff --git a/crates/lodedb-cloud-core/tests/object_artifact_store.rs b/crates/lodedb-cloud-core/tests/object_artifact_store.rs index 4f8a4d50..72016ff1 100644 --- a/crates/lodedb-cloud-core/tests/object_artifact_store.rs +++ b/crates/lodedb-cloud-core/tests/object_artifact_store.rs @@ -107,7 +107,7 @@ fn pointer_cas_creates_updates_and_conflicts() { ); // A stale expected body (a would-be concurrent writer still holding gen 1) - // conflicts — the pointer is now gen 2. + // conflicts; the pointer is now gen 2. let gen3 = commit_body("idx", 3, 0, json_sub("g0.json")); let err = store .compare_and_swap_pointer("idx", Some(&gen1), &gen3) @@ -143,7 +143,7 @@ fn tenants_are_isolated_by_prefix() { .unwrap(); // Tenant B, knowing the exact name and checksum, still cannot reach A's blob - // or pointer — the prefix namespaces content addressing per tenant. + // or pointer; the prefix namespaces content addressing per tenant. assert!(matches!( tenant_b.read_bytes("idx.gen/g0.json").unwrap_err(), ArtifactStoreError::NotFound(_) diff --git a/crates/lodedb-cloud-core/tests/restore.rs b/crates/lodedb-cloud-core/tests/restore.rs index a6a7b7aa..7e36c2b0 100644 --- a/crates/lodedb-cloud-core/tests/restore.rs +++ b/crates/lodedb-cloud-core/tests/restore.rs @@ -57,7 +57,7 @@ fn restore_copies_a_generation_that_opens_read_only() { #[test] fn pull_rebuilds_the_delta_journal_manifests() { // The engine's per-store journal manifest is working state the body never - // pins, so the transfer doesn't ship it — but the O(changed) mutation + // pins, so the transfer doesn't ship it. But the O(changed) mutation // path requires it, so a restored copy must be WRITABLE, not just // readable. `pull` reconstructs each journal manifest verbatim from the // body's sub-manifest. @@ -173,7 +173,7 @@ fn pull_refuses_over_a_destination_wal_with_pending_records() { // The destination WAL's records were acknowledged against the OLD lineage; // replaying them onto a pulled snapshot (or silently truncating them) // corrupts or loses acked writes. Pull must refuse until the caller - // checkpoints — force-pull (a sync flag) is the explicit discard. + // checkpoints; force-pull (a sync flag) is the explicit discard. const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; let remote = tempfile::tempdir().unwrap(); commit_engine_generation(remote.path(), KEY, 2, 1, "remote-side", None); diff --git a/crates/lodedb-cloud-core/tests/snapshot_identity.rs b/crates/lodedb-cloud-core/tests/snapshot_identity.rs index cf3cf337..f0c9ded5 100644 --- a/crates/lodedb-cloud-core/tests/snapshot_identity.rs +++ b/crates/lodedb-cloud-core/tests/snapshot_identity.rs @@ -8,7 +8,7 @@ use common::commit_engine_generation; use lodedb_cloud_core::{logical_id, snapshot_id, TransferPolicy}; use serde_json::Value; -/// The id IS the `body_sha256` the engine writes in `.commit.json` — not +/// The id IS the `body_sha256` the engine writes in `.commit.json`, not /// a lookalike digest. Pinned against a real engine-committed store by reading /// the pointer document raw. #[test] @@ -47,7 +47,7 @@ fn redaction_shares_the_logical_id_but_not_the_snapshot_id() { snapshot_id(&redacted).unwrap(), "different bytes must have different snapshot ids" ); - // A fully redacted body is its own logical form — the property the sync + // A fully redacted body is its own logical form, the property the sync // classifier's Republish detection rests on. assert_eq!( snapshot_id(&redacted).unwrap(), diff --git a/crates/lodedb-cloud-core/tests/sync_ops.rs b/crates/lodedb-cloud-core/tests/sync_ops.rs index 3e73a460..aa0b9f57 100644 --- a/crates/lodedb-cloud-core/tests/sync_ops.rs +++ b/crates/lodedb-cloud-core/tests/sync_ops.rs @@ -1,5 +1,5 @@ //! End-to-end tests for the `sync` verb: two working directories sharing one -//! remote, exercising every classification the way real use produces them — +//! remote, exercising every classification the way real use produces them, //! plus `contains` on both store backends. mod common; @@ -173,7 +173,7 @@ fn an_untrusted_base_requires_force() { let remote = tempfile::tempdir().unwrap(); // Two ends that never synced (no sidecar) holding different content. - // (Different base epochs — see the fork-collision note above.) + // (Different base epochs; see the fork-collision note above.) commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); run_sync(dir1.path(), remote.path(), SyncForce::None); commit_engine_generation(dir2.path(), KEY, 1, 2, "different", None); @@ -218,7 +218,7 @@ fn a_base_recorded_against_another_remote_is_not_trusted() { commit_engine_generation(other_dir.path(), KEY, 2, 2, "unrelated", None); run_sync(other_dir.path(), remote_b.path(), SyncForce::None); - // Against remote B, dir's local copy still equals remote A's base — but + // Against remote B, dir's local copy still equals remote A's base, but // that base must not be trusted here: no fast-forward pull of B's content. let err = sync( dir.path().to_str().unwrap(), @@ -324,7 +324,7 @@ fn a_partial_redaction_upgrade_republishes() { /// The Phase-0 fork-collision limit, pinned: two lineages diverging from one /// base reuse the same artifact file names with different bytes, and force -/// cannot resolve that against a name-addressed remote — sync fails closed +/// cannot resolve that against a name-addressed remote. Sync fails closed /// with a recovery hint instead of overwriting an immutable artifact. (The /// managed content-addressed layout of a later milestone absorbs this shape.) #[test] @@ -333,7 +333,7 @@ fn a_same_name_fork_fails_closed_with_a_recovery_hint_even_when_forced() { let dir2 = tempfile::tempdir().unwrap(); let remote = tempfile::tempdir().unwrap(); - // Both clones share base gen 1, then each commits gen 2 at epoch 2 — the + // Both clones share base gen 1, then each commits gen 2 at epoch 2, the // natural fork shape: same artifact names, different bytes. commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); run_sync(dir1.path(), remote.path(), SyncForce::None); @@ -381,7 +381,7 @@ fn a_same_name_fork_fails_closed_with_a_recovery_hint_even_when_forced() { /// An unpublished payload upgrade is not discarded by a fast-forward: if the /// base was recorded redacted, the local copy now syncs with text, and the -/// remote has meanwhile advanced, both sides have moved — force required. +/// remote has meanwhile advanced, both sides have moved and force is required. #[test] fn an_unpublished_payload_upgrade_blocks_a_fast_forward_pull() { let dir1 = tempfile::tempdir().unwrap(); @@ -469,7 +469,7 @@ fn an_in_sync_no_op_repairs_a_stale_base() { .state .unwrap(); - // Advance both ends together, then restore the OLD sidecar — the state a + // Advance both ends together, then restore the OLD sidecar, the state a // crashed post-transfer sidecar write leaves behind. commit_engine_generation(dir.path(), KEY, 2, 2, "v2", None); run_sync(dir.path(), remote.path(), SyncForce::None); @@ -522,7 +522,7 @@ fn a_corrupt_sidecar_is_surfaced_and_treated_as_no_base() { run_sync(dir.path(), remote.path(), SyncForce::None); // Corrupt the sidecar; the two ends are still identical, so sync is a - // harmless no-op — but the corruption is reported, by status too. + // harmless no-op, but the corruption is reported, by status too. let sidecar = lodedb_cloud_core::sync_state::sync_state_path(dir.path(), KEY); std::fs::write(&sidecar, b"{ not json").unwrap(); let report = status( @@ -550,7 +550,7 @@ fn a_corrupt_sidecar_is_surfaced_and_treated_as_no_base() { assert!(!report.sidecar_corrupt); // When the corruption actually causes a refusal (ends differ, base - // untrusted), the error itself carries the diagnosis — the CLI's warning + // untrusted), the error itself carries the diagnosis; the CLI's warning // path never runs on an error. commit_engine_generation(dir.path(), KEY, 2, 2, "v2", None); run_sync(dir.path(), remote.path(), SyncForce::None); // remote at v2 @@ -613,8 +613,8 @@ fn contains_on_both_backends() { fn sync_pull_refuses_a_pending_wal_and_force_pull_discards_it() { // dir2 is behind the remote AND holds acknowledged-but-uncheckpointed WAL // records: a plain sync must refuse (those records were acked against the - // old lineage), and --force-pull — the explicit "keep the remote copy" - // decision — discards them along with the local lineage. + // old lineage), and --force-pull (the explicit "keep the remote copy" + // decision) discards them along with the local lineage. let dir1 = tempfile::tempdir().unwrap(); let dir2 = tempfile::tempdir().unwrap(); let remote = tempfile::tempdir().unwrap(); diff --git a/crates/lodedb-cloud-core/tests/sync_plan.rs b/crates/lodedb-cloud-core/tests/sync_plan.rs index 1c51c7ba..28a58ba4 100644 --- a/crates/lodedb-cloud-core/tests/sync_plan.rs +++ b/crates/lodedb-cloud-core/tests/sync_plan.rs @@ -28,7 +28,7 @@ fn snap(tag: &str, generation: u64, has_text: bool, has_lexical: bool) -> SnapRe } /// A snapshot of commit `tag` whose *text store identity* disagrees with -/// [`snap`]'s — same commit, same store set, different payload bytes (a +/// [`snap`]'s: same commit, same store set, different payload bytes (a /// re-encoded or tampered text store). fn conflicting_text(tag: &str, generation: u64, has_lexical: bool) -> SnapRef { let mut reference = snap(tag, generation, true, has_lexical); @@ -88,7 +88,7 @@ fn the_decision_table() { (Some(&b), Some(&a), Some(&c), Diverged), // No base while the ends differ: no way to pick a direction. (Some(&a), None, Some(&b), Unknown), - // Generation regression against the base: rollback/tamper suspect — + // Generation regression against the base: rollback/tamper suspect, // even when the two ends agree with each other (a coordinated // rollback must be loud, not a silent no-op or republish). (Some(&rollback), Some(&a), Some(&a), Unknown), @@ -116,7 +116,7 @@ fn the_decision_table() { (Some(&a_red), Some(&a), Some(&a), InSync), (Some(&a_text), Some(&a), Some(&a), InSync), // Incomparable store sets of one commit: publishing either way drops - // a store the other has — force required. + // a store the other has, so force is required. (Some(&a_text), Some(&a), Some(&a_lex), Unknown), (Some(&a_lex), Some(&a), Some(&a_text), Unknown), // Same commit, same store set, but a shared store's payload identity @@ -148,7 +148,7 @@ fn the_decision_table() { /// Every combination of local/base/remote drawn from a small domain (absent, /// two full commits at different generations, the redacted form of the first, -/// and a low-generation commit) — the classifier must uphold its global +/// and a low-generation commit). The classifier must uphold its global /// invariants everywhere, not just on the table above. #[test] fn invariants_hold_over_the_exhaustive_domain() { diff --git a/crates/lodedb-cloud-core/tests/verify_status.rs b/crates/lodedb-cloud-core/tests/verify_status.rs index f49c3ea9..fe13b6d6 100644 --- a/crates/lodedb-cloud-core/tests/verify_status.rs +++ b/crates/lodedb-cloud-core/tests/verify_status.rs @@ -125,7 +125,7 @@ fn status_for_push_reflects_the_redacted_push_it_describes() { let source = LocalArtifactStore::new(src.path(), false); let dest = LocalArtifactStore::new(dst.path(), false); - // Redacted status counts only the json base — the text artifact would not + // Redacted status counts only the json base. The text artifact would not // ship, so it must not be reported as pending upload. let redacted = status_for_push(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); assert_eq!(redacted.artifacts_to_upload, 1); diff --git a/crates/lodedb-core/src/engine/mod.rs b/crates/lodedb-core/src/engine/mod.rs index ae88aad5..dc58d4e4 100644 --- a/crates/lodedb-core/src/engine/mod.rs +++ b/crates/lodedb-core/src/engine/mod.rs @@ -1124,9 +1124,9 @@ impl CoreEngine { /// Whether the ANN cluster index is currently resident in memory for an index /// (adopted from a persisted `.tvann` sidecar on a writable/eager open, or - /// built by a prior query). Returns `false` when it is not resident — a later + /// built by a prior query). Returns `false` when it is not resident (a later /// ANN query builds it, adopting a lazy open's deferred sidecar assignment - /// first — or when the index is exact. Observability; does not trigger a build. + /// first) or when the index is exact. Observability; does not trigger a build. pub fn ann_cluster_resident(&self, index_id: &str) -> Result { let index = self.index(index_id)?; Ok(index.cluster_index.borrow().is_some()) @@ -1829,7 +1829,7 @@ impl CoreEngine { // Same strictness as upsert_vectors, per element too: silently // dropping a non-string id would count the record as applied // while deleting nothing. An empty (but well-formed) list stays - // a valid no-op — refusing it could wedge replay of a legacy + // a valid no-op; refusing it could wedge replay of a legacy // local WAL tail. let document_ids = record .payload @@ -2373,8 +2373,8 @@ impl CoreEngine { // ("direct TurboVec snapshot is required but missing"). // // Fail closed and leave the WAL untouched on disk so the - // Python writer — which checkpoints its own WAL on open and - // writes a complete snapshot — stays the owner of these + // Python writer, which checkpoints its own WAL on open and + // writes a complete snapshot, stays the owner of these // stores. In the Python SDK, Python opens (and checkpoints) // before the native engine, so the native engine never // reaches this branch; a native init failure here also @@ -2712,7 +2712,7 @@ struct PersistentLock { /// RAII exclusive hold on a store directory's single-writer lock /// (`/.lodedb.lock`), for out-of-band tools that mutate a database -/// directory without opening an engine — the cloud transfer plane's +/// directory without opening an engine, such as the cloud transfer plane's /// pull/restore path. Contends with every engine writer (native, Python, /// Swift) across processes; dropping the guard releases the hold. pub struct DirWriterLock { @@ -2756,7 +2756,7 @@ impl LockMode { } impl PersistentLock { - /// Takes the single-writer lock on ``/.lodedb.lock`` — the same sentinel + /// Takes the single-writer lock on ``/.lodedb.lock``, the same sentinel /// the Python writer uses, so a native/FFI/Swift writer contends with a Python /// writer (and another native writer) across processes. Unix takes a BSD /// advisory lock (``flock(LOCK_EX|LOCK_NB)``); Windows opens the sentinel with diff --git a/crates/lodedb-core/src/storage/commit_manifest.rs b/crates/lodedb-core/src/storage/commit_manifest.rs index 9eb1c078..75e94b17 100644 --- a/crates/lodedb-core/src/storage/commit_manifest.rs +++ b/crates/lodedb-core/src/storage/commit_manifest.rs @@ -120,8 +120,8 @@ pub fn read_commit_manifest(path: &Path) -> CoreResult> { /// Parses and validates pointer-document bytes exactly as /// [`read_commit_manifest`] validates the on-disk file (schema version and /// body checksum), without touching the filesystem. For consumers that hold a -/// pointer document from somewhere other than disk — the transfer plane's -/// object-store pointer mirror — so validation has one implementation. +/// pointer document from somewhere other than disk (the transfer plane's +/// object-store pointer mirror), so validation has one implementation. pub fn parse_commit_manifest(bytes: &[u8]) -> CoreResult { let document: Value = serde_json::from_slice(bytes) .map_err(|error| corrupt(format!("commit manifest is not valid JSON: {error}")))?; @@ -207,8 +207,8 @@ pub fn write_commit_manifest(path: &Path, body: &Value, fsync: bool) -> CoreResu } /// Renders the exact pointer-document text a `.commit.json` carrying -/// `body` holds on disk — the canonical body JSON, its checksum, and the -/// schema envelope — without touching the filesystem. [`write_commit_manifest`] +/// `body` holds on disk (the canonical body JSON, its checksum, and the +/// schema envelope) without touching the filesystem. [`write_commit_manifest`] /// persists exactly this text, so the two can never drift. pub fn render_commit_manifest(body: &Value) -> CoreResult { if !body.is_object() { @@ -221,7 +221,7 @@ pub fn render_commit_manifest(body: &Value) -> CoreResult { )) } -/// The engine-canonical `body_sha256` for `body` — the digest a +/// The engine-canonical `body_sha256` for `body`, the digest a /// `.commit.json` pointer carrying exactly this body records. Exported /// for the transfer plane, whose identity and lineage decisions key on it; /// the alternative is a scratch-file round trip through diff --git a/crates/lodedb-core/src/storage/mod.rs b/crates/lodedb-core/src/storage/mod.rs index c98c9626..cf72a260 100644 --- a/crates/lodedb-core/src/storage/mod.rs +++ b/crates/lodedb-core/src/storage/mod.rs @@ -264,7 +264,7 @@ pub fn load_generation_store( /// Cheap store metadata for the append path: the committed generation and the /// index's vector dimension. Read from the commit manifest and the state base -/// payload only — unlike [`load_store`] it does not read or checksum the `.tvim` +/// payload only. Unlike [`load_store`] it does not read or checksum the `.tvim` /// vectors or the text/lexical/multivec/ann sidecars, so a `CoreAppender` open is /// O(state metadata) rather than O(store). #[derive(Debug, Clone, Copy)] diff --git a/crates/lodedb-core/src/storage/util.rs b/crates/lodedb-core/src/storage/util.rs index 7f2653a3..d8818eda 100644 --- a/crates/lodedb-core/src/storage/util.rs +++ b/crates/lodedb-core/src/storage/util.rs @@ -139,8 +139,8 @@ pub(crate) fn verify_file_sha256(path: &Path, expected: &str, context: &str) -> /// `Ok(Some(file))` holds the lock; `Ok(None)` means another holder currently /// blocks it and the caller should retry; `Err` is a real open/lock failure the /// caller wraps in its own message. Unix opens read+write+create and takes a -/// non-blocking advisory `flock` — exclusive, or shared when `exclusive` is false -/// (a shared `flock` interoperates with the Python writer's `fcntl.flock`). +/// non-blocking advisory `flock`, exclusive by default or shared when `exclusive` +/// is false (a shared `flock` interoperates with the Python writer's `fcntl.flock`). /// Windows has no advisory `flock` that the Python `msvcrt` byte lock can see, so /// it opens with an exclusive share mode (`share_mode(0)`); there is no shared /// form, so a shared request degrades to exclusive there. Other platforms open diff --git a/crates/lodedb-core/src/storage/wal.rs b/crates/lodedb-core/src/storage/wal.rs index a2cf9bc9..48f26a11 100644 --- a/crates/lodedb-core/src/storage/wal.rs +++ b/crates/lodedb-core/src/storage/wal.rs @@ -226,7 +226,7 @@ fn scan_records(data: &[u8]) -> CoreResult { /// on-disk WAL byte format byte-for-byte so one decode path serves both; /// callers ship the returned bytes verbatim. Records may carry `lsn: None` /// (segments are stamped at fold time) or `Some` (tests pin byte-compat -/// against `append_record`). Refuses an empty record list — a segment with no +/// against `append_record`). Refuses an empty record list; a segment with no /// records is a writer bug, not a no-op. pub fn encode_wal_segment(records: &[WalRecord]) -> CoreResult> { if records.is_empty() { @@ -397,7 +397,7 @@ fn encode_body(op: &str, mut payload: Value, lsn: Option) -> CoreResult) -> CoreResult> { @@ -910,7 +910,7 @@ mod tests { #[test] fn segment_bytes_match_append_record_for_stamped_records() { // A segment of stamped records must be byte-identical to the file the - // append path writes for the same sequence — one frame format, no drift. + // append path writes for the same sequence: one frame format, no drift. let dir = temp_dir("segment-byte-compat"); let path = dir.join("default.wal"); let records = vec![ diff --git a/crates/lodedb-core/src/vector/turbovec.rs b/crates/lodedb-core/src/vector/turbovec.rs index e17c31bd..019d9c47 100644 --- a/crates/lodedb-core/src/vector/turbovec.rs +++ b/crates/lodedb-core/src/vector/turbovec.rs @@ -348,7 +348,7 @@ impl TurboVecNativeIndex { /// Flat-input, arrays-output batch search for the near-zero-copy boundary. /// /// `queries` is the flat `[nq * dim]` query buffer (no per-query `Vec`), and the - /// result is flat `[nq * k]` `(scores, document_ids, k)` — no per-hit struct or + /// result is flat `[nq * k]` `(scores, document_ids, k)` with no per-hit struct or /// JSON. `k` is 0 when nothing matches. The PyO3 layer turns scores into a /// numpy array and document ids into a string list directly, so a batch query /// crosses the boundary without the per-hit serialize/parse `search_batch` pays. diff --git a/crates/lodedb-core/tests/core_engine.rs b/crates/lodedb-core/tests/core_engine.rs index 5c5ad1fe..91c1b81f 100644 --- a/crates/lodedb-core/tests/core_engine.rs +++ b/crates/lodedb-core/tests/core_engine.rs @@ -1323,7 +1323,7 @@ fn cleared_patch_matrix_does_not_resurrect_on_reload() { engine.persist().unwrap(); } // 2. Re-upsert the same live document without a matrix. The pooled vector is - // unchanged, so this folds as an O(changed) delta (not a base rewrite) — the + // unchanged, so this folds as an O(changed) delta (not a base rewrite), the // path where an omitted clear would let the base matrix survive. { let mut engine = CoreEngine::open(options.clone()).unwrap(); @@ -1461,7 +1461,7 @@ fn text_reupsert_clears_a_prior_patch_matrix_on_reload() { // 2. Re-add the same id through the TEXT path (which carries no matrix). The // chunk embedding equals the base document vector so the calibration // fingerprint is unchanged and this folds as an O(changed) delta (not a base - // rewrite) — the path where an omitted clear leaves the base matrix behind. + // rewrite), the path where an omitted clear leaves the base matrix behind. // This is the third replacement site the clear tracking must cover, alongside // upsert_vectors and delete. { diff --git a/crates/lodedb-ffi/src/lib.rs b/crates/lodedb-ffi/src/lib.rs index c4f45e9c..666cc2b5 100644 --- a/crates/lodedb-ffi/src/lib.rs +++ b/crates/lodedb-ffi/src/lib.rs @@ -210,9 +210,9 @@ pub unsafe extern "C" fn lodedb_engine_free(engine: *mut LodeEngine) { /// Creates a vector index from a minimal JSON create request. /// -/// The JSON object carries only the distinguishing fields — `index_id`, +/// The JSON object carries only the distinguishing fields: `index_id`, /// `vector_dim`, optional `bit_width` (defaults to 4), optional `model` (the -/// reopen-time embedder-guard identity), and optional `ann` tuning — and the core +/// reopen-time embedder-guard identity), and optional `ann` tuning. The core /// supplies the identity defaults (name, provider, task, route/storage profile, /// and `index_key`/`client_id_hash` from `index_id`). This is the single create /// entry point: an exact index simply omits `ann`, so bindings never hand-copy the diff --git a/crates/lodedb-gpu/src/lib.rs b/crates/lodedb-gpu/src/lib.rs index 46609438..9dcbab9f 100644 --- a/crates/lodedb-gpu/src/lib.rs +++ b/crates/lodedb-gpu/src/lib.rs @@ -292,7 +292,7 @@ extern "C" __global__ void merge_topk( /// CPU kernel, never an error the caller should surface to the user. #[derive(Debug, Clone)] pub enum GpuScanError { - /// No CUDA driver is loadable (CPU-only host) — do not retry this process. + /// No CUDA driver is loadable (CPU-only host); do not retry this process. Unavailable(String), /// The driver loaded but a CUDA call failed (no device, OOM, kernel error). Device(String), @@ -326,8 +326,8 @@ fn debug_enabled() -> bool { /// /// On by default: the fused path reads each cuBLAS score once instead of the `k` /// times `topk_argmax` rescans it, and returns the identical top-k (same cuBLAS -/// scores, same (score desc, slot asc) order) — verified by exact-parity tests on -/// A10 and L40S. Set `LODEDB_GPU_FUSED_TOPK=off` (or `0`/`false`/`no`) to force the +/// scores, same (score desc, slot asc) order). Exact-parity tests on +/// A10 and L40S verify this. Set `LODEDB_GPU_FUSED_TOPK=off` (or `0`/`false`/`no`) to force the /// legacy `topk_argmax` path. Mirrors the on-by-default `LODEDB_GPU_DIRECT_TURBOVEC` /// knob: any other value leaves it enabled. fn fused_topk_enabled() -> bool { @@ -787,7 +787,7 @@ impl GpuScanSession { // Reuse device scratch for this (nq, k) shape; allocate only on first sight. // The serving layer drives a small set of shapes, so steady-state queries - // do no device allocation or zeroing — only copies and kernels. + // do no device allocation or zeroing, only copies and kernels. // ceil(n / FUSED_CHUNK_ROWS) without div_ceil (MSRV 1.70); only used when fused. let num_chunks = self.n / FUSED_CHUNK_ROWS + usize::from(self.n % FUSED_CHUNK_ROWS != 0); diff --git a/docs/architecture.md b/docs/architecture.md index 87757a6e..97225c5c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -133,27 +133,27 @@ pulled in transitively by `sentence-transformers`, but importing LodeDB does not Each index is a per-index generation directory plus a single atomically-swapped root pointer: -- `.commit.json` — the **root commit manifest**: the one file whose atomic swap commits +- `.commit.json` is the **root commit manifest**: the one file whose atomic swap commits a generation. It pins (with checksums) the consistent set of artifacts for that generation. -- `.gen/` — generation-addressed artifacts for that index: - - `g.json` — the redacted JSON state base, plus its `.jsd` document journal under +- `.gen/` holds the generation-addressed artifacts for that index: + - `g.json`: the redacted JSON state base, plus its `.jsd` document journal under `g.json.json-delta/`, - - `g.tvim` — the TurboVec vector base (quantized vectors + metadata), plus its `.tvd` + - `g.tvim`: the TurboVec vector base (quantized vectors + metadata), plus its `.tvd` encoded-row journal under `g.tvim.tvim-delta/`, - - `g.tvtext` — the raw-text base (`store_text`, on by default): the full + - `g.tvtext`: the raw-text base (`store_text`, on by default) holding the full `document_id -> text` map, plus its `.txd` text journal under `g.tvtext.tvtext-delta/`, governed by the same root manifest, - - `g.tvlex` — the lexical-index base (`index_text`, which defaults to match `store_text`): - the full `document_id -> per-chunk token lists` map, plus its `.lxd` token journal under - `g.tvlex.tvlex-delta/`, governed by the same root manifest. + - `g.tvlex`: the lexical-index base (`index_text`, which defaults to match `store_text`) + holding the full `document_id -> per-chunk token lists` map, plus its `.lxd` token journal + under `g.tvlex.tvlex-delta/`, governed by the same root manifest. -A commit writes any new artifacts first — bases are generation-addressed and never -overwritten in place — then atomically swaps `.commit.json`; that swap is the only +A commit writes any new artifacts first (bases are generation-addressed and never +overwritten in place), then atomically swaps `.commit.json`; that swap is the only commit point. A crash mid-commit leaves the previously committed generation fully intact: on reopen a writer rolls back to it (dropping the uncommitted artifacts) rather than failing -closed, and a lock-free reader loads exactly the generation the root names — consistent -snapshot isolation, raw text included. Superseded generations are garbage-collected, keeping -the most recent few for in-flight readers. The redacted artifacts (`.json`/`.jsd`/`.tvim`/ +closed, and a lock-free reader loads exactly the generation the root names, so it gets +consistent snapshot isolation, raw text included. Superseded generations are +garbage-collected, keeping the most recent few for in-flight readers. The redacted artifacts (`.json`/`.jsd`/`.tvim`/ `.tvd`) never carry raw document or query text; only the `.tvtext` base + `.txd` journal hold raw text, and only the `.tvlex` base + `.lxd` journal hold payload-derived lexical terms. @@ -240,7 +240,7 @@ base plus a `.txd` delta journal, mirroring the state/vector journals so an incr journals only the upserted texts and deleted ids (O(changed), not a full-map rewrite) and a load replays the deltas onto the base. Every base and segment is checksum-guarded and fails closed on a corrupt/mismatched file. The store is deliberately **separate** from the redacted artifacts -above — none of them read it — so retrieval (`db.get`/`get_text`/`get_texts`, the `lodedb get` +above (none of them read it), so retrieval (`db.get`/`get_text`/`get_texts`, the `lodedb get` CLI command, `POST /get`, and the MCP `lodedb_get` tool) never weakens any payload-free guarantee. Removing a document journals a delete. Opening with `store_text=False` opts out entirely: no text is retained, the retrieval paths raise/return empty, and any existing store is @@ -315,8 +315,8 @@ The appender also ingests full text, so text writes are multi-producer too. Beca embed (embeddings live in the binding layer), the appender takes the *post-embedding* shape: it chunks the documents (`prepare_documents`, using the store writer's `chunk_character_limit` so chunk ids match), the binding embeds the returned chunks, and `append_embedded_documents` logs one -`apply_embedded_documents` record. This path is replay-independent -- it captures no base generation -and is never `PlanStale` like the single-writer `prepare_text_upsert`/`apply_text_upsert` -- because +`apply_embedded_documents` record. This path is replay-independent (it captures no base generation +and is never `PlanStale` like the single-writer `prepare_text_upsert`/`apply_text_upsert`) because the folding writer resolves a replacement's retired chunks from its own index state (so the appended record carries an empty removed-chunk list) and normalizes each record's text/tokens to its own `store_text`/`index_text` on fold. It is `append_text`/`append_text_many` in Python (`lodedb.Appender`, @@ -327,8 +327,8 @@ checkpointer* (`CoreCheckpointer`, exposed over the C ABI, in Python as `lodedb. Swift as `LodeCheckpointer`) removes that: one process folds the WAL into fresh committed generations continuously while appenders keep logging, so appended records become durable (and visible to a reader's `refresh`) without an application ever taking a writable open. It holds a crash-reclaimable -*lease* on a `/.lodedb.checkpoint.lock` sentinel -- distinct from the writer lock, so it excludes -neither appenders nor the writer and only elects one checkpointer at a time -- and takes the exclusive +*lease* on a `/.lodedb.checkpoint.lock` sentinel (distinct from the writer lock, so it excludes +neither appenders nor the writer and only elects one checkpointer at a time) and takes the exclusive writer lock solely for the brief window of each `checkpoint()`. Single-applier ordering is preserved because every fold runs under that same exclusive lock a writable open would take; the fold is incremental (only records above the applied-LSN watermark, onto the already-warm index), and if a @@ -342,5 +342,5 @@ its lifetime: it takes the shared lock per append (re-reading the committed gene reservation always clears a fold that advanced it between appends) and releases it in between, which is what lets the checkpointer take the exclusive lock to fold. This makes the appender's O(1) torn-tail repair span concurrent WAL growth: when bytes sit past its in-session watermark, it scans and truncates -only to the last CRC-valid frame -- preserving records a concurrent writer or checkpointer left there --- rather than dropping the tail back to a now-stale boundary. +only to the last CRC-valid frame rather than dropping the tail back to a now-stale +boundary. Records a concurrent writer or checkpointer left there survive. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 57cda4f1..c84e1f2f 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -140,8 +140,8 @@ The launch figures (rendered by `benchmarks/direct_gpu_sweep/diagrams.py`): `gpu_scan_ms` is the vector-scan proof (`gpu_stage_one_search_ms`); `batch_latency_ms` is end-to-end (it includes query embedding and result hydration); `recall_parity` shows the -CPU/GPU document-recall gap staying within tolerance. Two more figures — `search_p50_ms` -and `memory_copy` (host→device transfer) — are in +CPU/GPU document-recall gap staying within tolerance. Two more figures, `search_p50_ms` +and `memory_copy` (host→device transfer), are in [`benchmarks/direct_gpu_sweep/docs/`](../benchmarks/direct_gpu_sweep/docs). ## Laptop: embedding throughput + CPU scan latency @@ -159,23 +159,23 @@ single run (model pre-warmed), embed batch 64. Reproduce with `python benchmarks | `mps` | sentence-transformers | **1,393** | 0.25 / 0.41 | 8.42 / 9.58 | | `cpu` | sentence-transformers | 836 | 0.22 / 0.25 | **5.73 / 7.68** | -¹ Warm `add_many` throughput — the embedding model is pre-loaded before timing, so this is +¹ Warm `add_many` throughput: the embedding model is pre-loaded before timing, so this is steady state, not the one-time model load. ² The TurboVec scan is the same -CPU/NEON kernel regardless of embed device, so the ~0.25 ms p50 is consistent across rows — -this is the database's own work. ³ Embed one query, then scan; this is what interactive, +CPU/NEON kernel regardless of embed device, so the ~0.25 ms p50 is consistent across rows. +This is the database's own work. ³ Embed one query, then scan; this is what interactive, one-query-at-a-time serving pays. ![Embedding throughput by device](../benchmarks/laptop/docs/embed_throughput.png) ![End-to-end query latency by device](../benchmarks/laptop/docs/query_latency.png) -Reading: the **scan is sub-millisecond** and device-independent — that's LodeDB's job, and +Reading: the **scan is sub-millisecond** and device-independent. That's LodeDB's job, and it's fast. **MPS** leads bulk indexing (it's the GPU); for **single queries** the CPU is actually a touch faster end-to-end here, because MPS pays a per-call dispatch overhead that doesn't amortize over one query. Pick the embed device for your workload and re-measure on -your own hardware — these are single-run numbers on one machine. +your own hardware; these are single-run numbers on one machine. -### Apple-GPU (MPS) vector scan — first-class opt-in, off by default +### Apple-GPU (MPS) vector scan: first-class opt-in, off by default The MPS exact scan (`lodedb.engine.mps_turbovec`) is selectable via `LODEDB_MPS_DIRECT_TURBOVEC=auto|required`: dequantized fp16 rows live resident on the Apple GPU @@ -195,14 +195,14 @@ launch claim and current GPU-direct policy. The optional `[gpu]` extra adds a GPU-resident **exact** scan (`lodedb.engine.gpu_turbovec`): every row is reconstructed once to fp16 on the GPU, and query batches are scored with a GEMM plus a streaming device-side top-k. The benchmark compares it against the **vanilla TurboVec -CPU SIMD scan** over the same index — the same comparison the upstream TurboVec repo plots, +CPU SIMD scan** over the same index. This is the comparison the upstream TurboVec repo plots, with the GPU path in place of FAISS. **Measured** on Modal **A10** and **L40S** (CUDA), 100K vectors, 1,000 queries, k=64, median of repeated passes. Recall uses real OpenAI-DBpedia embeddings (d=1536, d=3072) and GloVe-200. -Reproduce on any CUDA host — see [`../benchmarks/gpu_vanilla_vs_augmented/`](../benchmarks/gpu_vanilla_vs_augmented). +Reproduce on any CUDA host; see [`../benchmarks/gpu_vanilla_vs_augmented/`](../benchmarks/gpu_vanilla_vs_augmented). -### Search speed — the win is batched +### Search speed: the win is batched The GPU path scores via a batched GEMM, so its throughput climbs with batch size while the CPU SIMD scan is batch-insensitive. d=1536, 4-bit, 100K, on the A10: @@ -216,13 +216,13 @@ CPU SIMD scan is batch-insensitive. d=1536, 4-bit, 100K, on the A10: | 1024 | 24,037 | **2.8×** | Crossover is ≈ batch 50. On an **L40S** the same sweep reaches **50,326 q/s at batch 1024 = -4.8×** the CPU baseline — the advantage scales with GPU class and would go further on an +4.8×** the CPU baseline. The advantage scales with GPU class and would go further on an A100/H100. Single queries and 2-bit codes favor the compact CPU scan (d=1536/2-bit is 0.8×), so route single queries to the CPU and batches to the GPU. ![GPU throughput vs. batch size](../benchmarks/gpu_vanilla_vs_augmented/docs/speed_batch.png) -### Recall — preserved +### Recall: preserved R@1-within-top-k, vanilla quantized scan vs. augmented GPU exact-reconstruction: @@ -233,19 +233,19 @@ R@1-within-top-k, vanilla quantized scan vs. augmented GPU exact-reconstruction: | glove-200 | 0.556 / 0.559 | 0.834 / **0.844** | At d=1536/3072 the quantization error is negligible, so exact scoring doesn't move recall. At -GloVe d=200 — where the error is largest — exact reconstruction helps a little (+0.010 at +GloVe d=200, where the error is largest, exact reconstruction helps a little (+0.010 at 4-bit k=1). The GPU path's value is throughput, not a recall jump. ![Recall: vanilla vs. augmented](../benchmarks/gpu_vanilla_vs_augmented/docs/recall.png) -### Memory — the GPU's cost +### Memory: the GPU's cost The throughput is bought with memory: fp16-resident vectors are ~4× the compact codes (d=1536, 4-bit: 3,072 vs. 768 bytes/vector). ![Memory per vector](../benchmarks/gpu_vanilla_vs_augmented/docs/memory.png) -### Update / persist — the CPU delta win +### Update / persist: the CPU delta win A separate augmentation, on the CPU side: applying 1,000 changed rows and persisting. The vanilla full `.tvim` rewrite is O(N); the encoded-row delta export is O(changed) and stays diff --git a/docs/graph.md b/docs/graph.md index e3d957dd..091489ac 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -180,8 +180,8 @@ There are two embedding modes: - **Vector-only (bring your own embeddings).** Open the graph with a `vector_dim` and the semantic index has no embedder: the adapter stores each node's own `embedding` and `vector_query` searches by `query.query_embedding`. This is what LlamaIndex's high-level - `PropertyGraphIndex` / `VectorContextRetriever` use, so it works with **any** `embed_model` - — set the dimension to the embedder's: + `PropertyGraphIndex` / `VectorContextRetriever` use, so it works with **any** `embed_model`. + Set the dimension to the embedder's: ```python # match your embed_model's dimension (e.g. 1536); nodes and queries stay in one space diff --git a/docs/integrations.md b/docs/integrations.md index 341008ca..ee96af1f 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -189,14 +189,14 @@ the local-bridge step first. ### kotaemon - **Repo:** Cinnamon/kotaemon (~26k★, Apache-2.0). Local, open-source RAG UI for chatting - with documents. **Install:** none — the adapter is dependency-free (it duck-types + with documents. **Install:** none. The adapter is dependency-free (it duck-types kotaemon's `DocumentWithEmbedding` and LlamaIndex's `MetadataFilters` rather than importing either package), so plain `lodedb` inside kotaemon's environment is enough. - **Pattern:** `lodedb.local.integrations.kotaemon.LodeDBVectorStore` implements kotaemon's `BaseVectorStore` contract (`add` / `delete` / `query` / `drop`, plus `count` and `__persist_flow__`). kotaemon resolves its vector store from a dotted `__type__` path via - theflow's `deserialize`, so selection is a settings change only — no kotaemon fork, no - registration call: + theflow's `deserialize`, so selection is a settings change only, with no kotaemon fork + and no registration call: ```python # flowsettings.py @@ -210,12 +210,12 @@ the local-bridge step first. LodeDB store under `/`. See [`examples/kotaemon_store.py`](../examples/kotaemon_store.py). - **Why it fits:** kotaemon's default is a Chroma collection per file index, rebuilt chunk - batch by chunk batch as users upload files — exactly the incremental-write workload + batch by chunk batch as users upload files. That is exactly the incremental-write workload LodeDB's O(changed) delta persistence targets, and the app is local-first so a zero-server embedded store is a category match. - **Embeddings:** kotaemon owns embedding (its pipelines pass precomputed vectors), so the - adapter uses LodeDB's vector-in path. kotaemon never configures an embedding dimension — - the store meets whatever model the user selected — so the LodeDB index is created lazily on + adapter uses LodeDB's vector-in path. kotaemon never configures an embedding dimension + (the store meets whatever model the user selected), so the LodeDB index is created lazily on first `add` and its shape is recorded in a `kotaemon_store.json` sidecar for reopens. Dimensions that are not a multiple of 8 (a LodeDB index requirement) are zero-padded up, which changes neither norms nor dot products before LodeDB's configured quantization. Chunk @@ -230,14 +230,14 @@ the local-bridge step first. default on normalized vectors (14/14 checks). - **Benchmark (kotaemon interface, 20k × 384-dim normalized vectors, batches of 100, one process per backend, M-series CPU):** vs the default `ChromaVectorStore`, byte-identical - vectors/ids/metadata/top-k on both sides — ingest 1.8s vs 16.9s (~9×); per-100-chunk add + vectors/ids/metadata/top-k on both sides. Ingest 1.8s vs 16.9s (~9×); per-100-chunk add (= durable commit at this seam) p50 8.6ms / p95 8.9ms / p99 9.0ms vs p50 83.9ms / p95 111ms / p99 126ms (the O(changed) tail story); query p50 0.17ms / p95 0.22ms vs 64.2ms / 65.1ms; 500-chunk scoped query p50 0.31ms vs 64.5ms; disk 10.5MB vs 424MB (~40×); peak RSS ~even (592MB vs 577MB, both dominated by the benchmark fixture); top-1 accuracy vs a brute-force oracle 1.000 vs 0.440 (near-tie random vectors are adversarial for HNSW; real-embedding gaps are smaller, but exact scan cannot miss the true neighbor). Chroma's - cold reopen is faster (0.07s vs 0.43s). Batch-query latency is N/A at this seam — + cold reopen is faster (0.07s vs 0.43s). Batch-query latency is N/A at this seam because kotaemon's `BaseVectorStore` has no batch-query API (retrieval issues single queries); LodeDB's batch path is covered by the repo's own benchmarks. Score scale differs by design: LodeDB returns cosine similarity, Chroma's default collection ranks by L2 @@ -247,7 +247,7 @@ the local-bridge step first. - **Scope:** vector store only; kotaemon's docstore (LanceDB/Elasticsearch/SimpleFile) and graph indices are separate seams and unchanged. Upstream, kotaemon bundles store choices in `libs/kotaemon/kotaemon/storages/vectorstores/`; a PR there would add a thin subclass plus - a `pip install lodedb` extra in their docs — the settings-path integration above works + a `pip install lodedb` extra in their docs; the settings-path integration above works today without it. ### cognee ([#74](https://github.com/Egoist-Machines/LodeDB/pull/74), in review) diff --git a/docs/late-interaction.md b/docs/late-interaction.md index b768c82c..e993efa3 100644 --- a/docs/late-interaction.md +++ b/docs/late-interaction.md @@ -17,8 +17,8 @@ index, with no engine change. Each document is **one row**: its id is the docume id, its vector is the mean-pooled patch vector, and its full patch matrix is kept in the per-row text sidecar (at the `storage` precision). An unfiltered query is answered by a resident exact scan that holds every patch in one in-memory matrix -and scores the whole corpus in a single GEMM plus a segmented max -- returning the -true top-k at a few milliseconds on thousands of pages, with no candidate-recall +and scores the whole corpus in a single GEMM plus a segmented max. It returns the +true top-k in a few milliseconds on thousands of pages, with no candidate-recall loss. See [Search paths](#search-paths) and [Storage precision](#storage-precision). ## Encoder is bring-your-own @@ -63,7 +63,7 @@ idx.search(query_tokens, k=5, filter={"file": "report.pdf"}) - **Resident** (default, unfiltered, within the memory budget): every patch is held in one in-memory matrix and the whole corpus is scored in a single GEMM plus a - segmented max -- a few milliseconds on thousands of pages. Built on the first + segmented max, a few milliseconds on thousands of pages. Built on the first query (`resident_build_seconds` in the benchmark) and then **maintained in place** on each `add`/`remove` (new documents fold into a pending delta, replaced/removed ones are tombstoned, and the base is compacted periodically), so an interleaved @@ -71,14 +71,14 @@ idx.search(query_tokens, k=5, filter={"file": "report.pdf"}) `resident=True|False|"auto"` (default `"auto"`) and `resident_max_bytes` (default 512 MB); a corpus that grows past the budget evicts to the streaming path. - **Filtered**: a query with a `filter` resolves the matching documents - engine-side and scores that subset exhaustively -- so a metadata filter both + engine-side and scores that subset exhaustively, so a metadata filter both narrows and stays exact. - **Streaming**: a corpus over the resident budget (or `resident=False`) is scored - by reading documents back from disk in bounded chunks -- slower (disk-bound) but - exact and near-constant-memory, so the exact path is never capped by RAM. + by reading documents back from disk in bounded chunks. It is slower (disk-bound) + but exact and near-constant-memory, so the exact path is never capped by RAM. Memory note: *patch decode and scoring* are bounded to the chunk budget (the one -exception is a single document whose patch matrix exceeds the budget -- it is +exception is a single document whose patch matrix exceeds the budget; it is scored whole, since a document is the unit of MaxSim). Two things are not bounded by the chunk budget: the streaming path first enumerates the document `(id, patch_count)` list, which is O(number of documents) in metadata; and the @@ -92,7 +92,7 @@ read-back ~38%, scoring ~1%). The resident scan removes both dominant costs. ### Concurrency -A single index handle is safe to use from multiple threads -- `add` / `remove` and +A single index handle is safe to use from multiple threads. `add` / `remove` and `search` / `search_many` can run concurrently (as with the underlying `LodeDB`, e.g. behind `lodedb serve`). Each mutation's commit and resident-cache update are applied as one ordered unit, and a query scores a consistent snapshot of the @@ -100,9 +100,9 @@ cache, so a query never observes a half-applied write or a stale cache. ### Batched queries -`search_many(queries, k=...)` scores a list of multi-vector queries together -- -all queries' tokens go through one GEMM per document chunk instead of one GEMM per -query -- and returns the top-k for each, in order. Use it to evaluate or answer +`search_many(queries, k=...)` scores a list of multi-vector queries +together (all queries' tokens go through one GEMM per document chunk instead of +one GEMM per query) and returns the top-k for each, in order. Use it to evaluate or answer many queries at once (~3x throughput in a 2k-document benchmark); single-query `search` latency is unchanged. diff --git a/docs/native_core_wal_payloads.md b/docs/native_core_wal_payloads.md index 37048e9f..8afcbc3d 100644 --- a/docs/native_core_wal_payloads.md +++ b/docs/native_core_wal_payloads.md @@ -23,7 +23,7 @@ root manifest has been published, so replay after a crash is idempotent. A WAL *segment* is an immutable standalone blob in exactly the file format above (header plus CRC-framed records), produced by `encode_wal_segment` with -no store open — the building block for out-of-band ingest (see +no store open. It is the building block for out-of-band ingest (see `lodedb.local.segments`). Two deliberate differences from the on-disk `.wal`: @@ -35,7 +35,7 @@ no store open — the building block for out-of-band ingest (see of an already-folded segment). - **Decoding is strict.** A segment is complete by construction, so a short header, torn tail, or trailing CRC failure means a corrupt transfer and fails - closed — unlike the crash-tolerant `.wal` reader, which drops a torn + closed, unlike the crash-tolerant `.wal` reader, which drops a torn final frame. Raw document text enters a segment only when the plan was built with diff --git a/src/lodedb/__init__.py b/src/lodedb/__init__.py index 9e21b765..9d0a0a8a 100644 --- a/src/lodedb/__init__.py +++ b/src/lodedb/__init__.py @@ -1,4 +1,4 @@ -"""lodedb — the public API for LodeDB. +"""lodedb is the public API for LodeDB. Re-exports the local-first, on-disk SDK (:mod:`lodedb.local`) and the ``lodedb`` CLI, so you can write ``import lodedb`` / ``from lodedb import LodeDB``. LodeDB diff --git a/src/lodedb/cloud/__init__.py b/src/lodedb/cloud/__init__.py index 992be666..df2df65a 100644 --- a/src/lodedb/cloud/__init__.py +++ b/src/lodedb/cloud/__init__.py @@ -1,4 +1,4 @@ -"""``lodedb.cloud`` — the OreCloud managed-cloud client (the ``[cloud]`` extra). +"""``lodedb.cloud`` is the OreCloud managed-cloud client (the ``[cloud]`` extra). First-party code, shipped in the lodedb wheel: :class:`Client` (one credential, one bound org/environment, per-user store handles), @@ -7,7 +7,7 @@ ``sync`` / ``status`` / ``verify`` / ``keys``) over the native transfer core bundled as ``lodedb._turbovec.cloud``, and the ``lodedb cloud`` CLI. The ``[cloud]`` extra installs only the client's third-party dependencies -(``httpx``, ``pynacl``) — there is no separate cloud distribution. +(``httpx``, ``pynacl``); there is no separate cloud distribution. Every name resolves lazily (PEP 562): importing this module (or ``lodedb`` itself) touches neither httpx nor the network, guarded by @@ -43,7 +43,7 @@ # HTTP-backed exports and the submodule each lives in. These modules import # httpx (and, for login, pynacl) at module level, so they are reached only -# through this lazy table — never on a plain import. +# through this lazy table, never on a plain import. _HTTP_EXPORTS = { "Client": "client", "connect": "serving", @@ -62,7 +62,7 @@ def open_cloud_target(target: str, options: dict[str, Any]) -> Any: """Opens a managed-cloud target via :func:`connect` and returns its store - handle — the one funnel behind both constructor front doors, + handle, the one funnel behind both constructor front doors, ``LodeDB.cloud("user-42")`` and the ``LodeDB("orecloud://…")`` config-string dispatch. @@ -70,7 +70,7 @@ def open_cloud_target(target: str, options: dict[str, Any]) -> Any: stays network-free; without the ``[cloud]`` extra's dependencies this raises an ``ImportError`` carrying the install hint instead of a bare traceback. ``options`` must be keywords of :func:`connect` (``token``, - ``host``, ...) — local-only construction options (``model=``, + ``host``, ...). Local-only construction options (``model=``, ``read_only=``, ...) are rejected up front, because embedding and storage for a cloud store are configured server-side, not per handle. """ @@ -96,8 +96,8 @@ def open_cloud_target(target: str, options: dict[str, Any]) -> Any: def __getattr__(name: str) -> Any: - """PEP 562: resolves the public surface lazily — transfer verbs from the - bundled native extension, HTTP-backed names from their submodule (with + """PEP 562: resolves the public surface lazily. Transfer verbs come from + the bundled native extension, HTTP-backed names from their submodule (with the install hint when the extra's dependencies are absent).""" if name in _NATIVE_EXPORTS: from lodedb import _turbovec diff --git a/src/lodedb/cloud/_agents_scaffold.py b/src/lodedb/cloud/_agents_scaffold.py index 22cbf71f..6f443875 100644 --- a/src/lodedb/cloud/_agents_scaffold.py +++ b/src/lodedb/cloud/_agents_scaffold.py @@ -2,14 +2,14 @@ repo one-shot for a coding agent, generated from the repo's REAL link (host, org, environment, store) rather than placeholders. -Three artifacts, all safe to commit (no secrets — credentials stay in env +Three artifacts, all safe to commit (no secrets; credentials stay in env vars or ~/.orecloud): -- `.claude/skills/orecloud/SKILL.md` — Agent Skills spec (name + description +- `.claude/skills/orecloud/SKILL.md`: Agent Skills spec (name + description frontmatter, short body); read by Claude Code, Cursor, Codex, and Copilot. Ours to regenerate: rewritten on every run so it tracks the link. - An `## OreCloud` section appended to `AGENTS.md` (created if absent); - never touched again once the heading exists — the section is theirs to + never touched again once the heading exists. The section is theirs to edit after the first drop. - An `orecloud-control` entry in `.mcp.json` pointing at the control-plane MCP endpoint with `${ORECLOUD_TOKEN}` env expansion; merged into an @@ -148,7 +148,7 @@ def scaffold_agent_artifacts( """Writes the three artifacts into `local_dir`. Returns (written, notes): repo-relative paths that were created/updated, and human-readable notes for anything deliberately left alone. Paths use forward slashes on every - platform — they land in the CLI's JSON output, so the shape is a contract, + platform; they land in the CLI's JSON output, so the shape is a contract, not a display choice.""" root = Path(local_dir) written: list[str] = [] @@ -163,7 +163,7 @@ def scaffold_agent_artifacts( if agents_path.exists(): existing = agents_path.read_text(encoding="utf-8") if AGENTS_SECTION_HEADING in existing: - # The section is the repo's to edit after the first drop — + # The section is the repo's to edit after the first drop; # regenerating over their changes would be data loss. notes.append(f"AGENTS.md already has an '{AGENTS_SECTION_HEADING}' section; left as is") else: diff --git a/src/lodedb/cloud/_config.py b/src/lodedb/cloud/_config.py index c9842615..e301f538 100644 --- a/src/lodedb/cloud/_config.py +++ b/src/lodedb/cloud/_config.py @@ -3,7 +3,7 @@ One JSON file, `~/.orecloud/credentials.json`, chmod 0600, holding the control-plane host and the personal access token from `lodedb cloud login`. A plain file (not the OS keychain) is deliberate for now: it works headless -— CI boxes, containers, SSH sessions — with the same permission story as +(CI boxes, containers, SSH sessions) with the same permission story as `~/.aws/credentials`. ORECLOUD_TOKEN overrides the file entirely, so ephemeral environments never need to write it; the host defaults to the hosted control plane, with ORECLOUD_HOST (or `--host`) overriding it for @@ -38,7 +38,7 @@ def credentials_file() -> Path: class Credentials: host: str token: str - source: str # "env" or "file" — surfaced by whoami for debuggability + source: str # "env" or "file", surfaced by whoami for debuggability class CredentialsError(RuntimeError): @@ -102,8 +102,8 @@ def delete_credentials() -> bool: # ---------------------------------------------------------------- orecloud.toml # The committable per-directory remote record: which control plane and -# org/environment/store a data directory syncs with. No secrets — credentials -# stay in ~/.orecloud — so the file is safe to commit next to the data. +# org/environment/store a data directory syncs with. No secrets (credentials +# stay in ~/.orecloud), so the file is safe to commit next to the data. REMOTE_FILE_NAME = "orecloud.toml" @@ -121,7 +121,7 @@ def remote_config_file(local_dir: str | Path) -> Path: def load_remote(local_dir: str | Path) -> RemoteConfig | None: """The directory's recorded remote, or None. A present-but-malformed file - raises — guessing around a broken record could aim a push at the wrong + raises; guessing around a broken record could aim a push at the wrong environment.""" import tomllib diff --git a/src/lodedb/cloud/_env_file.py b/src/lodedb/cloud/_env_file.py index 0d67ecfa..9056ebf5 100644 --- a/src/lodedb/cloud/_env_file.py +++ b/src/lodedb/cloud/_env_file.py @@ -24,7 +24,7 @@ def write_env_values(path: str | Path, values: dict[str, str]) -> Path: """Set `KEY=value` lines in the dotenv file at `path`, creating it if missing. Managed keys replace their existing line in place; every other - line — comments, other keys, blanks — is preserved. The result is always + line (comments, other keys, blanks) is preserved. The result is always owner-only (0600): secrets land here, and a pre-existing 0644 `.env` must not keep the minted token group/world-readable. Returns the path.""" target = Path(path) diff --git a/src/lodedb/cloud/cli.py b/src/lodedb/cloud/cli.py index e3593332..989a67cf 100644 --- a/src/lodedb/cloud/cli.py +++ b/src/lodedb/cloud/cli.py @@ -2,7 +2,7 @@ transfer core (`lodedb._turbovec.cloud`), plus the cloud verbs (login, tenancy, managed transfer). -Thin by design — argument parsing and report formatting only. Every operation +Thin by design, argument parsing and report formatting only. Every operation (and all of its validation) lives in the Rust core or `lodedb.cloud.transfer`, so the CLI cannot offer a code path the library does not have. @@ -13,15 +13,15 @@ - Errors print to stderr as `error: …` (+ a `hint: …` line naming the exact next command where one exists), with an exit code per failure class: 1 unexpected, 2 usage/local config, 3 auth (401/403), 4 not found (404), - 5 refused (402/409/422 — understood and denied), 6 transient (425/429/503 - — retry with backoff). + 5 refused (402/409/422, understood and denied), 6 transient (425/429/503, + retry with backoff). - Confirmation prompts never hang a pipe: without a TTY they fail immediately, naming `--yes`. Transfer verbs accept three REMOTE spellings: a directory path or `s3://` URL (the dumb targets, handled entirely by the Rust core), an explicit `orecloud://org/environment[/store]` managed target, or the literal word -`cloud` — the managed remote recorded in LOCAL_DIR/orecloud.toml by +`cloud`, the managed remote recorded in LOCAL_DIR/orecloud.toml by `lodedb cloud init`/`lodedb cloud link`. """ @@ -117,7 +117,7 @@ def _note(message: str) -> None: def _run(operation: Callable[[], T]) -> T: """Runs one binding operation, mapping its exceptions onto stderr + exit 1 (sync refusals onto the documented "refused" class, like their managed - twins — the message itself carries the force-flag/checkpoint hint).""" + twins; the message itself carries the force-flag/checkpoint hint).""" try: return operation() except _core.SyncConflictError as error: @@ -238,8 +238,8 @@ def sync( """Sync KEY between LOCAL_DIR and REMOTE: fast-forward in whichever direction moved. Refuses (with a nonzero exit) when the two ends diverged past the last - recorded sync, or when there is no sync record and the ends differ — - resolve with exactly one of --force-push / --force-pull. + recorded sync, or when there is no sync record and the ends differ. + Resolve with exactly one of --force-push / --force-pull. """ if force_push and force_pull: typer.secho( @@ -362,7 +362,7 @@ def _fail(message: str, code: int = EXIT_UNEXPECTED, hint: str | None = None) -> def _classify(error: CloudError) -> tuple[int, str | None]: """Maps a control-plane refusal to (exit code, fix hint). The hint names - the exact next command — errors double as prompts for agents.""" + the exact next command, so errors double as prompts for agents.""" status = error.status_code if status == 401: return EXIT_AUTH, ( @@ -399,7 +399,7 @@ def _classify(error: CloudError) -> tuple[int, str | None]: def _confirm(prompt: str) -> None: """`typer.confirm` with an agent guard: when stdin is not a TTY there is - nobody to answer — fail immediately, naming `--yes`, instead of hanging + nobody to answer, so fail immediately, naming `--yes`, instead of hanging the calling program.""" if not sys.stdin.isatty(): raise _fail( @@ -454,8 +454,8 @@ def _cloud(operation: Callable[[], T]) -> T: def _org_scope(client: CloudClient, org: str | None) -> str: """The org a command addresses: the explicit --org, else the token's - binding (environment tokens), else the account's only org — failing - with the actual choices when several orgs qualify.""" + binding (environment tokens), else the account's only org. When several + orgs qualify it fails with the actual choices.""" if org: return org info = _cloud(client.token_self) @@ -475,10 +475,10 @@ def _org_scope(client: CloudClient, org: str | None) -> str: def _tenancy( client: CloudClient, org: str | None, environment: str | None ) -> tuple[str, str]: - """The (org, environment) pair a command addresses — the SDK's - resolution (explicit flags win; environment tokens supply their binding - and refuse contradictions; personal tokens fall back to the only - org/environment), re-raised with the CLI flags in the hint.""" + """The (org, environment) pair a command addresses. Delegates to the + SDK's resolution (explicit flags win; environment tokens supply their + binding and refuse contradictions; personal tokens fall back to the only + org/environment) and re-raises failures with the CLI flags in the hint.""" from lodedb.cloud.client import resolve_tenancy try: @@ -550,7 +550,7 @@ def login( """Log in to an OreCloud control plane and store the credential locally. Default flow: this machine generates a keypair, your browser approves the - login, and the server returns the new token sealed to that keypair — the + login, and the server returns the new token sealed to that keypair. The secret never crosses in the clear and is never stored server-side. """ creds = _load_credentials() @@ -563,7 +563,7 @@ def login( if token is None: token = _browser_login(host, open_browser=not no_browser) - # Prove the credential works before persisting it — a typo'd --token must + # Prove the credential works before persisting it. A typo'd --token must # not leave broken state behind. with CloudClient(host, token) as client: me = _cloud(client.me) @@ -580,7 +580,7 @@ def _browser_login(host: str, open_browser: bool = True) -> str: """The sealed-box browser handoff against `host`: prints the confirmation code and approval URL to stderr, blocks until a signed-in browser approves, and returns the minted token secret (sealed in transit to this process's - keypair — it never crosses in the clear).""" + keypair, so it never crosses in the clear).""" with CloudClient(host) as anonymous: handoff = _cloud( lambda: LoginHandoff( @@ -596,8 +596,8 @@ def _browser_login(host: str, open_browser: bool = True) -> str: def _ensure_logged_in(host: str | None) -> "_config.Credentials": - """Stored credentials, or an inline sealed-box login when there are none — - the single human touchpoint of `lodedb cloud init`. A `host` that contradicts + """Stored credentials, or an inline sealed-box login when there are none. + This is the single human touchpoint of `lodedb cloud init`. A `host` that contradicts the stored credential fails closed rather than silently re-aiming the command at another control plane.""" creds = _load_credentials() @@ -656,7 +656,7 @@ def _agents_repo_token( """The repo's deterministic `init-` scoped key, written into the dotenv file. Reused when the file still holds a live token of that name; otherwise every stale same-name token is revoked and one fresh key is - minted — re-running init never accumulates keys. Returns (report data, + minted, so re-running init never accumulates keys. Returns (report data, notes).""" from datetime import datetime @@ -683,8 +683,8 @@ def alive(row: Mapping[str, object]) -> bool: held = [row for row in live if existing_secret.startswith(str(row["prefix"]))] if held: # Reuse the key the file already holds; only heal the host line - # (the token line stays untouched — we could not rewrite it anyway, - # secrets are shown once). + # (the token line stays untouched; secrets are shown once, so we + # could not rewrite it anyway). env.write_env_values(env_file, {"ORECLOUD_HOST": host}) return ( {"token": held[0], "env_file": str(env_file)}, @@ -748,13 +748,13 @@ def init( ), ] = None, ) -> None: - """Link LOCAL_DIR to one of the org's environments — the one-command path + """Link LOCAL_DIR to one of the org's environments, the one-command path from a local data directory to a managed remote. With --agents this is the whole setup for a coding agent: log in if needed (one browser approval), mint a scoped key into .env, and scaffold the repo's agent artifacts. Environments are fixed (production and testing, seeded at - signup); init resolves the slug from the credential — or validates an - explicit one — rather than creating anything.""" + signup); init resolves the slug from the credential (or validates an + explicit one) rather than creating anything.""" creds = _ensure_logged_in(host) data: dict[str, object] = {} notes: list[str] = [] @@ -818,8 +818,8 @@ def auth_print_headers() -> None: The headersHelper contract (Claude Code and friends run a command to obtain MCP auth headers), so the browser-approved login feeds MCP - servers without anyone pasting a key into a config file. Always JSON — - that IS the output contract.""" + servers without anyone pasting a key into a config file. Always JSON. + That IS the output contract.""" creds = _load_credentials() if creds is None: raise _fail( @@ -831,7 +831,7 @@ def auth_print_headers() -> None: @app.command() def logout() -> None: - """Forget the locally stored credential (the token itself stays valid — + """Forget the locally stored credential (the token itself stays valid; revoke it with `lodedb cloud tokens revoke` to kill it server-side).""" deleted = _config.delete_credentials() _emit( @@ -908,7 +908,7 @@ def human() -> None: def _write_credential_env(env_file: str | Path, host: str, secret: str) -> dict[str, object]: """Write ORECLOUD_HOST + ORECLOUD_TOKEN into the dotenv file (the secret - never touches stdout — printed output becomes part of an agent's + never touches stdout because printed output becomes part of an agent's transcript) and make sure the file is git-ignored. Returns the env_file path and any gitignore note, for merging into the command's output.""" from lodedb.cloud import _env_file as env @@ -927,7 +927,7 @@ def tokens_mint( kind: Annotated[str, typer.Option(help="personal | secret | publishable.")] = "personal", scope: Annotated[ list[str], typer.Option("--scope", help="Repeatable: admin, write, read:search, read:text.") - ] = ["admin"], # noqa: B006 — typer requires a literal default + ] = ["admin"], # noqa: B006 # typer requires a literal default name: Annotated[str, typer.Option(help="Label shown in listings.")] = "", org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, @@ -942,8 +942,8 @@ def tokens_mint( ), ] = None, ) -> None: - """Mint a token. The secret is printed once and never retrievable again — - or, with --env-file, written straight to the file and never printed. + """Mint a token. The secret is printed once and never retrievable again. + With --env-file it is written straight to the file and never printed. Environment tokens (secret/publishable) default to the resolved org/environment; personal tokens take neither flag.""" creds = _load_credentials() @@ -1000,7 +1000,7 @@ def mcp_install( """Print ready-to-paste MCP configs (Claude Code, Cursor, VS Code) for one end user's store. The endpoint speaks Streamable HTTP with Bearer auth; the token's scopes decide which tools the agent sees. The store - needs no create step — it provisions on the agent's first write.""" + needs no create step; it provisions on the agent's first write.""" import json as json_module from urllib.parse import quote @@ -1008,7 +1008,7 @@ def mcp_install( org, environment = _tenancy(client, org, environment) creds = _load_credentials() assert creds is not None # _client() above required it - # One URL per end user's store — always the 3-segment form. Slugs are + # One URL per end user's store, always the 3-segment form. Slugs are # [a-z0-9-] but store names (end-user ids) are free-form: quote them. url = ( f"{creds.host}/mcp/{quote(org, safe='')}/" @@ -1213,7 +1213,7 @@ def human() -> None: if minted is None: return # The key was minted bound to this environment, so the snippet never - # restates the tenancy — Client resolves it from the credential. + # restates the tenancy; Client resolves it from the credential. typer.echo() typer.echo("Connect from Python (the key is shown once — this is your copy):") typer.echo() @@ -1241,7 +1241,7 @@ def _resolve_index_key( client: CloudClient, org: str, environment: str, store: str, key: str | None ) -> str: """The explicit key, or the store's only index key when it is unambiguous - (mirrors `store create`'s omit-the-key ergonomics). Exact-name lookup — + (mirrors `store create`'s omit-the-key ergonomics). Exact-name lookup, never a page walk (an environment holds one store per end user).""" if key is not None: return key @@ -1574,7 +1574,7 @@ def store_export( ] = True, ) -> None: """Export ALL of a user's memories as JSON (pages the browse endpoint - to exhaustion) — the per-end-user data-export verb.""" + to exhaustion). This is the per-end-user data-export verb.""" import json as _json with _client() as client: @@ -1658,7 +1658,7 @@ def store_delete_memories( ) -> None: """Delete a user's memories in place (expired ones included), keeping the store registered. Async like every cloud write. To forget the user - entirely — row, entitlement slot and all — use `store delete`.""" + entirely (row, entitlement slot and all), use `store delete`.""" with _client() as client: org, environment = _tenancy(client, org, environment) if not yes: diff --git a/src/lodedb/engine/_atomic_io.py b/src/lodedb/engine/_atomic_io.py index 20cd3d53..b9829ff2 100644 --- a/src/lodedb/engine/_atomic_io.py +++ b/src/lodedb/engine/_atomic_io.py @@ -2,8 +2,8 @@ LodeDB publishes every persisted file by writing a sibling ``*.tmp`` and then ``os.replace``-ing it into place. ``os.replace`` is atomic *within a filesystem* -on POSIX and Windows — a concurrent reader sees either the whole old file or the -whole new file, never a torn one — so **atomicity is always on** and free. +on POSIX and Windows (a concurrent reader sees either the whole old file or the +whole new file, never a torn one), so **atomicity is always on** and free. It is not, however, *durable*: after a power loss or kernel panic (as opposed to a clean process crash) neither the temp file's bytes nor the rename are diff --git a/src/lodedb/engine/_commit_manifest.py b/src/lodedb/engine/_commit_manifest.py index da804a3a..c0fbe787 100644 --- a/src/lodedb/engine/_commit_manifest.py +++ b/src/lodedb/engine/_commit_manifest.py @@ -1,9 +1,9 @@ """Per-index atomic commit manifest over generation-addressed artifacts. Stage 1 made each *file* publish atomically (temp + ``os.replace``) and gave the -single-writer guarantee, but a logical commit spans several files — the JSON +single-writer guarantee, but a logical commit spans several files (the JSON state base, its ``.jsd`` journal, the ``.tvim`` vector base, its ``.tvd`` -journal — published in sequence. A crash (or a lock-free reader) between those +journal), published in sequence. A crash (or a lock-free reader) between those publishes could observe a torn cross-file set: the existing loader *detected* that and failed closed, but could not *recover* to the last good state. @@ -26,17 +26,17 @@ The root manifest embeds the committed state of both per-store manifests plus counts and the live base epoch. A commit writes any new epoch artifacts first, -then atomically swaps ``.commit.json`` — that swap is the **only** thing +then atomically swaps ``.commit.json``; that swap is the **only** thing that commits a generation. Because bases are epoch-addressed they are never overwritten in place, so a crashed commit leaves the previous epoch fully intact; recovery just re-points at it and deletes the unreferenced artifacts. A lock-free reader reads the root manifest once and loads exactly the artifacts -it names — a consistent generation, never a torn cross-file mix. +it names, a consistent generation, never a torn cross-file mix. The opt-in raw-text store (``store_text=True``) is part of this atomic set: it has its own base + ``.txd`` delta journal under the same ``.gen/`` epoch and its manifest is embedded in the root, so raw text (visible via the public -``get`` API) commits and rolls back with exactly the generation the root names — +``get`` API) commits and rolls back with exactly the generation the root names, never leaking an uncommitted overwrite. It remains a separate store: no telemetry/redacted/audit path reads it. """ @@ -148,7 +148,7 @@ def write_commit_manifest(path: str | Path, body: dict[str, Any], *, fsync: bool The body embeds both per-store manifests and is serialized **exactly once** per commit: that single ``json.dumps`` is the dominant cost of the write (it dwarfs the file I/O), so the one serialization is reused for both the - integrity checksum and the on-disk payload — the wrapper is assembled around + integrity checksum and the on-disk payload; the wrapper is assembled around it rather than re-dumping the body inside a dict. """ diff --git a/src/lodedb/engine/_filelock.py b/src/lodedb/engine/_filelock.py index 7005b922..0aa559d8 100644 --- a/src/lodedb/engine/_filelock.py +++ b/src/lodedb/engine/_filelock.py @@ -1,18 +1,18 @@ """Single-writer file lock for LodeDB persistence (stdlib only). A LodeDB handle loads on-disk state at open and persists its own in-memory view, -so two *independent* writers on one directory cannot safely compose — interleaved +so two *independent* writers on one directory cannot safely compose; interleaved deltas leave the journal un-replayable. LodeDB is therefore **single-writer per path**: a handle takes an exclusive OS advisory lock when it opens and holds it until it closes. A second open blocks until the first closes (so it then loads the accumulated state and composes cleanly) and fails fast with -:class:`ConcurrentWriterError` once the timeout elapses — the model SQLite uses +:class:`ConcurrentWriterError` once the timeout elapses, the model SQLite uses with a busy-timeout. The kernel releases the lock automatically on process exit, so a crashed or forgotten handle never wedges the path. POSIX uses ``fcntl.flock``; Windows uses ``msvcrt.locking``. The lock is taken on a dedicated sentinel file ``/.lodedb.lock``, never the data files (which are -replaced via ``os.replace`` — a new inode that would drop a lock held on the old +replaced via ``os.replace``, a new inode that would drop a lock held on the old one). Advisory locks are unreliable on network filesystems (NFS/SMB); LodeDB targets local disk. """ @@ -71,7 +71,7 @@ class WriterLock: """An exclusive, held-for-the-handle's-lifetime advisory lock on a directory. Not reentrant: a second :meth:`acquire` on the same path (this process or - another) contends with the first and fails after the timeout — that is the + another) contends with the first and fails after the timeout; that is the single-writer guarantee. Acquire blocks (polling) until the lock is free or the timeout elapses; :meth:`release` is idempotent. """ diff --git a/src/lodedb/engine/_lexical.py b/src/lodedb/engine/_lexical.py index e25be51e..d1de0e30 100644 --- a/src/lodedb/engine/_lexical.py +++ b/src/lodedb/engine/_lexical.py @@ -13,16 +13,16 @@ Three pieces: -- :func:`tokenize` — lowercases and splits on whitespace/punctuation while +- :func:`tokenize`: lowercases and splits on whitespace/punctuation while keeping alphanumeric runs intact and preserving internal ``-``/``.``/``/`` inside code-like tokens, so ``E1234``, ``ABC-123``, and ``2024-01-15`` survive as single findable tokens. This is the whole point of the feature: the embedding cannot see an exact code, but the lexical index can. -- :class:`Bm25Index` — a classic Okapi BM25 inverted index over a unit space +- :class:`Bm25Index`: a classic Okapi BM25 inverted index over a unit space (LodeDB indexes chunks), scoring a tokenized query into a ranked list; it supports both a one-shot bulk build and incremental add/remove of single units so a small mutation does not force a full rebuild. -- :func:`reciprocal_rank_fusion` — fuses ranked id lists with RRF, which needs +- :func:`reciprocal_rank_fusion`: fuses ranked id lists with RRF, which needs no score normalization and therefore composes cleanly with the vector scores. BM25 reference (Okapi): ``score(D, Q) = Σ idf(q) · tf·(k1+1) / (tf + k1·(1 - b + diff --git a/src/lodedb/engine/_predicate.py b/src/lodedb/engine/_predicate.py index 8e572397..28cd65cd 100644 --- a/src/lodedb/engine/_predicate.py +++ b/src/lodedb/engine/_predicate.py @@ -1,12 +1,12 @@ """Metadata filter predicate engine for query-time document filtering. This module is intentionally dependency-free (stdlib only) and must not import -``core`` — it is consumed by both the engine trust boundary +``core``; it is consumed by both the engine trust boundary (``LodeEngine._validate_query_filter`` validates; ``core._compile_query_filter`` compiles the per-document matcher used by the corpus scan) and the SDK ergonomics layer (``lodedb.local.db._normalize_filter``). -Grammar (backward compatible — a bare scalar is exact-match sugar for ``$eq``): +Grammar (backward compatible; a bare scalar is exact-match sugar for ``$eq``): metadata-filter := { (field | logical)* } # entries are AND-ed field := : (scalar | operator-map) @@ -111,7 +111,7 @@ def compile_metadata_filter( The filter must already be validated (see :func:`validate_metadata_filter`); compilation trusts the grammar and does not re-check it. It hoists everything - independent of the document out of the per-document path — operator dispatch + independent of the document out of the per-document path: operator dispatch collapses into a bound comparator, ``$in``/``$nin`` targets stay tuples, and ordered operands are parsed to numbers once instead of re-running ``float()`` on the constant operand for every row scanned. The returned predicate is pure @@ -291,7 +291,7 @@ def _compile_ordered( operand_number = _as_number(operand) if operand_number is None: # The operand is not a finite number, so a numeric compare is impossible - # (both sides must parse) — it is always lexicographic, and the stored + # (both sides must parse); it is always lexicographic, and the stored # value never needs parsing. def _ordered_str( meta: Mapping[str, str], _f: str = field, _v: str = operand diff --git a/src/lodedb/engine/gpu_turbovec.py b/src/lodedb/engine/gpu_turbovec.py index 5150b1ee..cbc85e09 100644 --- a/src/lodedb/engine/gpu_turbovec.py +++ b/src/lodedb/engine/gpu_turbovec.py @@ -17,10 +17,10 @@ quantization error (the GPU score is the more faithful estimate of the same quantized representation; agreement is within the LUT tolerance, roughly ``1/sqrt(dim)``-scaled). Queries are rotated on the host with the -deterministic rotation — exactly the GEMM the native search path runs. +deterministic rotation, exactly the GEMM the native search path runs. No custom CUDA kernels: scoring is a tiled fp16→fp32 cast plus cuBLAS -matmul with a streaming top-k — each row tile's (batch x tile) scores are +matmul with a streaming top-k. Each row tile's (batch x tile) scores are selected with `torch.topk` and merged into a running per-query top-k, so the full batch x corpus score matrix is never materialized (bounding GPU memory regardless of batch/corpus). It falls back to a full `cupy.argpartition` when @@ -58,7 +58,7 @@ _TILE_TRANSIENT_BYTES = 128 << 20 # Each scoring tile's (batch x tile) fp32 score block stays under this budget, # so the streaming top-k never materializes the full batch x corpus score -# matrix (the old full-matrix path needed batch*corpus*4 bytes — ~3.5 GB at +# matrix (the old full-matrix path needed batch*corpus*4 bytes, ~3.5 GB at # batch 1024 x 860K rows, which exceeded GPU memory and forced a CPU fallback). _TILE_SCORE_BYTES = 64 << 20 diff --git a/src/lodedb/engine/route_registry.py b/src/lodedb/engine/route_registry.py index 0d7913e0..4a8d6a35 100644 --- a/src/lodedb/engine/route_registry.py +++ b/src/lodedb/engine/route_registry.py @@ -394,7 +394,7 @@ def default_route_registry() -> RouteRegistry: The local DB serves with the direct-TurboVec route policy, whose ``index_backend == "turbovec_direct"`` skips the registry-classification check (see :meth:`EngineRoutePolicy.validate_index_request`). So a registry - with no explicit entries — every lookup resolves to the full-vector default — + with no explicit entries (every lookup resolves to the full-vector default) is all the local path needs, and nothing has to be loaded from disk. """ diff --git a/src/lodedb/graph/__init__.py b/src/lodedb/graph/__init__.py index 1bc11187..49c2ad2e 100644 --- a/src/lodedb/graph/__init__.py +++ b/src/lodedb/graph/__init__.py @@ -1,4 +1,4 @@ -"""lodedb.graph — a hybrid knowledge-graph layer over LodeDB. +"""lodedb.graph is a hybrid knowledge-graph layer over LodeDB. Topology (nodes, typed edges) lives in an embedded SQLite sidecar built for deterministic traversal; LodeDB serves as the rebuildable semantic index for diff --git a/src/lodedb/graph/_store.py b/src/lodedb/graph/_store.py index 88eddaa2..3abf0123 100644 --- a/src/lodedb/graph/_store.py +++ b/src/lodedb/graph/_store.py @@ -1,4 +1,4 @@ -"""SQLite topology store — the source-of-truth adjacency for a knowledge graph. +"""SQLite topology store, the source-of-truth adjacency for a knowledge graph. LodeDB itself is an exact vector index, not a graph engine: ``search`` ranks the top-``k`` semantically similar items, which is the wrong primitive for "every @@ -8,7 +8,7 @@ *semantic* index over node/edge text. This module owns the topology half: CRUD plus the adjacency queries (`neighbors`, `edges_for`) that back k-hop traversal. -It is stdlib-only (``sqlite3``) and holds no embeddings — vectors live in LodeDB. +It is stdlib-only (``sqlite3``) and holds no embeddings; vectors live in LodeDB. """ from __future__ import annotations diff --git a/src/lodedb/graph/knowledge_graph.py b/src/lodedb/graph/knowledge_graph.py index 01e9c61a..626ea40a 100644 --- a/src/lodedb/graph/knowledge_graph.py +++ b/src/lodedb/graph/knowledge_graph.py @@ -1,4 +1,4 @@ -"""KnowledgeGraph — a hybrid graph/knowledge-graph layer over LodeDB. +"""KnowledgeGraph is a hybrid graph/knowledge-graph layer over LodeDB. The architecture is the one every comparable agent-memory system uses (Zep/ Graphiti, cognee): keep the **topology** in a store built for traversal and use @@ -11,10 +11,10 @@ - **SQLite is the source of truth; LodeDB is a rebuildable index.** Writes hit SQLite first, then the index. If an index write is lost (crash between the two), the topology is still correct and :meth:`reindex` rebuilds the index - from it — so there is no cross-store atomicity problem to get wrong. + from it, so there is no cross-store atomicity problem to get wrong. - **Retrieval is hybrid.** :meth:`search_subgraph` runs a LodeDB similarity search to find entry-point nodes, then does deterministic k-hop expansion over - the SQLite adjacency — the complete-set traversal LodeDB's top-k search can't + the SQLite adjacency, the complete-set traversal LodeDB's top-k search can't express on its own. Nodes are embedded by their ``label`` text (LodeDB embeds internally) or by a @@ -466,7 +466,7 @@ def list_nodes(self) -> list[Node]: def list_edges(self) -> list[Edge]: """Returns every edge in the graph (complete-set enumeration). - Like :meth:`list_nodes`, full enumeration over the topology store — the + Like :meth:`list_nodes`, full enumeration over the topology store, the primitive behind an "all triplets" / "all edges of relation R" view. """ diff --git a/src/lodedb/local/__init__.py b/src/lodedb/local/__init__.py index 67d2d811..18e83495 100644 --- a/src/lodedb/local/__init__.py +++ b/src/lodedb/local/__init__.py @@ -2,12 +2,12 @@ Public surface: -- :class:`LodeDB` — the embedded SDK entrypoint. -- :class:`LodeSearchHit` — one redacted ``(score, id, metadata)`` result row. -- :class:`LodeLateInteractionIndex` — multi-vector (MaxSim) late-interaction retrieval. -- :func:`resolve_local_device` / :func:`build_local_embedding_backend` — +- :class:`LodeDB`: the embedded SDK entrypoint. +- :class:`LodeSearchHit`: one redacted ``(score, id, metadata)`` result row. +- :class:`LodeLateInteractionIndex`: multi-vector (MaxSim) late-interaction retrieval. +- :func:`resolve_local_device` / :func:`build_local_embedding_backend`: embedding device selection (MPS / CUDA / CPU). -- :func:`local_capability_report` — the data behind ``lodedb doctor``. +- :func:`local_capability_report`: the data behind ``lodedb doctor``. """ from lodedb.engine._filelock import ConcurrentWriterError diff --git a/src/lodedb/local/backends.py b/src/lodedb/local/backends.py index 428d1574..7be89b32 100644 --- a/src/lodedb/local/backends.py +++ b/src/lodedb/local/backends.py @@ -212,7 +212,7 @@ def _preferred_onnx_providers(device: str) -> tuple[str, ...]: CUDA hosts prefer the CUDA provider. The Apple **Core ML** provider is **off by default**: on the dynamic-shape preset graphs it fragments into many Core ML/CPU partitions and measured *slower* than the plain CPU provider for single-query embedding (about 16 ms vs 3 ms on an - M-series CPU), so it is opt-in via ``LODEDB_ONNX_COREML=1`` — the same stance the repo takes on + M-series CPU), so it is opt-in via ``LODEDB_ONNX_COREML=1``, the same stance the repo takes on the MPS vector scan, which is also off by default because NEON was faster. Everything else (CPU, and MPS without the opt-in) uses the CPU provider, which is the fast path for these models. """ diff --git a/src/lodedb/local/benchmark.py b/src/lodedb/local/benchmark.py index 4b109314..38eb2bd9 100644 --- a/src/lodedb/local/benchmark.py +++ b/src/lodedb/local/benchmark.py @@ -5,8 +5,8 @@ This benchmark therefore reports two distinct numbers and makes no GPU-vector-search-on-Mac claim: -1. **Embedding throughput** — docs/sec for the selected backend/device. -2. **CPU vector-scan latency** — per-query p50/p95 *search* latency (the native +1. **Embedding throughput**: docs/sec for the selected backend/device. +2. **CPU vector-scan latency**: per-query p50/p95 *search* latency (the native TurboVec scan + result assembly), measured with embedding excluded by timing the query embedding and the native vector scan separately. diff --git a/src/lodedb/local/cli.py b/src/lodedb/local/cli.py index ae85a638..5b887648 100644 --- a/src/lodedb/local/cli.py +++ b/src/lodedb/local/cli.py @@ -1,6 +1,6 @@ """`lodedb` CLI: serve | index | query | benchmark | doctor. -Thin wrapper over the local SDK (:class:`LodeDB`) and the existing engine — no +Thin wrapper over the local SDK (:class:`LodeDB`) and the existing engine; no retrieval/storage logic is duplicated here. ``doctor`` reuses :func:`local_capability_report`; ``index``/``query``/``benchmark`` drive ``LodeDB``; ``serve`` runs a minimal local HTTP loop over the same in-process @@ -351,7 +351,7 @@ def serve( "cloud", context_settings={ # Everything after `lodedb cloud` belongs to the cloud CLI, including - # --help and unknown options — forward it all untouched. + # --help and unknown options. Forward it all untouched. "allow_extra_args": True, "ignore_unknown_options": True, "help_option_names": [], @@ -463,7 +463,7 @@ def mcp_install( ) -> None: """Register the LodeDB MCP server with a coding assistant in one step. - Writes the correct ``command``/``args`` entry for ``--client`` to that host's config — + Writes the correct ``command``/``args`` entry for ``--client`` to that host's config, even when ``lodedb`` is not on ``PATH`` (it falls back to the ``uv run --project`` form, then an absolute path to the entry point). The edit is idempotent (an existing ``lodedb`` entry is updated, never duplicated) and leaves other servers untouched. For diff --git a/src/lodedb/local/db.py b/src/lodedb/local/db.py index f0b08044..7ef6fc04 100644 --- a/src/lodedb/local/db.py +++ b/src/lodedb/local/db.py @@ -1,10 +1,10 @@ -"""LodeDB — the local-first, embedded, no-auth vector database SDK. +"""LodeDB is the local-first, embedded, no-auth vector database SDK. ``LodeDB`` runs the engine in-process in a local profile that: - binds the embedded SDK to loopback, while `lodedb serve` may intentionally bind a private-network address for trusted-LAN demos; -- requires no authentication — the user never supplies or sees any credential; +- requires no authentication (the user never supplies or sees any credential); - keeps telemetry metrics-only (counts / bytes / latency, never payloads); - stores vectors in the compact TurboVec format and commits every change atomically via a per-index root manifest (``.commit.json``) over @@ -71,7 +71,7 @@ ) # Fixed local identifier for the single-process client context. It never leaves -# the process and has no security meaning — the local engine is auth-free. +# the process and has no security meaning; the local engine is auth-free. _LOCAL_CLIENT_ID = "lodedb-local" _LOCAL_CLIENT_ID_HASH = sha256_text(_LOCAL_CLIENT_ID) # The local DB is a single, stable index. Pinning it to the engine's @@ -234,8 +234,8 @@ class LodeDB: For config-driven code, where one string field (an env var, a YAML value) must express either a local path or a cloud store, the constructor itself - accepts the explicit URL form — ``LodeDB("orecloud://org/environment/store")`` - — and returns the same handle. Either way the handle duck-types this + accepts the explicit URL form (``LodeDB("orecloud://org/environment/store")``) + and returns the same handle. Either way the handle duck-types this class's read/write surface (``add``, ``search``, ``get``, ``remove``, ...) but runs over HTTPS, so local-only construction options (``model=``, ``store_text=``, ...) don't apply and local-only verbs (``persist``, @@ -260,18 +260,18 @@ def __new__(cls, path: str | Path | None = None, **kwargs: Any) -> Any: @classmethod def cloud(cls, target: str, **options: Any) -> Any: - """Opens a managed OreCloud store — the cloud counterpart of + """Opens a managed OreCloud store, the cloud counterpart of ``LodeDB(path)``, joining :meth:`open_readonly` / :meth:`open_vector_store` in the alternate-constructor family. - ``target`` is a bare store id (``"user-42"`` — org/environment + ``target`` is a bare store id (``"user-42"``; org/environment resolve from the environment-scoped credential, exactly like ``orecloud.Client().store()``), an ``"org/environment/store"`` triple for cross-environment scripts, or a full ``orecloud://`` URL. Keyword options are ``orecloud.connect``'s (``token=``, ``host=``, ``key=``, ``warm=``, ``timeout=``, ``read_your_writes=``, - ``transport=``). Returns the companion's store handle — duck-typed - to this class's verb surface, not a ``LodeDB`` instance — through + ``transport=``). Returns the companion's store handle (duck-typed + to this class's verb surface, not a ``LodeDB`` instance) through the same funnel as the ``LodeDB("orecloud://…")`` config-string dispatch. Without the ``lodedb[cloud]`` extra installed this raises an ``ImportError`` carrying the install hint; the companion is @@ -360,7 +360,7 @@ def __init__( next open (a half-written trailing record is discarded), and a clean ``close``/``persist`` folds it into a generation. WAL mode drops the lock-free concurrent-reader snapshot that ``open_readonly`` relies on for - *uncheckpointed* writes — it is meant for single-process deployments — + *uncheckpointed* writes (it is meant for single-process deployments), but the on-disk generation a reader sees is always a consistent committed one. Pass ``"generation"`` for the historical path where every change publishes a new crash-atomic, lock-free MVCC-readable generation. Unset @@ -369,7 +369,7 @@ def __init__( ``True``: the original text passed to ``add``/``add_many`` is kept in a dedicated on-disk sidecar so ``get``/``get_text``/``get_texts`` can return it (across reopens too). Pass ``store_text=False`` to opt out of retaining - text at all — telemetry, audit, and the redacted snapshot never carry text + text at all. Telemetry, audit, and the redacted snapshot never carry text regardless of this flag. Reopen the same path with the same ``store_text`` value you wrote with. ``index_text`` controls durable lexical-index persistence and **defaults to @@ -676,11 +676,11 @@ def open_vector_store( Sugar for ``LodeDB(path, vector_dim=vector_dim, bit_width=bit_width, ...)``. The index has **no internal embedding model**: the vector-in verbs (``add_vectors`` / ``add_vectors_many`` / ``search_by_vector`` / - ``search_many_by_vector``) work, and so does keyword search — + ``search_many_by_vector``) work, and so does keyword search. ``search(query, mode="lexical")`` (and ``search_many``) runs BM25 over the text carried on the stored vectors, since lexical ranking needs no embedder (requires ``store_text=True``, the default, or ``index_text=True``). - The embedding verbs — ``add`` / ``add_many`` and vector/hybrid ``search`` — + The embedding verbs (``add`` / ``add_many`` and vector/hybrid ``search``) raise :class:`VectorOnlyIndexError` because they would need to embed text. Vectors must have dimension ``vector_dim`` (any value your own embedder produces, e.g. 1536 or 3072), so this is the @@ -809,7 +809,7 @@ def add_vectors( The *vector-in* counterpart to :meth:`add`: the caller supplies the embedding directly (e.g. from their own model), and LodeDB stores it - verbatim without embedding or chunking text — so this is how an + verbatim without embedding or chunking text. This is how an external system (or a graph layer that embeds once) reuses its own vectors. The vector must have the index's embedding dimension (``minilm`` -> 384, ``bge`` -> 768; see :attr:`preset`). It is @@ -906,14 +906,14 @@ def search( wherever it can run while a vector-only store still searches without raising. Pass an explicit mode to override: - - ``"vector"`` — embedding cosine similarity only. - - ``"hybrid"`` — runs a lexical BM25 ranker alongside the vector scan and + - ``"vector"``: embedding cosine similarity only. + - ``"hybrid"``: runs a lexical BM25 ranker alongside the vector scan and fuses the two ranked lists with Reciprocal Rank Fusion, so exact tokens that the embedding misses (error codes like ``E1234``, serials like ``ABC-123``, dates like ``2024-01-15``) are surfaced when they appear in the document body. The default for local RAG, where a missed exact match is the difference between a usable and a useless answer. - - ``"lexical"`` — the BM25 ranking alone (no vector scan). Because it embeds + - ``"lexical"``: the BM25 ranking alone (no vector scan). Because it embeds nothing, ``mode="lexical"`` also works on a **vector-only** index (:meth:`open_vector_store`) that retains text: it ranks the text carried on the stored vectors. On such an index ``vector``/``hybrid`` still raise @@ -949,8 +949,8 @@ def search( predicate: - comparison ``$eq`` ``$ne`` ``$gt`` ``$gte`` ``$lt`` ``$lte`` ``$in`` - ``$nin`` ``$exists`` — e.g. ``{"year": {"$gte": 2020, "$lt": 2025}}``; - - composition ``$and`` / ``$or`` / ``$not`` (nestable) — e.g. + ``$nin`` ``$exists``, e.g. ``{"year": {"$gte": 2020, "$lt": 2025}}``; + - composition ``$and`` / ``$or`` / ``$not`` (nestable), e.g. ``{"$or": [{"topic": "ml"}, {"year": {"$gte": 2023}}]}``. A bare scalar is exact-match sugar for ``$eq``, so existing filters are @@ -958,7 +958,7 @@ def search( (``$gt``/``$gte``/``$lt``/``$lte``) compare numerically only when both the stored value and the operand parse as finite numbers (otherwise lexicographically), whereas ``$eq``/``$ne``/``$in``/``$nin`` always compare - as strings — so ``{"price": {"$eq": 9.9}}`` does not match a stored ``9.90`` + as strings, so ``{"price": {"$eq": 9.9}}`` does not match a stored ``9.90`` while ``{"price": {"$gte": 9.9, "$lte": 9.9}}`` does. ``$ne``/``$nin`` and ``$exists: False`` also match documents missing the field. Raw query text never leaves the process and never appears in telemetry. @@ -969,7 +969,7 @@ def search( if k <= 0: raise ValueError("k must be positive") # Gates on the resolved mode, not on text-capability alone: a vector-only - # handle can still run mode="lexical" (BM25 over retained text) — only + # handle can still run mode="lexical" (BM25 over retained text). Only # vector/hybrid, which embed the query, raise VectorOnlyIndexError here. resolved_mode = self._resolve_mode(mode) normalized_filter = _normalize_filter(filter) @@ -1328,7 +1328,7 @@ def get(self, id: str) -> str | None: This is the primary retrieval verb; :meth:`get_text` is a synonym. As a deliberate counterpart to ``add``, ``get(hit.id)`` is how an application - recovers the original text for a document a search selected — search hits + recovers the original text for a document a search selected; search hits themselves stay payload-free. Available unless the DB was opened with ``store_text=False`` (see :meth:`get_text` for the exact semantics). """ @@ -1377,8 +1377,8 @@ def get_texts(self, ids: list[str]) -> dict[str, str]: def get_document(self, id: str) -> dict[str, Any] | None: """Returns one document's redacted record by id, or ``None`` if absent. - The record is payload-free — ``{"id", "metadata", "chunk_count", - "content_hash"}`` — with **no** text and **no** vectors. This is the + The record is payload-free (``{"id", "metadata", "chunk_count", + "content_hash"}``), with **no** text and **no** vectors. This is the by-id metadata read a graph / knowledge-graph layer uses to resolve an edge's endpoints (or a node's attributes) without issuing a similarity search; use :meth:`get`/:meth:`get_text` to recover the raw text. @@ -1400,7 +1400,7 @@ def list_documents( """Returns document records, optionally filtered and paged by a keyset cursor. Unlike :meth:`search`, this is enumeration, not ranking: it returns matching - documents with no query vector and no scoring — the primitive a graph / + documents with no query vector and no scoring, the primitive a graph / knowledge-graph layer needs for deterministic traversal ("all edges whose ``src`` is X", "all nodes of ``type`` Person"). Each record is the payload-free ``{"id", "metadata", "chunk_count", "content_hash"}``. @@ -2282,7 +2282,7 @@ def _resolve_mode(self, mode: str | None) -> str: ``"vector"`` otherwise, so the fused ranking is the default wherever it can run. On a **vector-only** handle (no embedding model) an unset mode resolves to ``"lexical"`` instead: such a handle cannot embed the query - text, so a text query can only be a keyword (BM25) query — and the + text, so a text query can only be a keyword (BM25) query, and the lexical-source check below then applies. Returns the canonical lowercase mode. Raises: @@ -2293,7 +2293,7 @@ def _resolve_mode(self, mode: str | None) -> str: index cannot do (use :meth:`search_by_vector` for vector similarity, or ``mode="lexical"`` for keyword search). - :class:`ValueError` when a lexical/hybrid mode is requested on a handle - with no lexical source — neither ``index_text=True`` (a persisted BM25 + with no lexical source, neither ``index_text=True`` (a persisted BM25 postings store) nor ``store_text=True`` (the BM25 index rebuilt from the retained raw text), so there is nothing to build the index from. """ @@ -2554,7 +2554,7 @@ def _public_document_record(record: Mapping[str, Any]) -> dict[str, Any]: The engine returns ``document_id``; the public SDK uses ``id`` everywhere (``add(id=...)``, ``LodeSearchHit.id``, ``get(id)``), so the key is renamed - for consistency. Only redacted fields are surfaced — never text or vectors. + for consistency. Only redacted fields are surfaced, never text or vectors. """ metadata = record.get("metadata", {}) diff --git a/src/lodedb/local/integrations/__init__.py b/src/lodedb/local/integrations/__init__.py index 28e9d7f7..e443ea12 100644 --- a/src/lodedb/local/integrations/__init__.py +++ b/src/lodedb/local/integrations/__init__.py @@ -3,26 +3,26 @@ Each adapter is import-guarded behind its framework's optional extra, so this package imports cleanly without those heavy deps installed: -- ``langchain`` — ``langchain.LodeDBVectorStore`` (``pip install 'lodedb[langchain]'``). -- ``llama-index`` — ``llama_index.LodeDBVectorStore`` and +- ``langchain``: ``langchain.LodeDBVectorStore`` (``pip install 'lodedb[langchain]'``). +- ``llama-index``: ``llama_index.LodeDBVectorStore`` and ``llama_index_graph.LodeDBPropertyGraphStore`` (``pip install 'lodedb[llama-index]'``). -- ``mem0`` — ``mem0.LodeDBVectorStore`` (``pip install 'lodedb[mem0]'``). -- ``cognee`` — ``cognee.CogneeLodeDBAdapter`` + ``cognee.register_cognee_adapter`` +- ``mem0``: ``mem0.LodeDBVectorStore`` (``pip install 'lodedb[mem0]'``). +- ``cognee``: ``cognee.CogneeLodeDBAdapter`` + ``cognee.register_cognee_adapter`` (``pip install 'lodedb[cognee]'``). -- ``privategpt`` — ``privategpt.register_lodedb_provider`` registers the LlamaIndex adapter as a +- ``privategpt``: ``privategpt.register_lodedb_provider`` registers the LlamaIndex adapter as a PrivateGPT vector-store provider (needs the ``llama-index`` extra inside a PrivateGPT environment). It is a provider shim, not a new adapter: PrivateGPT's store layer *is* LlamaIndex's ``BasePydanticVectorStore``, which the LlamaIndex adapter already implements. -- ``kotaemon`` — ``kotaemon.LodeDBVectorStore`` implements kotaemon's ``BaseVectorStore`` +- ``kotaemon``: ``kotaemon.LodeDBVectorStore`` implements kotaemon's ``BaseVectorStore`` contract by duck typing (no extra needed; the module imports neither kotaemon nor llama-index). Selected from kotaemon via ``KH_VECTORSTORE`` in ``flowsettings.py``. -All wrap a LodeDB handle. The LangChain and LlamaIndex adapters are text-path — LodeDB embeds +All wrap a LodeDB handle. The LangChain and LlamaIndex adapters are text-path. LodeDB embeds text internally (``is_embedding_query=False``) and the framework's own embedding model is not used. The mem0 adapter is vector-in: mem0 owns the embeddings, LodeDB stores and searches them, and the full mem0 payload JSON is retained in LodeDB's raw-text sidecar (never in redacted -metadata). The cognee adapter (``CogneeLodeDBAdapter``) is vector-in the same way — cognee owns -the embeddings via its ``EmbeddingEngine`` — with one LodeDB index per cognee collection, the +metadata). The cognee adapter (``CogneeLodeDBAdapter``) is vector-in the same way (cognee owns +the embeddings via its ``EmbeddingEngine``), with one LodeDB index per cognee collection, the serialized DataPoint payload in the raw-text sidecar, and ``belongs_to_set`` membership stored as scalar presence keys so cognee's ``node_name`` filtering pushes into the metadata planner. The ``LodeDBPropertyGraphStore`` wraps :class:`lodedb.graph.KnowledgeGraph` instead diff --git a/src/lodedb/local/integrations/kotaemon.py b/src/lodedb/local/integrations/kotaemon.py index d06102b0..d2752428 100644 --- a/src/lodedb/local/integrations/kotaemon.py +++ b/src/lodedb/local/integrations/kotaemon.py @@ -21,8 +21,8 @@ "path": str(KH_USER_DATA_DIR / "vectorstore"), } -kotaemon does not configure an embedding dimension anywhere — the store meets -the dimension of whatever embedding model the user selected at runtime — so the +kotaemon does not configure an embedding dimension anywhere (the store meets +the dimension of whatever embedding model the user selected at runtime), so the LodeDB index is created lazily on the first ``add`` and its shape is recorded in a small ``kotaemon_store.json`` sidecar for reopens. LodeDB indexes require a dimension that is a multiple of 8, so vectors at any other dimension are @@ -302,7 +302,7 @@ def query( def drop(self) -> None: """Deletes the entire collection from disk. - Removes ``/`` — but only when this adapter's shape + Removes ``/``, but only when this adapter's shape sidecar is present, so a misconfigured path can never delete a directory the adapter did not create. A collection that was never created is a no-op. @@ -461,7 +461,7 @@ def _write_meta(self) -> None: A later open depends on this sidecar to know the collection's shape, so it follows the same durability mode as the LodeDB store it describes (the adapter opens the store without an explicit ``durability=``, which - resolves from ``LODEDB_DURABILITY`` — mirror that here). + resolves from ``LODEDB_DURABILITY``; mirror that here). """ payload = json.dumps( @@ -501,7 +501,7 @@ def _validated_collection_name(collection_name: str) -> str: The collection directory is joined under the configured base ``path`` and :meth:`LodeDBVectorStore.drop` deletes it recursively, so a name carrying - path separators, ``..``, or an absolute prefix must never reach that join — + path separators, ``..``, or an absolute prefix must never reach that join; it would let a config value write to (and drop) a directory outside the vector-store root. """ diff --git a/src/lodedb/local/integrations/langchain.py b/src/lodedb/local/integrations/langchain.py index 96f2804a..68b73cb9 100644 --- a/src/lodedb/local/integrations/langchain.py +++ b/src/lodedb/local/integrations/langchain.py @@ -4,13 +4,13 @@ can drop in the local-first store. LodeDB embeds internally (via its preset), so no LangChain ``Embeddings`` object is needed. -**Design note — page_content.** LodeDB's *redacted* artifacts (snapshot, journal, +**Design note: page_content.** LodeDB's *redacted* artifacts (snapshot, journal, telemetry, audit) never carry raw text, but the original text is retained by default in a separate ``.tvtext`` sidecar (``store_text=True``). This adapter fills LangChain ``Document.page_content`` from a **session-local** ``id -> text`` cache for documents added in the current process, and falls back to ``db.get_text(id)`` for documents from a prior session whenever the underlying LodeDB retains text. If the store was opened with -``store_text=False``, no text is kept and ``page_content`` is empty across a reopen — keep +``store_text=False``, no text is kept and ``page_content`` is empty across a reopen; keep your own text store keyed by the returned id/metadata in that case. Retrieval (ids, scores, metadata) is always durable. """ diff --git a/src/lodedb/local/integrations/llama_index.py b/src/lodedb/local/integrations/llama_index.py index a19f0631..bc110c30 100644 --- a/src/lodedb/local/integrations/llama_index.py +++ b/src/lodedb/local/integrations/llama_index.py @@ -9,7 +9,7 @@ LodeDB, which does the embedding. **LlamaIndex's own ``embed_model`` is therefore ignored for indexing and querying.** If you build the index through :class:`VectorStoreIndex`, set a cheap ``embed_model`` (e.g. ``MockEmbedding``) so LlamaIndex does not try to embed nodes with a -remote model — the vectors it computes are discarded; LodeDB re-embeds the text. (The SDK does +remote model. The vectors it computes are discarded; LodeDB re-embeds the text. (The SDK does have a vector-in path, but that is a separate *vector-only* index that owns no embedder and is out of scope for this text-path adapter; mixing externally embedded vectors with LodeDB's own embeddings in one index makes similarity scores meaningless.) diff --git a/src/lodedb/local/integrations/llama_index_graph.py b/src/lodedb/local/integrations/llama_index_graph.py index 4c3dee47..da6a35c3 100644 --- a/src/lodedb/local/integrations/llama_index_graph.py +++ b/src/lodedb/local/integrations/llama_index_graph.py @@ -3,7 +3,7 @@ Wraps :class:`lodedb.graph.KnowledgeGraph` as a ``llama_index.core.graph_stores.types.PropertyGraphStore`` so LlamaIndex's ``PropertyGraphIndex`` can use LodeDB's hybrid graph layer (SQLite topology + LodeDB -semantic index) as its graph store — the graph counterpart to the +semantic index) as its graph store, the graph counterpart to the :class:`~lodedb.local.integrations.llama_index.LodeDBVectorStore` vector-store adapter. **Node mapping.** A LlamaIndex :class:`EntityNode` maps to a graph node typed by its @@ -22,7 +22,7 @@ ``LodeDBPropertyGraphStore.from_path(path, vector_dim=D)``) to bring your own embeddings: then :meth:`upsert_nodes` stores each node's ``embedding`` directly at dimension ``D`` and :meth:`vector_query` searches by a precomputed query vector. This is the mode that makes -LlamaIndex's high-level ``PropertyGraphIndex`` work with **any** ``embed_model`` — set ``D`` to +LlamaIndex's high-level ``PropertyGraphIndex`` work with **any** ``embed_model``. Set ``D`` to the embedder's dimension (e.g. 1536) so node and query embeddings stay in the same space. **Semantic queries.** ``supports_vector_queries`` is True. In text-path mode @@ -273,7 +273,7 @@ def get_rel_map( """Returns the depth-bounded triplet neighbourhood around ``graph_nodes``. Breadth-first over the topology from the seed nodes, following the target side at each - hop, deduping triplets, dropping ``ignore_rels``, and capping at ``limit`` — the same + hop, deduping triplets, dropping ``ignore_rels``, and capping at ``limit``, the same expansion shape as LlamaIndex's reference store. """ diff --git a/src/lodedb/local/integrations/privategpt.py b/src/lodedb/local/integrations/privategpt.py index 5601dd2d..e9451e63 100644 --- a/src/lodedb/local/integrations/privategpt.py +++ b/src/lodedb/local/integrations/privategpt.py @@ -4,7 +4,7 @@ layer is built on **LlamaIndex's** ``BasePydanticVectorStore``, selected at startup by the ``vectorstore.database`` key in ``settings.yaml``. Because LodeDB already ships a LlamaIndex ``BasePydanticVectorStore`` (:class:`lodedb.local.integrations.llama_index.LodeDBVectorStore`, -the ``lodedb[llama-index]`` extra), wiring PrivateGPT up is **not a new adapter** — it is a thin +the ``lodedb[llama-index]`` extra), wiring PrivateGPT up is **not a new adapter**; it is a thin provider shim plus one line of registration. PrivateGPT's selection mechanism (``private_gpt/components/vector_store/``) is a small factory @@ -19,10 +19,10 @@ This module supplies that provider: -- :class:`LodeDBVectorStoreFactory` — a ``VectorStoreFactory`` that reads PrivateGPT's settings +- :class:`LodeDBVectorStoreFactory`: a ``VectorStoreFactory`` that reads PrivateGPT's settings (the embedding ``embed_dim``, the requested ``collection``, and an optional ``lodedb:`` block) and builds a :class:`LodeDBVectorStore`, one local on-disk index per collection. -- :func:`register_lodedb_provider` — registers the factory under ``"lodedb"`` so PrivateGPT can +- :func:`register_lodedb_provider`: registers the factory under ``"lodedb"`` so PrivateGPT can select it. Call it once before PrivateGPT builds its ``VectorStoreComponent``. **Text-path, like the LlamaIndex adapter.** LodeDB embeds text internally with the model picked @@ -34,8 +34,8 @@ **What lives where.** Everything LodeDB-specific (this shim, the settings keys it reads, the registration call) lives here. The only thing that must happen inside PrivateGPT's own process -is *triggering* the registration — ``_PROVIDERS`` is process-local with no entry-point -auto-discovery — which is a one-line import (or a two-line custom launcher); see +is *triggering* the registration (``_PROVIDERS`` is process-local with no entry-point +auto-discovery), which is a one-line import (or a two-line custom launcher); see ``examples/privategpt_provider.py`` and ``docs/integrations.md``. """ diff --git a/src/lodedb/local/mcp_install.py b/src/lodedb/local/mcp_install.py index db406f84..e378b046 100644 --- a/src/lodedb/local/mcp_install.py +++ b/src/lodedb/local/mcp_install.py @@ -4,7 +4,7 @@ server is otherwise a manual, per-host step: each client keeps its config in a different place and format (``claude_desktop_config.json``, ``.cursor/mcp.json``, LM Studio's ``mcp.json``, ``~/.codex/config.toml``), and getting ``command`` right -is error-prone when ``lodedb`` is not on ``PATH`` — the common case inside a +is error-prone when ``lodedb`` is not on ``PATH``, the common case inside a ``uv``/virtualenv install. This module resolves the right invocation for the current environment and writes (or removes) the ``lodedb`` entry for a chosen client. @@ -105,13 +105,13 @@ def resolve_server_invocation(*, prefer_uv: bool = False) -> ServerInvocation: Order, picking the first that applies: - 1. ``lodedb`` on ``PATH`` — the cleanest entry (``command="lodedb"``). Skipped when + 1. ``lodedb`` on ``PATH``, the cleanest entry (``command="lodedb"``). Skipped when ``prefer_uv`` is set. - 2. A source checkout with ``uv`` available — ``uv run --project lodedb`` so the + 2. A source checkout with ``uv`` available: ``uv run --project lodedb`` so the entry resolves inside the project's environment without ``PATH``. 3. The absolute path to the installed ``lodedb`` entry-point script (e.g. the venv's ``bin/lodedb``). - 4. Last resort — `` -m lodedb`` (``lodedb.__main__`` runs the CLI). + 4. Last resort: `` -m lodedb`` (``lodedb.__main__`` runs the CLI). ``prefer_uv`` forces the ``uv run`` form when in a checkout (useful when the on-``PATH`` ``lodedb`` belongs to a *different* environment than the one being configured). @@ -200,7 +200,7 @@ def _home() -> Path: def _claude_desktop_config_path() -> Path: """Returns Claude Desktop's ``claude_desktop_config.json`` for this OS. - macOS / Windows only — Claude Desktop has no official Linux build, so Linux raises a + macOS / Windows only. Claude Desktop has no official Linux build, so Linux raises a clear :class:`MCPInstallError` (use ``--config`` to override if you run a community build). """ diff --git a/src/lodedb/local/mcp_server.py b/src/lodedb/local/mcp_server.py index 1375e48f..4a7e6e46 100644 --- a/src/lodedb/local/mcp_server.py +++ b/src/lodedb/local/mcp_server.py @@ -79,7 +79,7 @@ def _remove(db: LodeDB, id: str) -> dict[str, Any]: def _stats(db: LodeDB) -> dict[str, Any]: - """Returns redacted store stats (counts, storage bytes) — never document text.""" + """Returns redacted store stats (counts, storage bytes), never document text.""" return db.stats() @@ -200,7 +200,7 @@ def lodedb_remove(id: str) -> dict[str, Any]: @server.tool() def lodedb_stats() -> dict[str, Any]: - """Return redacted store stats (counts, storage bytes) — never raw text.""" + """Return redacted store stats (counts, storage bytes), never raw text.""" return _stats(db) diff --git a/src/lodedb/local/migrate/detect.py b/src/lodedb/local/migrate/detect.py index ffbb5e27..dba6c347 100644 --- a/src/lodedb/local/migrate/detect.py +++ b/src/lodedb/local/migrate/detect.py @@ -3,13 +3,13 @@ The migrator's first step is to look at a project checkout and decide, without running it, which migration path applies. Detection reads three kinds of signal: -- **Dependency files** — ``pyproject.toml`` / ``requirements*.txt`` / ``setup.py`` / +- **Dependency files**: ``pyproject.toml`` / ``requirements*.txt`` / ``setup.py`` / ``package.json`` tell us which package manager to use and which frameworks / providers are even installed. -- **Config and infra files** — Docker Compose, ``settings.yaml``, ``.env.example``, +- **Config and infra files**: Docker Compose, ``settings.yaml``, ``.env.example``, Alembic migrations, and raw SQL hint at a provider (a ``qdrant`` service, a ``CREATE EXTENSION vector``) without any Python import. -- **Python source (AST)** — imported names and a few constructor/call patterns tell +- **Python source (AST)**: imported names and a few constructor/call patterns tell us which framework *owns* the vector store and which direct provider is wired. The routing rule is the one both issues require: **framework detection wins over @@ -22,7 +22,7 @@ Detection never executes project code; it parses files. Reports stay payload-free: they carry detected names, counts of matching files, and the chosen install -command — never file contents. +command, never file contents. """ from __future__ import annotations diff --git a/src/lodedb/local/migrate/report.py b/src/lodedb/local/migrate/report.py index 64cccb13..6e6732b1 100644 --- a/src/lodedb/local/migrate/report.py +++ b/src/lodedb/local/migrate/report.py @@ -2,7 +2,7 @@ Every artifact the migrator emits (the plan, the ``migration.json`` manifest, any log line) must keep LodeDB's privacy boundary: counts, bytes, timings, ids or id -hashes, dimensions, versions, and warnings only — never raw documents, queries, +hashes, dimensions, versions, and warnings only, never raw documents, queries, vectors, embeddings, payloads, or credentials. The helpers here are the single place that knows how to redact a connection string and how to fingerprint ids and source locations without leaking their contents. @@ -57,7 +57,7 @@ def redact_connection_string(value: str) -> str: """Returns a credential-free, log-safe rendering of a connection string. A DSN such as ``postgresql://user:pw@db.example.com:5432/app`` becomes - ``postgresql://`` — enough to show the provider scheme without ever + ``postgresql://``, enough to show the provider scheme without ever surfacing the username, password, host, port, or database name. A bare path or name (no ``scheme://``) is treated as non-secret and returned unchanged. """ diff --git a/src/lodedb/local/migrate/runner.py b/src/lodedb/local/migrate/runner.py index ecd507ef..c1bc3d5b 100644 --- a/src/lodedb/local/migrate/runner.py +++ b/src/lodedb/local/migrate/runner.py @@ -12,7 +12,7 @@ store unless ``--overwrite-target`` (or ``--resume``) is set. - **Payload-free manifest.** It writes ``migration.json`` with counts, dimensions, fingerprints, options, skipped-row reasons (by id hash), validation results, - versions, and timestamps — never raw text, vectors, payloads, or credentials. + versions, and timestamps, never raw text, vectors, payloads, or credentials. The runner picks the *target writer* from the plan's route and mode: a framework route replays through that framework's shipped LodeDB adapter (so the on-disk shape diff --git a/src/lodedb/local/migrate/sources/__init__.py b/src/lodedb/local/migrate/sources/__init__.py index 3efcd902..e416fd56 100644 --- a/src/lodedb/local/migrate/sources/__init__.py +++ b/src/lodedb/local/migrate/sources/__init__.py @@ -3,10 +3,10 @@ Each module reads exactly one source store and streams a uniform :class:`~lodedb.local.migrate.sources.base.SourceExport`: -- ``langchain_inmemory`` — a LangChain ``InMemoryVectorStore`` dump (text-replay). -- ``llama_index_simple`` — a persisted LlamaIndex ``StorageContext`` (text-replay). -- ``mem0_qdrant`` — a mem0 Qdrant collection (vector-preserve). -- ``pgvector`` — a direct pgvector table (vector-preserve). +- ``langchain_inmemory``: a LangChain ``InMemoryVectorStore`` dump (text-replay). +- ``llama_index_simple``: a persisted LlamaIndex ``StorageContext`` (text-replay). +- ``mem0_qdrant``: a mem0 Qdrant collection (vector-preserve). +- ``pgvector``: a direct pgvector table (vector-preserve). Importers are read-only by construction; none issues a write, delete, drop, or schema change against a source. diff --git a/src/lodedb/local/migrate/sources/base.py b/src/lodedb/local/migrate/sources/base.py index 1fc48178..dfa6619e 100644 --- a/src/lodedb/local/migrate/sources/base.py +++ b/src/lodedb/local/migrate/sources/base.py @@ -7,8 +7,8 @@ Two replay modes mirror the shipped adapters: -- ``text-replay`` — the row carries canonical text + metadata; LodeDB embeds it. -- ``vector-preserve`` — the row carries the source's own embedding (and payload +- ``text-replay``: the row carries canonical text + metadata; LodeDB embeds it. +- ``vector-preserve``: the row carries the source's own embedding (and payload text/metadata); LodeDB stores the vector verbatim. Importers are read-only by construction: they open the source, iterate, and close. diff --git a/src/lodedb/local/onnx_artifacts.py b/src/lodedb/local/onnx_artifacts.py index 66fc406b..b51d3415 100644 --- a/src/lodedb/local/onnx_artifacts.py +++ b/src/lodedb/local/onnx_artifacts.py @@ -7,10 +7,10 @@ Resolution order for a model id: -1. **Cached** — a previous materialization under the cache root; no network. -2. **Prebuilt snapshot** — many Hugging Face repos already ship an ``onnx/`` +1. **Cached**: a previous materialization under the cache root; no network. +2. **Prebuilt snapshot**: many Hugging Face repos already ship an ``onnx/`` export; fetch it with ``huggingface_hub`` (no Optimum / torch needed). -3. **Optimum export** — fall back to exporting the feature-extraction graph with +3. **Optimum export**: fall back to exporting the feature-extraction graph with Optimum, run as a **subprocess** so Optimum/torch never import into this process (keeping a plain ``import lodedb`` lean). diff --git a/swift/LodeDBCore/Package.swift b/swift/LodeDBCore/Package.swift index ba383094..81f62d05 100644 --- a/swift/LodeDBCore/Package.swift +++ b/swift/LodeDBCore/Package.swift @@ -18,7 +18,7 @@ import Foundation // zip and records its `swift package compute-checksum` value, so consuming a // released build via SwiftPM means setting those two vars (or pinning a manifest // that hard-codes them). Turnkey tag consumption with no env vars is not wired -// yet — a fresh external checkout that sets neither will not find the local path. +// yet; a fresh external checkout that sets neither will not find the local path. let nativeCore: Target = { let env = ProcessInfo.processInfo.environment if let urlString = env["LODEDB_FFI_BINARY_URL"], diff --git a/swift/LodeDBCore/Sources/LodeDBCore/Appender.swift b/swift/LodeDBCore/Sources/LodeDBCore/Appender.swift index 3f45b30f..37af706c 100644 --- a/swift/LodeDBCore/Sources/LodeDBCore/Appender.swift +++ b/swift/LodeDBCore/Sources/LodeDBCore/Appender.swift @@ -25,7 +25,7 @@ public struct LodeAppendDocument: Sendable, Equatable { /// operated in WAL commit mode: the appender logs to the WAL, and only a WAL-mode /// writer replays it. A writer that opens the store in generation commit mode /// never replays the WAL, so records appended here would be acknowledged yet never -/// folded in — hence `open` rejects generation-mode options outright. Like +/// folded in, hence `open` rejects generation-mode options outright. Like /// `LodeDB`, a single instance is not thread-safe; serialize calls to it. /// /// On Windows the shared lock degrades to an exclusive hold, so appenders exclude diff --git a/swift/LodeDBCore/Sources/LodeDBCore/MetalVectorScanner.swift b/swift/LodeDBCore/Sources/LodeDBCore/MetalVectorScanner.swift index ad6ad000..f74de397 100644 --- a/swift/LodeDBCore/Sources/LodeDBCore/MetalVectorScanner.swift +++ b/swift/LodeDBCore/Sources/LodeDBCore/MetalVectorScanner.swift @@ -24,7 +24,7 @@ public enum ScanBackend: String, Sendable { /// TurboVec NEON scan over the quantized index, which on-device benchmarks show is /// faster than a GPU/MPS scan (and CUDA is irrelevant on Apple platforms). This /// scanner is for callers that hold raw f32 vectors and want a GPU-accelerated exact -/// scan — e.g. exact reranking of a candidate set, or scanning app-managed vectors — +/// scan (e.g. exact reranking of a candidate set, or scanning app-managed vectors) /// where the Metal path can win at large `count`. Accelerating the core's *quantized* /// scan with Metal is deferred pending device benchmarks that show NEON is the /// bottleneck. diff --git a/swift/LodeDBCore/Sources/LodeDBCore/NativeCoreBridge.swift b/swift/LodeDBCore/Sources/LodeDBCore/NativeCoreBridge.swift index ca10c611..abf0dc74 100644 --- a/swift/LodeDBCore/Sources/LodeDBCore/NativeCoreBridge.swift +++ b/swift/LodeDBCore/Sources/LodeDBCore/NativeCoreBridge.swift @@ -593,7 +593,7 @@ final class NativeCheckpointer { } } -/// Serde-shaped payload for `lodedb_engine_create_index_json` — a minimal +/// Serde-shaped payload for `lodedb_engine_create_index_json`, a minimal /// `CoreIndexCreateRequest`. Only the distinguishing fields cross the boundary; the /// core supplies the identity defaults (name, provider, task, route/storage /// profile, and index_key/client_id_hash from index_id). `nil` optionals (`model`, diff --git a/tests/test_cloud_agents_scaffold.py b/tests/test_cloud_agents_scaffold.py index 7b281fe6..ed545346 100644 --- a/tests/test_cloud_agents_scaffold.py +++ b/tests/test_cloud_agents_scaffold.py @@ -1,5 +1,5 @@ """`lodedb cloud init --agents` artifact generators: content correctness and the -leave-user-files-alone rules (no server needed — pure filesystem).""" +leave-user-files-alone rules (no server needed, pure filesystem).""" import json @@ -47,7 +47,7 @@ def test_existing_agents_md_gains_section_once(tmp_path): assert content.startswith("# My repo") assert AGENTS_SECTION_HEADING in content - # Second run: the section is the repo's to edit now — never regenerated. + # Second run: the section is the repo's to edit now, never regenerated. (tmp_path / "AGENTS.md").write_text(content.replace("lodedb cloud sync", "MY EDIT")) written, notes = scaffold_agent_artifacts(tmp_path, **LINK) assert "AGENTS.md" not in written diff --git a/tests/test_cloud_client_api.py b/tests/test_cloud_client_api.py index 2ee4a2b7..cd594b32 100644 --- a/tests/test_cloud_client_api.py +++ b/tests/test_cloud_client_api.py @@ -60,7 +60,7 @@ def test_push_is_redacted_by_default(committed_store, tmp_path): assert body["tvtext"] is None assert body["tvlex"] is None assert body["json"] is not None - # The source itself still has its text store — redaction is per-transfer. + # The source itself still has its text store; redaction is per-transfer. assert read_pointer_body(source, key)["tvtext"] is not None diff --git a/tests/test_cloud_connect_targets.py b/tests/test_cloud_connect_targets.py index 4e19d1ba..8565e528 100644 --- a/tests/test_cloud_connect_targets.py +++ b/tests/test_cloud_connect_targets.py @@ -1,7 +1,7 @@ """`connect()` target forms: the full `org/environment/store` triple (and its `orecloud://` spelling) parses locally with zero HTTP, while a bare store id (`user-42`, the form behind `LodeDB.cloud("user-42")`) resolves its -org/environment from the credential via token introspection — exactly like +org/environment from the credential via token introspection, exactly like `Client().store()`. HTTP is a `httpx.MockTransport`, so these run serverless; the introspection contract itself is covered in `server/tests`.""" @@ -63,7 +63,7 @@ def test_two_segment_bare_form_stays_malformed(): def test_bare_store_resolves_tenancy_from_the_credential(): """`connect("user-42")` introspects the token once and lands on the bound - org/environment — the seam `LodeDB.cloud("user-42")` rides.""" + org/environment, the seam `LodeDB.cloud("user-42")` rides.""" transport = _transport( { "/v1/tokens/self": { @@ -84,7 +84,7 @@ def test_bare_store_resolves_tenancy_from_the_credential(): def test_full_triple_makes_no_http_call(): - """A fully qualified target must not introspect — the transport rejects + """A fully qualified target must not introspect. The transport rejects every request, so connecting cold (warm=False) proves zero HTTP.""" store = connect( "acme/prod/user-42", @@ -102,7 +102,7 @@ def test_full_triple_makes_no_http_call(): def test_bare_store_with_an_unbound_multi_org_token_refuses_with_choices(): """A personal token spanning several orgs cannot pin a bare store id: connect surfaces resolve_tenancy's 422 naming the choices (and must not - leak its freshly opened pool — closing after is a no-op either way).""" + leak its freshly opened pool; closing after is a no-op either way).""" transport = _transport( { "/v1/tokens/self": {"kind": "personal", "scopes": [], "org": None, "environment": None}, diff --git a/tests/test_cloud_credentials.py b/tests/test_cloud_credentials.py index 69302bf2..d58391d3 100644 --- a/tests/test_cloud_credentials.py +++ b/tests/test_cloud_credentials.py @@ -36,7 +36,7 @@ def test_env_host_overrides_the_default(isolated_config, monkeypatch): def test_env_token_never_borrows_the_file_host(isolated_config, monkeypatch): """A CI token must not aim at whatever control plane a developer's stored - login points to — without ORECLOUD_HOST it targets the hosted default.""" + login points to; without ORECLOUD_HOST it targets the hosted default.""" _config.save_credentials("https://other.example", "ore_pat_stored") monkeypatch.setenv("ORECLOUD_TOKEN", "ore_pat_ci") creds = _config.load_credentials() @@ -113,7 +113,7 @@ def me(self): def test_default_host_is_public_on_the_cloud_module(): """`from lodedb.cloud import DEFAULT_HOST` works without the extra's - dependencies — servers rendering docs conditionally rely on it.""" + dependencies; servers rendering docs conditionally rely on it.""" from lodedb import cloud assert cloud.DEFAULT_HOST == _config.DEFAULT_HOST diff --git a/tests/test_cloud_env_file.py b/tests/test_cloud_env_file.py index 9e588644..879266c4 100644 --- a/tests/test_cloud_env_file.py +++ b/tests/test_cloud_env_file.py @@ -1,5 +1,5 @@ """`lodedb.cloud._env_file`: dotenv writing that never clobbers, plus the -gitignore guard — the pieces that keep `tokens mint --env-file` and +gitignore guard, the pieces that keep `tokens mint --env-file` and `init --agents` from leaking or destroying anything.""" import stat diff --git a/tests/test_cloud_import_root.py b/tests/test_cloud_import_root.py index e8516b5f..93d1a76f 100644 --- a/tests/test_cloud_import_root.py +++ b/tests/test_cloud_import_root.py @@ -13,7 +13,7 @@ import pytest # The tests below import (or monkeypatch into) the client modules, which -# pull httpx/pynacl — skip cleanly without the [cloud] extra installed. +# pull httpx/pynacl, so skip cleanly without the [cloud] extra installed. pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") @@ -40,7 +40,7 @@ def test_missing_cloud_deps_raise_install_hint(monkeypatch): # the extra's dependencies are installed in the dev venv. monkeypatch.setitem(sys.modules, "lodedb.cloud.client", None) with pytest.raises(ImportError, match=r'pip install "lodedb\[cloud\]"'): - cloud.Client # noqa: B018 — the attribute fetch is the behavior under test + cloud.Client # noqa: B018 # the attribute fetch is the behavior under test def test_unknown_names_stay_attribute_errors(): @@ -52,7 +52,7 @@ def test_unknown_names_stay_attribute_errors(): def test_underscore_probes_never_touch_the_lazy_table(monkeypatch): """Attribute machinery probes (`__path__`-style dunders) must resolve - locally — blocked submodules stay untouched.""" + locally; blocked submodules stay untouched.""" monkeypatch.setitem(sys.modules, "lodedb.cloud.client", None) monkeypatch.setitem(sys.modules, "lodedb.cloud.serving", None) with pytest.raises(AttributeError): diff --git a/tests/test_cloud_routing_hint.py b/tests/test_cloud_routing_hint.py index e6a5292c..32fc09d7 100644 --- a/tests/test_cloud_routing_hint.py +++ b/tests/test_cloud_routing_hint.py @@ -1,7 +1,7 @@ """The `X-Ore-Store` routing hint: data-plane calls stamp org/env/store so a store-sticky ingress can hash-route them for cache locality. Pure transport -concern — the server never reads the header, and calls without it stay -correct — so the tests assert wire shape only.""" +concern (the server never reads the header, and calls without it stay +correct), so the tests assert wire shape only.""" from __future__ import annotations @@ -47,7 +47,7 @@ def test_data_plane_verbs_carry_the_store_hint(capture): def test_warm_stats_carries_the_hint(capture): """The pre-hydration call must land on the same pod the hinted queries - will — a warm on one pod followed by queries on another would defeat it.""" + will. A warm on one pod followed by queries on another would defeat it.""" client, seen = capture client.serving_stats("acme", "prod", "user-42", warm=True) assert seen[0].headers["x-ore-store"] == "acme/prod/user-42" diff --git a/tests/test_cloud_store_verbs.py b/tests/test_cloud_store_verbs.py index 5cd053fa..eb86fcd3 100644 --- a/tests/test_cloud_store_verbs.py +++ b/tests/test_cloud_store_verbs.py @@ -1,6 +1,6 @@ """CloudStore verb wiring against a stub transport: what payload each verb puts on the wire and how it folds the acceptance back into session state -(read-your-writes floor, `last_write_id`). No server involved — the accepted +(read-your-writes floor, `last_write_id`). No server involved; the accepted write contract itself is covered end-to-end in `server/tests`.""" from __future__ import annotations @@ -80,7 +80,7 @@ def browse_documents(self, org: str, environment: str, payload: dict) -> dict: def test_unprovisioned_batch_verbs_keep_query_cardinality(): """Both batched search verbs answer an unprovisioned store with one empty - hit list PER query — callers zip queries to results.""" + hit list PER query, so callers zip queries to results.""" store = CloudStore(_UnprovisionedClient(), "acme", "prod", "user-42", owns_client=False) assert store.search_many(["a", "b", "c"]) == [[], [], []] assert store.search_many_by_vector([[0.1, 0.2], [0.3, 0.4]]) == [[], []] @@ -117,7 +117,7 @@ def browse_documents(self, org: str, environment: str, payload: dict) -> dict: def test_browse_carries_the_session_floor_and_retries_425(): """Browse is a read like search: after a write on this handle it sends the session's read-your-writes floor as min_seq and briefly retries a - 425 instead of surfacing it — the write is durable, only its visibility + 425 instead of surfacing it; the write is durable, only its visibility trails by a fold cycle.""" client = _BrowseClient(too_early_first=True) store = _store(client) diff --git a/tests/test_cloud_surface_parity.py b/tests/test_cloud_surface_parity.py index c2718ac2..e79879d6 100644 --- a/tests/test_cloud_surface_parity.py +++ b/tests/test_cloud_surface_parity.py @@ -46,7 +46,7 @@ # Keyword-only params only the cloud has: write provenance and retention # (`ttl_seconds`/`agent_id`/`run_id`), inline text on hits (`include_text`), -# and serving warm-up (`warm`). All additive — code written against the +# and serving warm-up (`warm`). All additive; code written against the # shared shape runs unchanged. CLOUD_ONLY_KEYWORDS = {"ttl_seconds", "agent_id", "run_id", "include_text", "warm"} diff --git a/tests/test_cloud_sync.py b/tests/test_cloud_sync.py index 32193ac3..f440f01f 100644 --- a/tests/test_cloud_sync.py +++ b/tests/test_cloud_sync.py @@ -87,7 +87,7 @@ def test_diverged_requires_force_and_force_resolves(committed_store, tmp_path): # The clone advances once and publishes; the source advances twice on its # own lineage. (Different commit counts keep the two lineages on different - # base epochs — same-epoch artifact-name collisions across divergent + # base epochs; same-epoch artifact-name collisions across divergent # lineages are the fork-collision case the Phase-2 content-addressed # layout absorbs; a dumb remote fails them closed at the # immutable-artifact check.) @@ -98,7 +98,7 @@ def test_diverged_requires_force_and_force_resolves(committed_store, tmp_path): add_documents(source, docs("source-side-b")) # Diverged: sync refuses, names the classification and both force flags, - # and exits with the documented "refused" class (5) — the same class the + # and exits with the documented "refused" class (5), the same class the # managed sync's 409 maps to, not the generic 1. result = runner.invoke(app, ["sync", str(source), str(remote), key]) assert result.exit_code == 5 @@ -157,7 +157,7 @@ def test_sidecar_is_invisible_to_the_engine(committed_store, tmp_path): assert result.exit_code == 0 assert (source / f"{key}.orecloud").exists() - # Reopen through the engine and commit — this is exactly what broke when + # Reopen through the engine and commit. This is exactly what broke when # the sidecar was named `.orecloud.json`. add_documents(source, docs("after-sync")) (found_key,) = cloud.keys(str(source)) diff --git a/tests/test_cloud_target_dispatch.py b/tests/test_cloud_target_dispatch.py index d12097d0..18cc32f2 100644 --- a/tests/test_cloud_target_dispatch.py +++ b/tests/test_cloud_target_dispatch.py @@ -15,7 +15,7 @@ import pytest # The tests below import (or monkeypatch into) the client modules, which -# pull httpx/pynacl — skip cleanly without the [cloud] extra installed. +# pull httpx/pynacl, so skip cleanly without the [cloud] extra installed. pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") @@ -56,8 +56,8 @@ def test_missing_cloud_deps_raise_install_hint(monkeypatch): def test_cloud_target_returns_the_client_handle(monkeypatch): - """A cloud target hands back whatever the client's connect() opens — - LodeDB.__init__ never runs on it — with the options forwarded.""" + """A cloud target hands back whatever the client's connect() opens + (LodeDB.__init__ never runs on it), with the options forwarded.""" captured: dict[str, object] = {} handle = object() _fake_connect(monkeypatch, captured, handle) @@ -90,8 +90,8 @@ def test_local_only_options_are_rejected_up_front(monkeypatch): def test_cloud_classmethod_accepts_a_bare_store_id(monkeypatch): - """`LodeDB.cloud("user-42")` forwards the short target verbatim — the - client resolves org/environment from the credential — and returns the + """`LodeDB.cloud("user-42")` forwards the short target verbatim (the + client resolves org/environment from the credential) and returns the handle through the same funnel as the config-string form.""" captured: dict[str, object] = {} handle = object() @@ -129,7 +129,7 @@ def test_cloud_classmethod_rejects_local_only_options(monkeypatch): def test_bare_store_ids_do_not_dispatch_from_the_constructor(tmp_path, monkeypatch): """Only `LodeDB.cloud()` accepts scheme-less short forms: a bare id passed to the constructor is an ordinary (relative) local path, never a cloud - target — so the client's connect must not be consulted.""" + target, so the client's connect must not be consulted.""" def _exploding_connect(target, **options): raise AssertionError("the constructor must not dispatch scheme-less targets") diff --git a/tests/test_cloud_trampoline.py b/tests/test_cloud_trampoline.py index 783e1f1b..07aabc2c 100644 --- a/tests/test_cloud_trampoline.py +++ b/tests/test_cloud_trampoline.py @@ -1,6 +1,6 @@ """The `lodedb cloud` trampoline: forwards argv to the first-party cloud CLI (`lodedb.cloud.cli`, whose modules pull the [cloud] extra's dependencies), -and answers a clear install hint — not a traceback — when those dependencies +and answers a clear install hint, not a traceback, when those dependencies are absent. Both paths are forced deterministically, so the suite never depends on the venv's extras.""" @@ -15,7 +15,7 @@ from lodedb.local.cli import app # The forwarding test monkeypatches into `lodedb.cloud.cli`, which pulls -# httpx/pynacl — skip cleanly without the [cloud] extra installed (the hint +# httpx/pynacl, so skip cleanly without the [cloud] extra installed (the hint # path blocks the module itself, so it needs no deps either way). pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") @@ -43,7 +43,7 @@ def test_missing_cloud_deps_print_install_hint(monkeypatch): def test_argv_forwards_to_the_cloud_cli(monkeypatch): - """Everything after `lodedb cloud` reaches the cloud app untouched — + """Everything after `lodedb cloud` reaches the cloud app untouched: subcommand, positionals, and options the trampoline knows nothing about.""" captured: dict[str, object] = {} fake_app = typer.Typer() diff --git a/tests/test_import_boundary.py b/tests/test_import_boundary.py index 36dfbd54..8b29423d 100644 --- a/tests/test_import_boundary.py +++ b/tests/test_import_boundary.py @@ -12,7 +12,7 @@ which would otherwise have to be declared and shipped. 3. Built-in text embedding is opt-in (the ``[embeddings]`` / ``[torch]`` extras), so importing the package must not pull any embedding runtime - (onnxruntime / transformers / sentence-transformers / torch) — a base + (onnxruntime / transformers / sentence-transformers / torch); a base ``pip install lodedb`` is a vector store and must import cleanly without them. """ @@ -64,8 +64,8 @@ "psycopg", "psycopg2", "asyncpg", "qdrant_client", "chromadb", "lancedb", # The [cloud] extra's dependencies (httpx; pynacl imports as `nacl`): the # first-party cloud client (lodedb.cloud) reaches them only through its - # lazy PEP 562 exports and the CLI trampoline, so a plain import — even of - # lodedb.cloud itself — must stay network-free. + # lazy PEP 562 exports and the CLI trampoline, so a plain import, even of + # lodedb.cloud itself, must stay network-free. "httpx", "nacl", ) _loaded = {_name.split(".", 1)[0] for _name in sys.modules} @@ -76,8 +76,8 @@ # The optional image extra is Pillow (the "clip" preset / add_image path). The CLIP # backend imports sentence-transformers and Pillow only inside the methods that -# encode, so importing LodeDB — including the embedding-backends, presets, and -# backends modules where the CLIP wiring lives — must not pull Pillow. +# encode, so importing LodeDB, including the embedding-backends, presets, and +# backends modules where the CLIP wiring lives, must not pull Pillow. _IMAGE_EXTRA_LAZINESS_PROBE = """ import importlib, sys for _m in ( diff --git a/tests/test_kotaemon_adapter.py b/tests/test_kotaemon_adapter.py index ff02c312..c6712988 100644 --- a/tests/test_kotaemon_adapter.py +++ b/tests/test_kotaemon_adapter.py @@ -3,8 +3,8 @@ The adapter is duck-typed and dependency-free (it imports neither kotaemon nor llama-index), so these tests run in the base suite with no skip guard. They mirror the shapes of kotaemon's own ``test_vectorstore.py`` (add / add-from-docs -/ delete / query / persist-reopen / drop) — including its 3-dimensional toy -vectors, which exercise the zero-padding path — plus the retrieval-pipeline +/ delete / query / persist-reopen / drop), including its 3-dimensional toy +vectors, which exercise the zero-padding path, plus the retrieval-pipeline behaviors kotaemon relies on at runtime: the ``doc_ids`` chunk scope and duck-typed LlamaIndex ``MetadataFilters`` push-down. """ diff --git a/tests/test_local_backends.py b/tests/test_local_backends.py index d73a45af..3affc13a 100644 --- a/tests/test_local_backends.py +++ b/tests/test_local_backends.py @@ -85,7 +85,7 @@ def test_forced_onnx_runtime_errors_without_onnxruntime(monkeypatch): import lodedb.local.backends as backends monkeypatch.setattr(backends, "onnxruntime_available", lambda: False) - # A working torch tier must not mask the forced-onnx request — the hint still wins. + # A working torch tier must not mask the forced-onnx request; the hint still wins. monkeypatch.setattr(backends, "sentence_transformers_available", lambda: True) with pytest.raises(ModuleNotFoundError, match=r"lodedb\[embeddings\]"): build_local_embedding_backend( diff --git a/tests/test_local_cli_and_server.py b/tests/test_local_cli_and_server.py index 155c835f..1f36dfec 100644 --- a/tests/test_local_cli_and_server.py +++ b/tests/test_local_cli_and_server.py @@ -466,7 +466,7 @@ def test_cli_index_default_get_and_no_store_text_round_trip(tmp_path, monkeypatc """`lodedb index` retains text by default; `--no-store-text` opts out. The CLI builds its own LodeDB, so we inject the deterministic hash backend by - patching the symbol the CLI uses — keeping this test model-free like the rest + patching the symbol the CLI uses. This keeps the test model-free like the rest of the suite while exercising the real command wiring end to end. """ diff --git a/tests/test_local_enumeration.py b/tests/test_local_enumeration.py index 36e80070..857f93c3 100644 --- a/tests/test_local_enumeration.py +++ b/tests/test_local_enumeration.py @@ -2,7 +2,7 @@ ``LodeDB.list_documents()`` and ``get_document()`` expose the engine's payload-free document records on the SDK, with a ``filter=`` complete-set -enumeration (no ``k`` cap, no query vector) — the primitive a graph / +enumeration (no ``k`` cap, no query vector), the primitive a graph / knowledge-graph layer needs for deterministic traversal. """ diff --git a/tests/test_local_persistence_recovery.py b/tests/test_local_persistence_recovery.py index 556fa4d8..6820816d 100644 --- a/tests/test_local_persistence_recovery.py +++ b/tests/test_local_persistence_recovery.py @@ -2,7 +2,7 @@ LodeDB reuses the engine's ``.tvim``/``.tvd``/``.jsd`` persistence, which is documented to **fail closed** on restart when a sidecar is missing or corrupt for -a non-empty index — rather than silently returning an empty or partial index. +a non-empty index, rather than silently returning an empty or partial index. These tests corrupt/remove the on-disk sidecars and assert that reopen raises. """ diff --git a/tests/test_local_raw_text.py b/tests/test_local_raw_text.py index 50a9d663..09f8a361 100644 --- a/tests/test_local_raw_text.py +++ b/tests/test_local_raw_text.py @@ -4,7 +4,7 @@ base plus a ``.txd`` delta journal, so the original text is retrievable by id (``get`` / ``get_text`` / ``get_texts``), durably across reopens and committed O(changed) per write. The redacted artifacts stay payload-free -regardless — telemetry, audit, the ``.json`` snapshot, the ``.jsd`` journal, and +regardless: telemetry, audit, the ``.json`` snapshot, the ``.jsd`` journal, and the ``.tvim``/``.tvd`` vector sidecars never carry raw document text. Opening with ``store_text=False`` opts out of retaining text at all. diff --git a/tests/test_local_vector_in.py b/tests/test_local_vector_in.py index 53bdead8..6b5cf0a8 100644 --- a/tests/test_local_vector_in.py +++ b/tests/test_local_vector_in.py @@ -157,7 +157,7 @@ def test_mixed_text_and_vector_in_same_index(tmp_path): assert {r["id"] for r in db.list_documents()} == {"t1", "v1"} # Text search still works on the text doc. (HashEmbeddingBackend is a # non-semantic hash, so restrict to the text doc by filter to make ranking - # deterministic — the point here is coexistence, not retrieval quality.) + # deterministic; the point here is coexistence, not retrieval quality.) assert db.search("the quick brown fox", k=5, filter={"kind": "text"})[0].id == "t1" # vector search finds the vector doc assert db.search_by_vector(_onehot(0), k=1)[0].id == "v1" diff --git a/tests/test_segments.py b/tests/test_segments.py index 7c4e27c8..6bcedda0 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -210,7 +210,7 @@ def test_add_then_remove_in_one_fold_batch_stays_reopenable(tmp_path): """Regression: a document added and removed within one fold batch (one persist) wrote its never-committed row into the delta's removed set, and the strict replay rejected the store on every fresh open ("removed-id - count mismatch") — the warm handle kept serving while every new reader, + count mismatch"); the warm handle kept serving while every new reader, hydration, or pull failed. Found by randomized concurrency testing. The bug lives on the delta path, which only exists over a committed vector diff --git a/tests/test_wal_commit_mode.py b/tests/test_wal_commit_mode.py index dbcdd73b..f11c0b1e 100644 --- a/tests/test_wal_commit_mode.py +++ b/tests/test_wal_commit_mode.py @@ -5,7 +5,7 @@ write. It is crash-atomic: the WAL is replayed on open (a torn trailing record is discarded) and folded into a generation on a clean close. These tests cover the knob plumbing, equivalence with the classic generation mode, the checkpoint, -and crash recovery — including a hard ``os._exit`` kill of a writer mid-run. +and crash recovery, including a hard ``os._exit`` kill of a writer mid-run. """ from __future__ import annotations @@ -370,7 +370,7 @@ def _crash_writer(path_str: str) -> None: db.add("alpha durable", id="a") db.add("beta durable", id="b") db.add("gamma durable", id="c") - # Hard kill: bypass close(), so no checkpoint and no lock release run — this + # Hard kill: bypass close(), so no checkpoint and no lock release run. This # is the writer-killed-mid-commit case the WAL must survive. os._exit(0)