diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85aa0e99..b378e08e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,8 +40,8 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: astral-sh/setup-uv@v5 - - name: Sync (base runtime + dev/embeddings/torch/mcp/langchain/llama-index/mem0/image extras) - run: uv sync --extra dev --extra embeddings --extra torch --extra mcp --extra langchain --extra llama-index --extra mem0 --extra image + - name: Sync (base runtime + dev/embeddings/torch/mcp/langchain/llama-index/mem0/image/cloud extras) + run: uv sync --extra dev --extra embeddings --extra torch --extra mcp --extra langchain --extra llama-index --extra mem0 --extra image --extra cloud - name: Lint run: uv run ruff check . diff --git a/AGENTS.md b/AGENTS.md index 97eac7c3..f7ac1bc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,10 @@ tests/ local SDK suite + import-boundary guard or pass `embedder=`). Built-in text embedding is opt-in: `[embeddings]` (`onnxruntime` + `transformers`, torch-free) and `[torch]` (`sentence-transformers` — the PyTorch fallback, the `clip` preset, and `device="mps"`); other extras are `[image]`, `[onnx-export]`, `[mcp]`, - `[langchain]`, `[llama-index]`, `[mem0]`, `[gpu]`, and `[all]`. The patched TurboVec core is + `[langchain]`, `[llama-index]`, `[mem0]`, `[gpu]`, `[cloud]` (the first-party managed-cloud + client `lodedb.cloud` + the `lodedb cloud` CLI, over the bundled native transfer core; the + extra adds only `httpx` + `pynacl`, both reached lazily and never on a plain import — the + extra is deliberately NOT part of `[all]`), and `[all]`. The patched TurboVec core is vendored under `third_party/turbovec/` and bundled into the wheel as `lodedb._turbovec` — not a PyPI dependency. The embedding runtimes load lazily (only when a preset/CLIP backend is built), so a plain `import lodedb` must not import `onnxruntime` / `transformers` / @@ -84,7 +87,7 @@ tests/ local SDK suite + import-boundary guard ## Develop ```bash -uv sync --extra dev --extra embeddings --extra torch --extra mcp --extra langchain --extra llama-index --extra mem0 # build venv (compiles TurboVec) +uv sync --extra dev --extra embeddings --extra torch --extra mcp --extra langchain --extra llama-index --extra mem0 --extra cloud # build venv (compiles TurboVec) uv run pytest -q # run the suite uv run ruff check . # lint (line-length 100) ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fdf9fb2..88307b83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **The OreCloud managed-cloud client, first-party, as the `[cloud]` extra.** + `lodedb.cloud` ships in this package: `Client` (one credential, one bound + org/environment, per-user store handles), `connect`/`CloudStore` (one end + user's store over HTTPS, duck-typing the local verb surface), the transfer + verbs (`push`/`pull`/`sync`/`status`/`verify`/`keys`) on a native transfer + core bundled into the extension as `lodedb._turbovec.cloud` (one + commit-format implementation shared with the engine — the new + `crates/lodedb-cloud-core`, which also learns the 1.3.2 `tvvf` rescore + sidecar so rescore stores transfer completely), and the full `lodedb + cloud` CLI (login, tokens, store/environment/org management, transfer, + agent scaffolding). The extra installs only the client's dependencies + (`httpx`, `pynacl`); everything cloud resolves lazily, so a plain + `import lodedb` stays network-free (guarded by + `tests/test_import_boundary.py`). There is no separate cloud package. +- `LodeDB.cloud("user-42")` opens a managed-cloud store through the same + class as a local path, joining the `open_readonly`/`open_vector_store` + alternate-constructor family: a bare store id resolves its + org/environment from the credential (`token=`, + `ORECLOUD_TOKEN`/`ORECLOUD_HOST`, or `lodedb cloud login`), and the + `"org/environment/store"` triple and full `orecloud://` URLs are accepted + too. The call returns the cloud store handle (same + `add`/`search`/`get`/`remove` verbs over HTTPS). For config-driven code, + the plain constructor also dispatches the explicit URL form — + `LodeDB("orecloud://org/environment/store")` — through the same funnel. + Local-only construction options (`model=`, `read_only=`, ...) are rejected + on cloud targets with a targeted error, and a cloud target without the + `[cloud]` extra's dependencies raises the install hint. + ## [1.3.2] - 2026-07-16 ### Added diff --git a/Cargo.lock b/Cargo.lock index fd4a4c0f..0ca1cd0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "approx" version = "0.5.1" @@ -11,12 +20,35 @@ dependencies = [ "num-traits", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.13.0" @@ -32,6 +64,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytemuck" version = "1.25.0" @@ -58,6 +96,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cblas-sys" version = "0.1.4" @@ -85,12 +129,57 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "coe-rs" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -100,6 +189,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -166,6 +264,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dyn-stack" version = "0.11.0" @@ -249,6 +358,12 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -256,7 +371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -281,7 +396,7 @@ dependencies = [ "num-complex", "num-traits", "paste", - "rand", + "rand 0.8.6", "rand_distr", "rayon", "reborrow", @@ -303,12 +418,121 @@ dependencies = [ "reborrow", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "gemm" version = "0.18.2" @@ -451,8 +675,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -463,10 +689,43 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -480,6 +739,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -487,154 +752,482 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "itoa" -version = "1.0.18" +name = "http" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] [[package]] -name = "jobserver" -version = "0.1.34" +name = "http-body" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ - "getrandom 0.3.4", - "libc", + "bytes", + "http", ] [[package]] -name = "libc" -version = "0.2.186" +name = "http-body-util" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] [[package]] -name = "libloading" -version = "0.9.0" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ - "cfg-if", - "windows-link", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", ] [[package]] -name = "libm" -version = "0.2.16" +name = "hyper-rustls" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] [[package]] -name = "linux-raw-sys" -version = "0.12.1" +name = "hyper-util" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] [[package]] -name = "lodedb-core" -version = "1.3.2" +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ - "lodedb-gpu", - "rayon", - "rustix", - "serde", - "serde_json", - "sha2", - "turbovec", - "zstd", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", ] [[package]] -name = "lodedb-ffi" -version = "1.3.2" +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "lodedb-core", - "serde", - "serde_json", + "cc", ] [[package]] -name = "lodedb-gpu" -version = "1.3.2" +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "cudarc", + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "matrixcompare" -version = "0.3.0" +name = "icu_locale_core" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ - "matrixcompare-core", - "num-traits", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "matrixcompare-core" -version = "0.1.0" +name = "icu_normalizer" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] [[package]] -name = "matrixmultiply" -version = "0.3.10" +name = "icu_normalizer_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "autocfg", - "rawpointer", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] [[package]] -name = "memchr" -version = "2.8.2" +name = "icu_properties_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] -name = "nalgebra" -version = "0.32.6" +name = "icu_provider" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ - "approx", - "matrixmultiply", - "nalgebra-macros", - "num-complex", - "num-rational", - "num-traits", - "rand", - "rand_distr", - "simba", - "typenum", + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", ] [[package]] -name = "nalgebra-macros" -version = "0.2.2" +name = "idna" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "proc-macro2", - "quote", - "syn", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "nano-gemm" -version = "0.1.3" +name = "idna_adapter" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb5ba2bea1c00e53de11f6ab5bd0761ba87dc0045d63b0c87ee471d2d3061376" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ - "equator 0.2.2", - "nano-gemm-c32", - "nano-gemm-c64", - "nano-gemm-codegen", - "nano-gemm-core", - "nano-gemm-f32", - "nano-gemm-f64", - "num-complex", + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lodedb-cloud-core" +version = "1.3.2" +dependencies = [ + "lodedb-core", + "object_store", + "serde_json", + "sha2", + "tempfile", + "tokio", +] + +[[package]] +name = "lodedb-core" +version = "1.3.2" +dependencies = [ + "lodedb-gpu", + "rayon", + "rustix", + "serde", + "serde_json", + "sha2", + "turbovec", + "zstd", +] + +[[package]] +name = "lodedb-ffi" +version = "1.3.2" +dependencies = [ + "lodedb-core", + "serde", + "serde_json", +] + +[[package]] +name = "lodedb-gpu" +version = "1.3.2" +dependencies = [ + "cudarc", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matrixcompare" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445" +dependencies = [ + "matrixcompare-core", + "num-traits", +] + +[[package]] +name = "matrixcompare-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nalgebra" +version = "0.32.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +dependencies = [ + "approx", + "matrixmultiply", + "nalgebra-macros", + "num-complex", + "num-rational", + "num-traits", + "rand 0.8.6", + "rand_distr", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nano-gemm" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5ba2bea1c00e53de11f6ab5bd0761ba87dc0045d63b0c87ee471d2d3061376" +dependencies = [ + "equator 0.2.2", + "nano-gemm-c32", + "nano-gemm-c64", + "nano-gemm-codegen", + "nano-gemm-core", + "nano-gemm-f32", + "nano-gemm-f64", + "num-complex", ] [[package]] @@ -737,7 +1330,7 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "bytemuck", "num-traits", - "rand", + "rand 0.8.6", ] [[package]] @@ -769,12 +1362,54 @@ dependencies = [ "libm", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.5", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "ordered-float" version = "4.6.0" @@ -784,12 +1419,41 @@ dependencies = [ "num-traits", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pest" version = "2.8.6" @@ -833,6 +1497,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.33" @@ -854,6 +1524,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -912,29 +1591,122 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.46" +name = "quick-xml" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ - "proc-macro2", + "memchr", + "serde", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "quinn" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] [[package]] -name = "rand" -version = "0.8.6" +name = "quinn-proto" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -944,7 +1716,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -956,6 +1738,21 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.4.3" @@ -963,7 +1760,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand", + "rand 0.8.6", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -1007,6 +1813,77 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.4" @@ -1017,9 +1894,68 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "safe_arch" version = "0.7.4" @@ -1038,6 +1974,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "seq-macro" version = "0.3.6" @@ -1087,6 +2061,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1094,7 +2080,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1117,6 +2103,34 @@ dependencies = [ "wide", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "statrs" version = "0.17.1" @@ -1126,9 +2140,15 @@ dependencies = [ "approx", "nalgebra", "num-traits", - "rand", + "rand 0.8.6", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -1140,6 +2160,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sysctl" version = "0.6.0" @@ -1150,17 +2190,39 @@ dependencies = [ "byteorder", "enum-as-inner", "libc", - "thiserror", + "thiserror 1.0.69", "walkdir", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1174,6 +2236,173 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "turbovec" version = "0.9.0" @@ -1181,8 +2410,8 @@ dependencies = [ "faer", "ndarray", "ordered-float", - "rand", - "rand_chacha", + "rand 0.8.6", + "rand_chacha 0.3.1", "rand_distr", "rayon", "statrs", @@ -1206,6 +2435,30 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.5" @@ -1222,6 +2475,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1237,6 +2499,94 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wide" version = "0.7.33" @@ -1253,7 +2603,42 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1262,6 +2647,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1271,12 +2683,105 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.52" @@ -1297,6 +2802,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index fcdfeb0a..355b57b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "crates/lodedb-cloud-core", "crates/lodedb-core", "crates/lodedb-ffi", "crates/lodedb-gpu", diff --git a/README.md b/README.md index 840f2a0b..56164eff 100644 --- a/README.md +++ b/README.md @@ -605,6 +605,51 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star +## Managed cloud (OreCloud) + +LodeDB stores can live in [OreCloud](https://db.egoistmachines.com), the managed-cloud +companion: durable sync for local stores, hosted read serving, and a per-user memory +data plane for agent applications. The client ships as the `[cloud]` extra: + +```sh +pip install "lodedb[cloud]" + +lodedb cloud login # one browser approval +lodedb cloud init ./my-store # link the directory to a managed remote +lodedb cloud keys ./my-store # list the directory's index keys +lodedb cloud sync ./my-store cloud # mirror the local store to the cloud +``` + +A cloud store opens through the same class as a local one, via the +`LodeDB.cloud` alternate constructor: + +```python +from lodedb import LodeDB + +db = LodeDB("./notes") # local, as always +db = LodeDB.cloud("user-42") # managed cloud, same verbs +db.add("the quick brown fox") # embedded server-side +db.search("fox", k=5) +``` + +A bare store id is enough — the org/environment half resolves from the +credential (`token=`, the `ORECLOUD_TOKEN`/`ORECLOUD_HOST` environment pair, +or `lodedb cloud login`); pass the full `"org/environment/store"` triple in +cross-environment scripts. For config-driven code, where one string field (an +env var, a YAML value) must express either a local path or a cloud store, the +plain constructor accepts the explicit URL form — +`LodeDB("orecloud://org/environment/store")` — and returns the same handle. +An application serving many end users should hold one `Client` and open +per-user handles from it (`Client().store(user_id)` — one credential +resolution, one shared HTTP pool); `from lodedb.cloud import Client` works as +the import root. + +Work locally with `lodedb` exactly as before. The client is first-party code +in this package — `lodedb.cloud` and the `lodedb cloud` CLI, with push/pull/ +sync running on the same bundled native core as the engine — and the extra +installs only its dependencies (`httpx`, `pynacl`). Everything cloud loads +lazily, so a plain `import lodedb` stays network-free. + ## Concurrency & durability - **Single writer, many readers, per path.** One handle holds the path open for *writing* at diff --git a/crates/lodedb-cloud-core/Cargo.toml b/crates/lodedb-cloud-core/Cargo.toml new file mode 100644 index 00000000..79267c51 --- /dev/null +++ b/crates/lodedb-cloud-core/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "lodedb-cloud-core" +description = "Native transfer core for the LodeDB managed cloud — durable push/pull/sync over the open commit format." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +# The engine core. The cloud transfer plane links its `storage` module directly +# so the managed cloud and the embedded product share ONE commit-format +# implementation (schema, layout, checksum, sub-manifests incl. `tvmv`). A path +# dep now that both live in this workspace — engine changes and the transfer +# plane move together under one suite. +lodedb-core = { path = "../lodedb-core" } +# Object-storage backend (S3/GCS/Azure) behind the same ArtifactStore trait. The +# `aws` feature adds the S3 builder; the crate's conditional-put API (Create / +# Update{e_tag}) is what makes the root-pointer compare-and-swap strongly +# consistent on object storage. Its InMemory backend also drives the tests. +object_store = { version = "0.12.5", features = ["aws"] } +serde_json = "1.0" +sha2 = "0.10" +# The object_store backends are async; the runtime is enabled with net + time so a +# real S3 client works (not just the in-memory test backend). ObjectArtifactStore +# keeps the sync ArtifactStore trait by owning a current-thread runtime. +tokio = { version = "1.47.5", default-features = false, features = ["rt", "net", "time"] } +# The pointer serialize/validate round-trip reuses lodedb-core's own +# write/read_commit_manifest (which operate on file paths) via a scratch file, so +# object-store pointer documents are byte-identical to on-disk ones. Also used by +# the test fixtures to build committed generations in throwaway directories. +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/lodedb-cloud-core/src/artifact_store.rs b/crates/lodedb-cloud-core/src/artifact_store.rs new file mode 100644 index 00000000..acee15e3 --- /dev/null +++ b/crates/lodedb-cloud-core/src/artifact_store.rs @@ -0,0 +1,91 @@ +//! Storage abstraction for committed, generation-addressed artifacts. +//! +//! The substrate is the engine's existing on-disk layout: a committed generation +//! is a set of immutable, sha256-addressed `g.*` artifacts under +//! `.gen/` pinned by an atomic `.commit.json` root pointer. An +//! [`ArtifactStore`] exposes exactly the operations that layout needs: +//! +//! - **names** are store-relative paths mirroring the on-disk layout, e.g. +//! `idx.gen/g7.json` or `idx.gen/g7.json.json-delta/delta-00000000.jsd`; +//! - the **pointer key** is the index key (`idx` -> `idx.commit.json`). +//! +//! Implementations treat content artifacts as immutable and content-addressed +//! (written once, never overwritten), and commit a generation *only* by swapping +//! the root pointer. [`LocalArtifactStore`](crate::LocalArtifactStore) is the +//! default; object-storage backends (S3/GCS/Azure) land in later milestones with +//! the artifact names becoming object keys under a per-tenant prefix and the +//! pointer swap becoming a conditional write. + +use crate::error::Result; +use serde_json::Value; + +/// Reads/writes immutable artifacts and swaps a root pointer atomically. +/// +/// The interface is deliberately small: byte-level artifact I/O plus a pointer +/// compare-and-swap. That is everything the generation-addressed commit format +/// needs as a cloud substrate. +pub trait ArtifactStore { + /// Returns one artifact's bytes; [`ArtifactStoreError::NotFound`] if the name + /// is absent. + /// + /// [`ArtifactStoreError::NotFound`]: crate::ArtifactStoreError::NotFound + fn read_bytes(&self, name: &str) -> Result>; + + /// Writes one immutable artifact unless it already exists. + /// + /// `sha256` is the expected lowercase-hex digest of `data`; a mismatch is an + /// integrity error so corruption is never stored. A name already present + /// with identical bytes is a no-op (idempotent re-push); present with + /// *different* bytes is a conflict — artifacts are immutable and are never + /// overwritten in place. + fn write_bytes_if_absent(&self, name: &str, data: &[u8], sha256: &str) -> Result<()>; + + /// Whether an artifact named `name` is present, without asserting anything + /// about its content. + /// + /// The default reads the artifact and maps + /// [`NotFound`](crate::ArtifactStoreError::NotFound) to `false`; backends + /// with a cheap existence primitive (a filesystem stat, an object-store + /// HEAD) should override it to avoid fetching the bytes. + fn contains(&self, name: &str) -> Result { + match self.read_bytes(name) { + Ok(_) => Ok(true), + Err(crate::ArtifactStoreError::NotFound(_)) => Ok(false), + Err(error) => Err(error), + } + } + + /// Returns the committed root-manifest body for `key`, or `None`. + /// + /// The body carries the generation number, the base epoch, counts, and the + /// per-store sub-manifests that name every artifact in the generation. It is + /// the read side that [`compare_and_swap_pointer`](Self::compare_and_swap_pointer) + /// and the inventory helpers need to learn the currently-committed generation. + fn read_pointer(&self, key: &str) -> Result>; + + /// Publishes `new_body` as the root pointer iff the currently committed body is + /// exactly `old_body` (`None` means the pointer must not yet exist); otherwise + /// [`ArtifactStoreError::PointerConflict`]. This swap is the only commit point: + /// artifacts are published first, then the pointer flips all-or-nothing. + /// + /// The precondition is the full committed body, not just its generation number. + /// A generation number is not a unique version token — two independent lineages + /// can share one with different content — so comparing the whole body is what + /// makes this a sound compare-and-swap rather than an ABA-prone check. + /// + /// [`ArtifactStoreError::PointerConflict`]: crate::ArtifactStoreError::PointerConflict + fn compare_and_swap_pointer( + &self, + key: &str, + old_body: Option<&Value>, + new_body: &Value, + ) -> Result<()>; +} + +/// The generation number recorded in a committed body, if present. +/// +/// Used only to annotate a [`PointerConflict`](crate::ArtifactStoreError::PointerConflict) +/// for readability — the swap precondition is the full body, not this number. +pub(crate) fn body_generation(body: &Value) -> Option { + body.get("generation").and_then(Value::as_u64) +} diff --git a/crates/lodedb-cloud-core/src/blob_layout.rs b/crates/lodedb-cloud-core/src/blob_layout.rs new file mode 100644 index 00000000..414764e7 --- /dev/null +++ b/crates/lodedb-cloud-core/src/blob_layout.rs @@ -0,0 +1,120 @@ +//! Content-addressed blob naming for the managed remote layout. +//! +//! From Phase 2 on, the managed service stores artifacts content-addressed — +//! `blobs/sha256/aa/` under a per-tenant prefix — instead of under +//! their engine path names. Content addressing is what absorbs the +//! fork-collision case (two branches committing different artifacts under the +//! same engine name, e.g. `idx.gen/g7.json`, coexist as two blobs), and the +//! two-level `aa/` fan-out keeps listings and prefix operations tractable. +//! +//! Nothing in Phase 0 stores blobs this way yet; these helpers exist now, pure +//! and tested, to pin the naming contract every later phase (transfer plane, +//! share redaction, GC) builds on. The digest is always the lowercase-hex +//! SHA-256 of the blob's bytes — the same digest the engine records per +//! artifact and [`ArtifactStore::write_bytes_if_absent`] verifies. +//! +//! [`ArtifactStore::write_bytes_if_absent`]: crate::ArtifactStore::write_bytes_if_absent + +use crate::error::{ArtifactStoreError, Result}; + +/// The fixed prefix every blob name lives under. +const BLOB_PREFIX: &str = "blobs/sha256"; + +/// Returns the store-relative blob name for a content digest: +/// `blobs/sha256/aa/`, where `aa` is the digest's first two hex chars. +/// +/// Rejects anything that is not exactly 64 lowercase hex characters — blob +/// names participate in authorization paths, so a malformed digest must fail +/// closed rather than mint a name outside the contract. +pub fn blob_name(sha256: &str) -> Result { + validate_sha256(sha256)?; + Ok(format!("{BLOB_PREFIX}/{}/{sha256}", &sha256[..2])) +} + +/// Parses a blob name back to its content digest — the exact inverse of +/// [`blob_name`]. Rejects any name that deviates from the contract (wrong +/// prefix, fan-out directory disagreeing with the digest, malformed digest). +pub fn parse_blob_name(name: &str) -> Result { + let malformed = || { + ArtifactStoreError::Integrity(format!( + "blob name {name:?} does not match {BLOB_PREFIX}/aa/" + )) + }; + let rest = name.strip_prefix(BLOB_PREFIX).ok_or_else(malformed)?; + let rest = rest.strip_prefix('/').ok_or_else(malformed)?; + let (fan_out, sha256) = rest.split_once('/').ok_or_else(malformed)?; + validate_sha256(sha256).map_err(|_| malformed())?; + if fan_out != &sha256[..2] { + return Err(malformed()); + } + Ok(sha256.to_string()) +} + +/// Requires exactly 64 lowercase hex characters. Also used by the generation +/// inventory to validate every manifest digest at the trust boundary: digests +/// become staging file names and object keys, so a non-hex "digest" must fail +/// closed before it can name a path. +pub(crate) fn validate_sha256(sha256: &str) -> Result<()> { + let valid = sha256.len() == 64 + && sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); + if valid { + Ok(()) + } else { + Err(ArtifactStoreError::Integrity(format!( + "{sha256:?} is not a lowercase-hex sha256 digest" + ))) + } +} + +#[cfg(test)] +mod tests { + use super::{blob_name, parse_blob_name}; + use crate::error::ArtifactStoreError; + + const SHA: &str = "aa30b1cc05c10ac8a1f4e6d46f7d4b1a9c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"; + + #[test] + fn names_a_blob_under_its_two_char_fan_out() { + assert_eq!(blob_name(SHA).unwrap(), format!("blobs/sha256/aa/{SHA}")); + } + + #[test] + fn parse_inverts_blob_name() { + assert_eq!(parse_blob_name(&blob_name(SHA).unwrap()).unwrap(), SHA); + } + + #[test] + fn rejects_malformed_digests() { + for bad in [ + "", + "abc", + &SHA[..63], // too short + &format!("{SHA}0"), // too long + &SHA.to_uppercase(), // uppercase + &format!("g{}", &SHA[1..]), // non-hex + ] { + assert!(matches!( + blob_name(bad).unwrap_err(), + ArtifactStoreError::Integrity(_) + )); + } + } + + #[test] + fn rejects_names_off_contract() { + for bad in [ + format!("blobs/sha256/{SHA}"), // no fan-out level + format!("blobs/sha256/bb/{SHA}"), // fan-out disagrees + format!("blobs/md5/aa/{SHA}"), // wrong algorithm segment + format!("sha256/aa/{SHA}"), // wrong prefix + format!("blobs/sha256/aa/{}", &SHA[..63]), // malformed digest + ] { + assert!(matches!( + parse_blob_name(&bad).unwrap_err(), + ArtifactStoreError::Integrity(_) + )); + } + } +} diff --git a/crates/lodedb-cloud-core/src/client_ops.rs b/crates/lodedb-cloud-core/src/client_ops.rs new file mode 100644 index 00000000..f3784bed --- /dev/null +++ b/crates/lodedb-cloud-core/src/client_ops.rs @@ -0,0 +1,621 @@ +//! Client-edge operations: the six verbs a frontend invokes by target string. +//! +//! Frontends (the `orecloud` CLI via the Python binding today; `lodedb cloud` / +//! `LodeDBCloud` later) name each end of an operation with one string — a local +//! directory path or an `s3://bucket/prefix` URL. This module owns the +//! composition from those strings to the typed primitives (`export_generation`, +//! `verify_generation`, `status_for_push`, the sync classifier), so every +//! frontend shares one tested implementation and the FFI binding stays a pure +//! argument/result translator with no logic of its own. +//! +//! When the local end is a directory, transfers additionally maintain the sync +//! sidecar (`.orecloud`, see [`sync_state`](crate::sync_state)): a +//! successful push or pull records the published/restored generation as the +//! new base, which is what lets [`sync`] classify later runs as fast-forwards +//! or divergences instead of blindly overwriting either end. +//! +//! Long-lived service code (e.g. a serving worker) should NOT route through +//! here: it constructs its stores once and calls the typed primitives directly, +//! rather than re-resolving target strings per operation. + +use crate::artifact_store::ArtifactStore; +use crate::error::{ArtifactStoreError, Result}; +use crate::generation_inventory::{ + inventory_from_body, list_index_keys, write_restored_journal_manifests, +}; +use crate::manifest_transfer::{ + export_generation_pinned, export_generation_with_body, publish_staged, + stage_generation_pinned, TransferResult, +}; +use crate::snapshot_identity::{logical_id, snapshot_id}; +use crate::status::{compare_generations, StatusReport}; +use crate::store_target::artifact_store_from_target; +use crate::sync_plan::{classify, SnapRef, SyncClassification}; +use crate::sync_state::{read_sync_state, write_sync_state, SidecarRead, SyncState}; +use crate::transfer_policy::TransferPolicy; +use crate::verify::{verify_candidate_opens, verify_generation, OpenReport, VerifyReport}; +use serde_json::Value; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// What a completed pull produced: the transfer itself plus the proof that the +/// restored copy opens through the engine (with its loaded counts). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PullOutcome { + pub transfer: TransferResult, + pub open: OpenReport, +} + +/// An explicit override of the sync classification. +/// +/// A force overrides the *classification* only — never the stores' integrity +/// invariants. In particular, divergent lineages that reuse an artifact file +/// name with different bytes (a same-epoch fork) still fail closed against +/// plain directory/`s3://` remotes; see [`sync`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncForce { + /// No override: only fast-forwards run; `Diverged`/`Unknown` are errors. + None, + /// Push local over the remote regardless of classification (still raced + /// through the pointer CAS, still checksum-verified). + Push, + /// Pull the remote over local regardless of classification (still + /// verify-opened before the sidecar records it). + Pull, +} + +/// What one [`sync`] run decided and did. +/// +/// `classification` is the pre-transfer three-pointer classification; +/// `action` is what actually ran (`"none"`, `"push"`, or `"pull"`), `forced` +/// whether a force flag overrode the classification. `transfer`/`open` are +/// present when that action moved data (`open` only for pulls). +/// `sidecar_corrupt` is a warning: a sidecar file was present but failed +/// validation, so the base was treated as absent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyncOutcome { + pub index_key: String, + pub classification: String, + pub action: String, + pub forced: bool, + pub transfer: Option, + pub open: Option, + pub sidecar_corrupt: bool, +} + +/// Lists the index keys with a committed generation in the local directory +/// `dir`. +pub fn keys(dir: &str) -> Result> { + list_index_keys(Path::new(dir)) +} + +/// Pushes `index_key`'s committed generation from the local `dir` to `remote` +/// under `policy`. +/// +/// When `dir` is a directory (not an object-store URL), a successful push also +/// records the published generation in the sync sidecar, making it the base +/// for later [`sync`]/[`status`] classification. +pub fn push( + dir: &str, + remote: &str, + index_key: &str, + policy: TransferPolicy, +) -> Result { + let source = artifact_store_from_target(dir)?; + let dest = artifact_store_from_target(remote)?; + let (transfer, published) = export_generation_with_body(&*source, &*dest, index_key, policy)?; + record_base(dir, remote, index_key, &published)?; + Ok(transfer) +} + +/// Restores `index_key`'s committed generation from `remote` into the local +/// `dir`, then proves the engine can open the restored copy read-only — the +/// roadmap's "restore verifies before accepting" as one operation, so no +/// frontend can offer a pull that skips the check. +/// +/// The pull ships the remote verbatim ([`TransferPolicy::full`]): a remote that +/// was pushed redacted already omits text, so there is nothing to filter on the +/// way back. A successful pull records the restored generation in the sync +/// sidecar (see [`push`]). +pub fn pull(remote: &str, dir: &str, index_key: &str) -> Result { + if !is_local_dir(dir) { + return Err(ArtifactStoreError::Backend(format!( + "pull's local end must be a directory path, got {dir:?}" + ))); + } + let source = artifact_store_from_target(remote)?; + let dest = artifact_store_from_target(dir)?; + // Restores mutate the database directory, so they contend on the engine's + // own single-writer lock: a live writer's in-memory state is based on the + // pointer we are about to replace, and letting both proceed loses one + // side's commit. The guard drops at the end of the restore. + fs::create_dir_all(dir)?; + let _writer_lock = acquire_writer_lock(dir)?; + ensure_no_pending_wal( + dir, + index_key, + "checkpoint them by opening the store once (or discard them with \ + `sync --force-pull`) before restoring over this directory", + )?; + let source_raw = source.read_pointer(index_key)?.ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "no committed generation to export for index key {index_key:?}" + )) + })?; + let dest_body = dest.read_pointer(index_key)?; + let (transfer, open, restored) = restore_staged( + &*source, + &*dest, + dir, + index_key, + source_raw, + dest_body, + false, + )?; + record_base(dir, remote, index_key, &restored)?; + Ok(PullOutcome { transfer, open }) +} + +/// The shared local-restore tail: stage the artifacts, prove the candidate +/// opens through the engine (from a scratch layout), and only then publish +/// the pointer — so every acceptance failure leaves the destination on its +/// previous committed generation. `discard_wal` truncates the destination +/// WAL right after the swap (force-pull semantics); callers must have +/// refused a pending WAL beforehand when not forcing. The journal manifests +/// are rebuilt strictly last (see the inline comment). +/// +/// Known crash windows, both narrow and loud rather than corrupting: a crash +/// between the swap and the WAL truncation can leave discarded records +/// beside the new lineage (force-pull only), and a failure between the swap +/// and the journal rebuild leaves the restored copy readable but +/// fail-closed on its first O(changed) mutation. Closing them fully needs a +/// recoverable restore transaction (engine-side recovery on open) — a +/// follow-up, not this milestone. +/// +/// Caller holds the directory writer lock. +fn restore_staged( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + dir: &str, + index_key: &str, + source_raw: Value, + dest_body: Option, + discard_wal: bool, +) -> Result<(TransferResult, OpenReport, Value)> { + let staged = stage_generation_pinned( + source, + dest, + index_key, + TransferPolicy::full(), + source_raw, + dest_body, + ) + .map_err(explain_fork_collision)?; + // Acceptance checks run against a scratch candidate BEFORE the pointer + // moves: a checksum-consistent but semantically unopenable generation must + // never become the committed destination. (The scratch carries its own + // journal manifests, so the real destination stays untouched.) + let open = verify_candidate_opens(Path::new(dir), index_key, &staged.source_body)?; + let (transfer, restored) = publish_staged(dest, index_key, staged)?; + if discard_wal { + lodedb_core::storage::wal::truncate(&contained_wal_path(dir, index_key)?, false)?; + } + // Rebuild the engine's per-store delta-journal manifests (working state + // the body doesn't pin): without them a restored copy opens read-only + // fine but fails closed on its first O(changed) mutation. Strictly AFTER + // the pointer swap: writing them first would, on a failed CAS, leave the + // candidate's journals attached to some other writer's committed body. + write_restored_journal_manifests(Path::new(dir), index_key, &restored)?; + Ok((transfer, open, restored)) +} + +/// Takes the engine's exclusive single-writer lock on `dir` for the duration +/// of a local restore. +fn acquire_writer_lock(dir: &str) -> Result { + lodedb_core::engine::acquire_dir_writer_lock(Path::new(dir)).map_err(ArtifactStoreError::Core) +} + +/// Refuses a restore when the destination WAL still holds acknowledged +/// operations: replaying them onto the pulled lineage — or silently dropping +/// them — would corrupt or lose acknowledged writes. `hint` names the caller's +/// resolution path. +pub(crate) fn ensure_no_pending_wal(dir: &str, index_key: &str, hint: &str) -> Result<()> { + let ops = pending_wal_ops(dir, index_key)?; + if ops > 0 { + return Err(ArtifactStoreError::PendingWal { + ops, + hint: hint.to_string(), + }); + } + Ok(()) +} + +/// The WAL path for `index_key` under `dir`, confined to `dir`: the key can +/// arrive from CLI/remote input, so a `../` or absolute spelling must fail +/// closed here rather than name a WAL-shaped file outside the store root. +pub(crate) fn contained_wal_path(dir: &str, index_key: &str) -> Result { + crate::paths::resolve_within( + Path::new(dir), + &lodedb_core::storage::wal::wal_path(Path::new(dir), index_key), + ) +} + +/// The number of valid, replayable records in `dir`'s WAL for `index_key` +/// (0 when the WAL is absent or empty). Public for the client edge: a +/// pull-direction sync consults it before downloading a single blob. +pub fn pending_wal_ops(dir: &str, index_key: &str) -> Result { + let wal = contained_wal_path(dir, index_key)?; + Ok(lodedb_core::storage::wal::scan_stats(&wal)?.op_count) +} + +/// Compares the local `dir` against `remote` for a push of `index_key` under +/// `policy`. Read-only on both ends. +/// +/// Beyond the byte-level comparison, the report carries lineage: whether a +/// sync sidecar is present, its recorded base generation, and the +/// three-pointer classification (`in_sync`/`local_ahead`/`remote_ahead`/ +/// `diverged`/`republish`/`unknown`) a [`sync`] of the same arguments would +/// act on. +pub fn status( + dir: &str, + remote: &str, + index_key: &str, + policy: TransferPolicy, +) -> Result { + let source = artifact_store_from_target(dir)?; + let dest = artifact_store_from_target(remote)?; + // Read each pointer once and derive both the byte-level comparison and the + // lineage classification from the same snapshot, so the two can't disagree. + let local_body = source + .read_pointer(index_key)? + .map(|body| policy.redact_body(&body)); + let remote_body = dest.read_pointer(index_key)?; + let local_inventory = inventory_from_body(index_key, local_body.as_ref())?; + let remote_inventory = inventory_from_body(index_key, remote_body.as_ref())?; + let mut report = compare_generations( + index_key, + local_inventory.as_ref(), + remote_inventory.as_ref(), + ); + + let sidecar = read_sidecar_if_dir(dir, index_key)?; + let base = trusted_base(&sidecar, remote); + report.sidecar_present = base.is_some(); + report.sidecar_corrupt = sidecar.corrupt; + report.base_generation = base.map(|base| base.generation); + let local_ref = local_body.as_ref().map(snap_ref).transpose()?; + let remote_ref = remote_body.as_ref().map(snap_ref).transpose()?; + let classification = classify(local_ref.as_ref(), base, remote_ref.as_ref()); + report.classification = Some(classification.as_str().to_string()); + Ok(report) +} + +/// Re-hashes every artifact `index_key`'s committed generation pins in `target` +/// (a local directory or object-store URL) against the manifest's checksums. +pub fn verify(target: &str, index_key: &str) -> Result { + let store = artifact_store_from_target(target)?; + verify_generation(&*store, index_key) +} + +/// Synchronizes `index_key` between the local `dir` and `remote`: classifies +/// the three pointers (local under `policy`, the sidecar's recorded base, and +/// the remote), then runs at most one fast-forward transfer. +/// +/// - `InSync` is a no-op; `LocalAhead`/`Republish` push; `RemoteAhead` pulls +/// (restore + verify-open, exactly like [`pull`]). +/// - `Diverged`/`Unknown` refuse with +/// [`ArtifactStoreError::SyncConflict`] — overwriting either end would +/// discard a commit, a decision only the caller can make via `force`. +/// - A forced push/pull skips the refusal but nothing else: the transfer still +/// goes through the pointer CAS and (for pulls) the verify-open. Force also +/// cannot override the stores' immutability invariant: two lineages that +/// diverged from one base and reuse the same artifact file names with +/// different bytes (a same-epoch fork) fail closed with a recovery hint — +/// resolving that shape against a dumb remote needs the managed +/// content-addressed layout of a later milestone. +/// +/// Transfers are conditional on the exact remote/local state that was +/// classified: a concurrent advance of the destination between classification +/// and publish surfaces as a `PointerConflict` (retry the sync). How strong +/// that guarantee is depends on the destination backend: object stores use a +/// genuinely atomic conditional write, while a directory's pointer swap is +/// read-check-then-replace ([`LocalArtifactStore`](crate::LocalArtifactStore) +/// documents this) — so running sync concurrently with an *active engine +/// writer* on the same directory is out of contract, exactly as LodeDB's +/// single-writer model already requires. Any successful transfer records the +/// new base in the sidecar. +pub fn sync( + dir: &str, + remote: &str, + index_key: &str, + policy: TransferPolicy, + force: SyncForce, +) -> Result { + if !is_local_dir(dir) { + return Err(ArtifactStoreError::Backend(format!( + "sync's local end must be a directory path, got {dir:?}" + ))); + } + let local_store = artifact_store_from_target(dir)?; + let remote_store = artifact_store_from_target(remote)?; + + let sidecar = read_sync_state(Path::new(dir), index_key)?; + let base = trusted_base(&sidecar, remote); + // Classification sees the local generation as `policy` would publish it + // (redacted), but the raw body is kept too: a pull's pointer CAS must + // precondition on what the local pointer actually holds, not on the + // redacted view. + let local_raw = local_store.read_pointer(index_key)?; + let local_body = local_raw.as_ref().map(|body| policy.redact_body(body)); + let remote_body = remote_store.read_pointer(index_key)?; + let local_ref = local_body.as_ref().map(snap_ref).transpose()?; + let remote_ref = remote_body.as_ref().map(snap_ref).transpose()?; + let classification = classify(local_ref.as_ref(), base, remote_ref.as_ref()); + let had_trusted_base = base.is_some(); + + let outcome = |action: &str, + forced: bool, + transfer: Option, + open: Option| SyncOutcome { + index_key: index_key.to_string(), + classification: classification.as_str().to_string(), + action: action.to_string(), + forced, + transfer, + open, + sidecar_corrupt: sidecar.corrupt, + }; + + let (do_push, forced) = match force { + SyncForce::Push => (true, true), + SyncForce::Pull => (false, true), + SyncForce::None => match classification { + SyncClassification::InSync => { + // No transfer — but if the two ends agree while the recorded + // base is missing (fresh clone of an already-synced pair, a + // remote switch to an identical mirror) or stale (a prior + // transfer whose sidecar write failed), record the agreed + // state so later runs classify as fast-forwards. + let base_is_current = had_trusted_base + && base.map(|base| &base.snapshot_id) + == remote_ref.as_ref().map(|r| &r.snapshot_id); + if !base_is_current { + if let Some(remote_body) = &remote_body { + record_base(dir, remote, index_key, remote_body)?; + } + } + return Ok(outcome("none", false, None, None)); + } + SyncClassification::LocalAhead | SyncClassification::Republish => (true, false), + SyncClassification::RemoteAhead => (false, false), + SyncClassification::Diverged | SyncClassification::Unknown => { + // The refusal carries the corruption diagnosis: a corrupt + // sidecar is the likeliest reason a previously-synced pair + // reads as unknown, and the CLI's warning path never runs on + // an error. + let mut hint = "re-run with --force-push to keep the local copy or --force-pull \ + to keep the remote copy" + .to_string(); + if sidecar.corrupt { + hint.push_str( + " (note: the sync sidecar was present but corrupt and was ignored, so \ + the recorded base could not be trusted)", + ); + } + return Err(ArtifactStoreError::SyncConflict { + classification: classification.as_str().to_string(), + hint, + }); + } + }, + }; + + if do_push { + // Pinned to the exact pair classified above: the source body ships + // verbatim (a concurrent local commit cannot swap in an unclassified + // snapshot) and the CAS preconditions on the classified remote body + // (a concurrent remote advance fails as a PointerConflict). + let source_raw = local_raw.ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "no committed generation to push for index key {index_key:?}" + )) + })?; + let (transfer, published) = export_generation_pinned( + &*local_store, + &*remote_store, + index_key, + policy, + source_raw, + remote_body, + ) + .map_err(explain_fork_collision)?; + record_base(dir, remote, index_key, &published)?; + Ok(outcome("push", forced, Some(transfer), None)) + } else { + // Pull mirrors `pull`: take the writer lock (a live engine writer and + // a restore must never interleave), refuse or discard a pending WAL, + // stage + candidate-verify + publish, then record the base. The CAS + // preconditions on the raw local body read above — a concurrent + // engine commit fails as a PointerConflict rather than being + // clobbered. + let source_raw = remote_body.ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "no committed generation to pull for index key {index_key:?}" + )) + })?; + fs::create_dir_all(dir)?; + let _writer_lock = acquire_writer_lock(dir)?; + let discard_wal = forced; + if !discard_wal { + ensure_no_pending_wal( + dir, + index_key, + "checkpoint them by opening the store once, or re-run with --force-pull \ + to discard them along with the local lineage", + )?; + } + let (transfer, open, restored) = restore_staged( + &*remote_store, + &*local_store, + dir, + index_key, + source_raw, + local_raw, + discard_wal, + )?; + record_base(dir, remote, index_key, &restored)?; + Ok(outcome("pull", forced, Some(transfer), Some(open))) + } +} + +/// Whether a target string names a local directory (no URL scheme). +fn is_local_dir(target: &str) -> bool { + !target.contains("://") +} + +/// Adds recovery guidance when a sync transfer hits the fork-collision limit. +/// +/// Two lineages that diverged from one base commonly reuse the same engine +/// artifact names (`.gen/g.*`) with different bytes; the store's +/// immutability invariant then refuses the overwrite — deliberately, and force +/// flags do not override it (dumb path/`s3://` remotes store artifacts by +/// engine name until the managed content-addressed layout, which absorbs this, +/// arrives in a later milestone). Matching on the store's message text is +/// acceptable here because both sites live in this crate and the immutability +/// tests pin the wording. +pub(crate) fn explain_fork_collision(error: ArtifactStoreError) -> ArtifactStoreError { + match error { + ArtifactStoreError::Integrity(message) + if message.contains("refusing to overwrite an immutable artifact") => + { + ArtifactStoreError::Integrity(format!( + "{message}. The two ends hold divergent lineages that reuse the same artifact \ + file name — a fork this version cannot resolve against a plain \ + directory/s3 remote, even with a force flag. Recover by pulling the side you \ + want to keep into a fresh directory, or pushing it to a fresh remote prefix." + )) + } + other => other, + } +} + +/// Builds the identity for a committed body: the id pair, the generation, and +/// a per-store identity for each payload store (a store is carried iff its +/// sub-manifest is non-null — the same test the engine uses to load it; the +/// identity is a digest of that sub-manifest, which names every artifact and +/// checksum the store pins). +pub(crate) fn snap_ref(body: &Value) -> Result { + let store_id = |store: &str| { + body.get(store) + .filter(|value| !value.is_null()) + .map(|value| crate::digest::sha256_hex(&serde_json::to_vec(value).unwrap_or_default())) + }; + Ok(SnapRef { + snapshot_id: snapshot_id(body)?, + logical_id: logical_id(body)?, + generation: body.get("generation").and_then(Value::as_u64).unwrap_or(0), + text_id: store_id("tvtext"), + lexical_id: store_id("tvlex"), + }) +} + +/// A remote target's canonical identity for sidecar trust comparison. +/// +/// Local directory targets normalize to an absolute path (resolving symlinks +/// and `.`/`..` like the store's own containment checks do), because a +/// relative spelling is cwd-dependent — the same string can name unrelated +/// directories from different working directories, and trusting on the raw +/// string would let one remote inherit another's history. +/// +/// `s3://` targets additionally carry the effective endpoint: the URL alone +/// names a backend only through `AWS_ENDPOINT` (MinIO/R2 use the same +/// `s3://bucket/prefix` spelling against different services), so the endpoint +/// is part of the identity. Credentials and region are deliberately not — +/// rotating keys or fixing a region points at the *same* bucket. A +/// normalization failure or identity mismatch can only produce a false +/// *mismatch* — failing toward force, never toward a wrong fast-forward. +fn remote_identity(remote: &str) -> String { + if is_local_dir(remote) { + return crate::paths::canonical_identity(Path::new(remote)); + } + let url = remote.trim_end_matches('/'); + if remote.starts_with("s3://") { + if let Ok(endpoint) = std::env::var("AWS_ENDPOINT") { + if !endpoint.is_empty() { + return format!("{url}#endpoint={}", endpoint.trim_end_matches('/')); + } + } + } + url.to_string() +} + +/// The sidecar base, but only when it was recorded against `remote` (compared +/// by [`remote_identity`]). +/// +/// The base is a claim about what *this pair of ends* last agreed on; trusting +/// it against a different remote would classify an unrelated remote's content +/// as a fast-forward using another remote's history. +fn trusted_base<'a>(sidecar: &'a SidecarRead, remote: &str) -> Option<&'a SnapRef> { + let identity = remote_identity(remote); + sidecar + .state + .as_ref() + .filter(|state| state.remote == identity) + .map(|state| &state.base) +} + +/// Reads the sidecar when the target is a directory; a URL target has none. +fn read_sidecar_if_dir(dir: &str, index_key: &str) -> Result { + if is_local_dir(dir) { + read_sync_state(Path::new(dir), index_key) + } else { + Ok(SidecarRead { + state: None, + corrupt: false, + }) + } +} + +/// Records `body` as the sidecar base after a successful transfer, when the +/// local end is a directory. URL-addressed local ends carry no sidecar. +fn record_base(dir: &str, remote: &str, index_key: &str, body: &Value) -> Result<()> { + if !is_local_dir(dir) { + return Ok(()); + } + let state = SyncState { + index_key: index_key.to_string(), + remote: remote_identity(remote), + base: snap_ref(body)?, + updated_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs()) + .unwrap_or(0), + }; + write_sync_state(Path::new(dir), &state) +} + +#[cfg(test)] +mod tests { + use super::remote_identity; + + /// The same `s3://` spelling under different `AWS_ENDPOINT`s names + /// different backends (MinIO/R2), so the endpoint is part of the + /// identity. Env mutation is process-global; nothing else in this test + /// binary reads `AWS_ENDPOINT`. + #[test] + fn s3_identity_includes_the_effective_endpoint() { + std::env::remove_var("AWS_ENDPOINT"); + let plain = remote_identity("s3://bucket/prefix/"); + assert_eq!(plain, "s3://bucket/prefix"); + + std::env::set_var("AWS_ENDPOINT", "http://minio.local:9000"); + let minio = remote_identity("s3://bucket/prefix"); + std::env::remove_var("AWS_ENDPOINT"); + + assert_ne!(plain, minio); + assert!(minio.contains("minio.local")); + // Back to the default environment, the identity is stable again. + assert_eq!(remote_identity("s3://bucket/prefix"), plain); + } +} diff --git a/crates/lodedb-cloud-core/src/digest.rs b/crates/lodedb-cloud-core/src/digest.rs new file mode 100644 index 00000000..3ad59be4 --- /dev/null +++ b/crates/lodedb-cloud-core/src/digest.rs @@ -0,0 +1,17 @@ +//! Shared artifact checksum helper. +//! +//! Every artifact is addressed by the lowercase-hex SHA-256 of its bytes — the +//! exact digest the engine records per artifact (`sha256_file_hex`). Both the +//! write path (`LocalArtifactStore::write_bytes_if_absent`) and the verify path +//! (`verify_generation`) re-hash bytes and compare against the manifest's recorded +//! value, so the hashing lives in one place to guarantee they agree byte-for-byte. + +use sha2::{Digest, Sha256}; + +/// Returns the lowercase-hex SHA-256 of `data`. +pub(crate) fn sha256_hex(data: &[u8]) -> String { + Sha256::digest(data) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/lodedb-cloud-core/src/error.rs b/crates/lodedb-cloud-core/src/error.rs new file mode 100644 index 00000000..faaca31a --- /dev/null +++ b/crates/lodedb-cloud-core/src/error.rs @@ -0,0 +1,123 @@ +//! Error type for OreCloud's artifact-store layer. +//! +//! Mirrors the failure surface the Python Milestone-1 modules expressed with +//! stdlib exceptions (`FileNotFoundError`, `RuntimeError`), mapped onto the +//! idiomatic Rust `Result`. The engine's own commit-manifest errors +//! (`lodedb_core::CoreError`) are wrapped rather than flattened, so a corrupt +//! root pointer keeps its diagnostic on the way up. + +use lodedb_core::CoreError; +use std::fmt::{Display, Formatter}; + +/// The result type for every fallible artifact-store operation. +pub type Result = std::result::Result; + +/// A failure raised by an [`ArtifactStore`](crate::ArtifactStore) operation. +#[derive(Debug)] +pub enum ArtifactStoreError { + /// A requested artifact name is not present in the store (the analogue of + /// the Python `FileNotFoundError`). Callers such as `export_generation` + /// surface "no committed generation" as this variant. + NotFound(String), + /// A root-pointer compare-and-swap precondition failed: the committed + /// generation was not the one the caller expected. Kept distinct because it + /// is the *retryable* failure — a concurrent writer advanced the pointer. + PointerConflict { + key: String, + expected: Option, + found: Option, + }, + /// The bytes or path do not match what the committed manifest promises: a + /// checksum mismatch, an attempt to overwrite an immutable artifact with + /// different bytes, a sub-manifest in an unsupported legacy layout, or a + /// name/key that escapes the store root. All mean "do not trust these + /// bytes / this path", so they collapse to one hard-failure category. + Integrity(String), + /// The engine's commit-manifest layer rejected a root-pointer read or write + /// (schema-version or body-checksum failure). + Core(CoreError), + /// An underlying filesystem operation failed. + Io(std::io::Error), + /// A sync refused to transfer because the three-pointer classification + /// requires an explicit force: the two ends diverged past the recorded + /// base, or there is no trustworthy base at all. `classification` is the + /// stable lowercase name (`diverged`/`unknown`); `hint` names the flag + /// that overrides it. Distinct from [`PointerConflict`](Self::PointerConflict): + /// that is a *race* (retryable), this is a *decision* only the caller can + /// make. + SyncConflict { + classification: String, + hint: String, + }, + /// A restore refused to replace the destination pointer because the + /// destination's WAL still holds acknowledged-but-uncheckpointed + /// operations. Replaying those records onto a pulled lineage (or silently + /// dropping them) would corrupt or lose acknowledged writes, so the caller + /// must checkpoint the store first — or explicitly discard the records + /// with a force-pull. Like [`SyncConflict`](Self::SyncConflict), this is a + /// *decision*, not a race. + PendingWal { ops: usize, hint: String }, + /// A storage backend (e.g. an object store) failed for a reason that is not a + /// missing object or a pointer precondition — a network error, a permission + /// denial, or any other transport-level failure. The message carries the + /// backend's own diagnostic. Missing objects map to [`NotFound`](Self::NotFound) + /// and failed conditional writes to [`PointerConflict`](Self::PointerConflict), + /// so callers can still branch on those without inspecting this string. + Backend(String), +} + +impl Display for ArtifactStoreError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotFound(name) => write!(formatter, "artifact {name:?} not found"), + Self::PointerConflict { + key, + expected, + found, + } => write!( + formatter, + "pointer {key:?} compare-and-swap failed: expected generation {expected:?}, \ + found {found:?}" + ), + Self::SyncConflict { + classification, + hint, + } => write!( + formatter, + "sync refused: local and remote are {classification}; {hint}" + ), + Self::PendingWal { ops, hint } => write!( + formatter, + "the destination database holds {ops} uncheckpointed WAL operation(s); {hint}" + ), + Self::Integrity(message) => write!(formatter, "{message}"), + Self::Core(error) => write!(formatter, "{error}"), + Self::Io(error) => write!(formatter, "{error}"), + Self::Backend(message) => write!(formatter, "{message}"), + } + } +} + +impl std::error::Error for ArtifactStoreError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Core(error) => Some(error), + Self::Io(error) => Some(error), + _ => None, + } + } +} + +impl From for ArtifactStoreError { + /// Lets `read_commit_manifest`/`write_commit_manifest` results propagate with + /// `?`, preserving the engine's structured error. + fn from(error: CoreError) -> Self { + Self::Core(error) + } +} + +impl From for ArtifactStoreError { + fn from(error: std::io::Error) -> Self { + Self::Io(error) + } +} diff --git a/crates/lodedb-cloud-core/src/generation_inventory.rs b/crates/lodedb-cloud-core/src/generation_inventory.rs new file mode 100644 index 00000000..2205d403 --- /dev/null +++ b/crates/lodedb-cloud-core/src/generation_inventory.rs @@ -0,0 +1,502 @@ +//! Read-only inventory of the artifacts a committed generation references. +//! +//! Given a committed root manifest, this enumerates every artifact it pins — +//! each per-store base plus its delta segments — as [`ArtifactRef`] records +//! carrying name, checksum, size, kind, base epoch, and base-vs-delta. It reads +//! only the committed root (via `read_commit_manifest`); it never stats the +//! files, reads the `.wal` tail, or reads the on-disk per-store `manifest.json`. +//! So the inventory describes exactly one consistent committed generation. +//! +//! [`diff_inventories`] compares a local inventory against a remote one and +//! reports which artifacts are missing remotely, distinguishing "delta segments +//! onto an existing base epoch" from "a whole new base epoch" — the signal a +//! push needs to stay O(changed). + +use crate::error::{ArtifactStoreError, Result}; +use crate::paths::resolve_within; +use lodedb_core::storage::commit_manifest::{ + commit_manifest_path, read_commit_manifest, COMMIT_MANIFEST_SUFFIX, +}; +use lodedb_core::storage::lexical_store::LEXICAL_INDEX_DELTA_DIR_SUFFIX; +use lodedb_core::storage::multivec_store::MULTIVEC_DELTA_DIR_SUFFIX; +use lodedb_core::storage::state_journal::STATE_JOURNAL_DIR_SUFFIX; +use lodedb_core::storage::text_store::DOCUMENT_TEXT_DELTA_DIR_SUFFIX; +use lodedb_core::storage::tvim_delta::TVIM_DELTA_DIR_SUFFIX; +use lodedb_core::storage::tvvf_store::TVVF_DELTA_DIR_SUFFIX; +use serde_json::{Map, Value}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +/// The generation directory suffix (`.gen`), mirroring +/// `commit_manifest::generation_dir`. Artifact names are store-relative and +/// begin with this directory, so we build the relative prefix here rather than +/// deriving an absolute path. +const GEN_DIR_SUFFIX: &str = ".gen"; + +/// Each per-store sub-manifest key in the root body, paired with the directory +/// suffix its delta segments live under (the delta dir is the base file name + +/// suffix, e.g. `g7.json` -> `g7.json.json-delta/`). The key doubles as the base +/// file extension, since the engine derives base paths as `g.`. A +/// store is inventoried iff its sub-manifest is non-null; `tvmv` (multi-vector / +/// late-interaction) is included — omitting it would silently drop those artifacts +/// from a backup. `tvann` (the persisted ANN cluster partition) is base-only — +/// the engine treats a missing/corrupt `.tvann` as a cache miss and rebuilds — +/// but it still ships: a body referencing a base we never uploaded would fail +/// byte-verification on pull, and shipping it saves the restored/hydrated copy +/// a corpus-sized re-cluster. Its delta suffix follows the engine's uniform +/// `.-delta` derivation (no deltas are ever recorded today). +const STORE_KINDS: &[(&str, &str)] = &[ + ("json", STATE_JOURNAL_DIR_SUFFIX), + ("tvim", TVIM_DELTA_DIR_SUFFIX), + ("tvtext", DOCUMENT_TEXT_DELTA_DIR_SUFFIX), + ("tvlex", LEXICAL_INDEX_DELTA_DIR_SUFFIX), + ("tvmv", MULTIVEC_DELTA_DIR_SUFFIX), + ("tvann", ".tvann-delta"), + // `tvvf` (the rescore original-vector sidecar, engine 1.3.2+) is a journaled + // {base, deltas} store like tvim: vector payload, never text-gated, and the + // engine refuses to open a rescore store without it — so it ships always. + ("tvvf", TVVF_DELTA_DIR_SUFFIX), +]; + +/// One artifact (a base or a delta segment) referenced by a committed generation. +/// +/// `name` is the store-relative path (e.g. `idx.gen/g7.json`); `kind` is the +/// owning store (`json`/`tvim`/`tvtext`/`tvlex`/`tvmv`/`tvann`/`tvvf`); `epoch` is the base +/// epoch the artifact lives under; `is_base` is true for the base snapshot and +/// false for a delta segment appended onto it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArtifactRef { + pub name: String, + pub sha256: String, + pub size_bytes: u64, + pub kind: String, + pub epoch: u64, + pub is_base: bool, +} + +/// The full set of artifacts pinned by one committed generation. +/// +/// `root_body` is the committed manifest body verbatim, suitable as the payload +/// for a destination's `compare_and_swap_pointer` when shipping this generation +/// elsewhere. +#[derive(Debug, Clone)] +pub struct GenerationInventory { + pub index_key: String, + pub generation: u64, + pub base_epoch: u64, + pub document_count: u64, + pub chunk_count: u64, + pub root_body: Value, + pub artifacts: Vec, +} + +/// What a local generation has that a remote one does not. +/// +/// `ships_base` is true when the transfer must upload a base artifact — a new +/// base epoch (cold build or compaction) *or* a replacement base under an epoch +/// the remote already holds at a different checksum (two divergent lineages at +/// the same epoch number). It is false only when every uploaded artifact is a +/// delta segment onto a base the remote already holds byte-for-byte (the +/// O(changed) common case). Callers choosing delta-only vs full-base handling +/// must branch on this, not on the epoch number alone. +#[derive(Debug, Clone)] +pub struct InventoryDiff { + pub to_upload: Vec, + pub ships_base: bool, + pub upload_bytes: u64, +} + +/// Builds an inventory from an already-read commit-manifest body. +/// +/// Returns `None` when `body` is `None` (no committed generation). This is the +/// store-agnostic core: a caller holding a body from any [`ArtifactStore`] (not +/// just the local filesystem) builds the inventory the same way. +/// +/// [`ArtifactStore`]: crate::ArtifactStore +pub fn inventory_from_body( + index_key: &str, + body: Option<&Value>, +) -> Result> { + let Some(body) = body else { + return Ok(None); + }; + // The pointer's file-name key must match the body's own `index_key`, and the + // engine additionally rejects an empty/missing `index_key` as corrupt. Requiring + // exact equality covers both: a mismatch would inventory one directory while + // `load_store` reads another, and an empty key would publish a body the engine + // refuses to open. Only reachable via a tampered/corrupt pointer (the body + // checksum is already validated), so fail closed. + let body_key = body + .get("index_key") + .and_then(Value::as_str) + .unwrap_or_default(); + if body_key != index_key { + return Err(ArtifactStoreError::Integrity(format!( + "commit manifest body index_key {body_key:?} does not match requested key \ + {index_key:?}" + ))); + } + let base_epoch = body_u64(body, "base_epoch"); + // Fail closed on a store this table does not know: a future engine that + // adds a sub-manifest (as `tvann` was added) must not have its artifacts + // silently dropped from a backup — an inventory that understates what the + // body pins ships a generation whose blobs were never uploaded. Store + // sub-manifests are recognizable by shape (an object carrying a journaled + // `base`), which no scalar body field (`generation`, `native_dim`, …) has. + if let Some(object) = body.as_object() { + for (key, value) in object { + let looks_like_store = value.is_object() && value.get("base").is_some(); + if looks_like_store && !STORE_KINDS.iter().any(|(kind, _)| kind == key) { + return Err(ArtifactStoreError::Integrity(format!( + "commit body carries an unknown store sub-manifest {key:?}; this \ + OreCloud build does not know how to inventory it — upgrade OreCloud \ + before transferring this generation" + ))); + } + } + } + let mut artifacts: Vec = Vec::new(); + // Inventory each store whose sub-manifest is non-null, mirroring exactly how the + // engine decides to load a store (`store_manifest(kind).is_some()`). tvim + // included: the engine reads a non-null tvim manifest regardless of the body's + // informational `tvim_present` flag, so gating on that flag could drop a base + // the restored generation needs. + for (kind, dir_suffix) in STORE_KINDS { + artifacts.extend(refs_for_store( + index_key, + kind, + body.get(*kind), + dir_suffix, + base_epoch, + )?); + } + Ok(Some(GenerationInventory { + index_key: index_key.to_string(), + generation: body_u64(body, "generation"), + base_epoch, + document_count: body_u64(body, "document_count"), + chunk_count: body_u64(body, "chunk_count"), + root_body: body.clone(), + artifacts, + })) +} + +/// Reads the live committed root for `index_key` and inventories it. +/// +/// Returns `None` when no `.commit.json` is present (an uncommitted or +/// legacy-only store). Read-only: it touches only the committed root pointer. +/// The key is confined to `persistence_dir` via [`resolve_within`], so an +/// `index_key` from CLI/remote input cannot escape the store root. +pub fn inventory_committed_generation( + persistence_dir: &Path, + index_key: &str, +) -> Result> { + let pointer = resolve_within( + persistence_dir, + &commit_manifest_path(persistence_dir, index_key), + )?; + let body = read_commit_manifest(&pointer)?.map(|manifest| manifest.body); + inventory_from_body(index_key, body.as_ref()) +} + +/// Lists the index keys with a committed root manifest under `persistence_dir`. +/// +/// Lets callers enumerate what to back up without reaching into engine +/// internals. Returns a sorted list; empty when the directory is absent. +pub fn list_index_keys(persistence_dir: &Path) -> Result> { + if !persistence_dir.is_dir() { + return Ok(Vec::new()); + } + let mut keys = Vec::new(); + for entry in fs::read_dir(persistence_dir)? { + let entry = entry?; + if let Some(name) = entry.file_name().to_str() { + if let Some(key) = name.strip_suffix(COMMIT_MANIFEST_SUFFIX) { + keys.push(key.to_string()); + } + } + } + keys.sort(); + Ok(keys) +} + +/// Returns the artifacts present in `local` but missing/differing in `remote`. +/// +/// An artifact is "present" remotely only when the remote names it with the same +/// sha256, so a divergent checksum is treated as needing upload. `remote` is +/// `None` when the destination holds no committed generation yet. +pub fn diff_inventories( + local: &GenerationInventory, + remote: Option<&GenerationInventory>, +) -> InventoryDiff { + let remote_by_name: HashMap<&str, &str> = remote + .map(|inventory| { + inventory + .artifacts + .iter() + .map(|artifact| (artifact.name.as_str(), artifact.sha256.as_str())) + .collect() + }) + .unwrap_or_default(); + let to_upload: Vec = local + .artifacts + .iter() + .filter(|artifact| { + remote_by_name.get(artifact.name.as_str()).copied() != Some(artifact.sha256.as_str()) + }) + .cloned() + .collect(); + // Ship a "full base" whenever a base artifact is in the upload set: a missing + // remote, a different base epoch, or the same epoch number whose base differs + // by checksum (divergent lineages) all surface here as a base ref to upload. + let ships_base = remote.is_none() || to_upload.iter().any(|artifact| artifact.is_base); + let upload_bytes = to_upload.iter().map(|artifact| artifact.size_bytes).sum(); + InventoryDiff { + to_upload, + ships_base, + upload_bytes, + } +} + +/// Expands one per-store sub-manifest into its base + delta artifact refs. +/// +/// Pulls sha256 and byte size straight from the manifest entries (no file stat). +/// A sub-manifest that is absent or empty yields nothing; one carrying the legacy +/// pre-journal marker (`present` with no journaled `base`) fails closed rather +/// than silently omit its artifacts from a backup — rewrite the generation to +/// migrate it to the journaled layout first. +fn refs_for_store( + index_key: &str, + kind: &str, + sub_manifest: Option<&Value>, + dir_suffix: &str, + base_epoch: u64, +) -> Result> { + // Distinguish an absent store (missing key or explicit null -> no artifacts) + // from a present-but-malformed one. The engine's `store_manifest` treats any + // non-null value as present, so a non-null, non-object manifest is corrupt and + // must fail closed rather than be silently skipped — skipping would publish a + // body the engine then refuses to open. + let sub_manifest = match sub_manifest { + None | Some(Value::Null) => return Ok(Vec::new()), + Some(value) => value.as_object().ok_or_else(|| { + ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest is non-null but not a JSON object" + )) + })?, + }; + let Some(base) = sub_manifest.get("base").and_then(Value::as_object) else { + if truthy_map(sub_manifest, "present") { + return Err(ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest uses the legacy pre-journal layout, which the generation \ + inventory does not support; rewrite the generation first" + ))); + } + // A non-null sub-manifest object must carry a journaled base: the engine + // reads `.gen/g.` when opening this store. Treating a + // base-less object as "no artifacts" would silently drop that base from a + // backup and leave the restored generation unopenable. An absent store is + // null (handled above), never a base-less object, so fail closed here. + return Err(ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest is a non-null object without a base; expected a journaled \ + {{base, deltas}} manifest" + ))); + }; + + let gen_dir = format!("{index_key}{GEN_DIR_SUFFIX}"); + // The engine derives the base path from an epoch + store kind, NOT from the + // recorded file_name. Use the derived name as authoritative so the inventory + // names exactly the file the engine will open, and reject a manifest whose + // recorded base file_name disagrees (a tampered or inconsistent pointer) + // rather than copying the wrong file under a name the engine cannot find. + // Most stores live at `g.`; `tvvf` keeps its own epoch + // counter in the sub-manifest and lives at `vf.tvvf` + // (`tvvf_store::base_path`), so its refs must derive from that epoch — the + // generation's base_epoch names a file the engine never writes. + let (store_epoch, base_name) = if kind == "tvvf" { + let vf_epoch = sub_manifest + .get("vf_epoch") + .and_then(Value::as_u64) + .ok_or_else(|| { + ArtifactStoreError::Integrity( + "tvvf sub-manifest carries no vf_epoch; cannot derive its base path" + .to_string(), + ) + })?; + (vf_epoch, format!("vf{vf_epoch}.tvvf")) + } else { + (base_epoch, format!("g{base_epoch}.{kind}")) + }; + let recorded = str_field(base, "file_name"); + if !recorded.is_empty() && recorded != base_name { + return Err(ArtifactStoreError::Integrity(format!( + "{kind} base file_name {recorded:?} does not match the engine-derived name \ + {base_name:?}" + ))); + } + let mut refs = vec![ArtifactRef { + name: format!("{gen_dir}/{base_name}"), + sha256: checked_sha256(kind, base)?, + size_bytes: u64_field(base, "file_bytes"), + kind: kind.to_string(), + epoch: store_epoch, + is_base: true, + }]; + + let delta_dir = format!("{base_name}{dir_suffix}"); + if let Some(deltas) = sub_manifest.get("deltas").and_then(Value::as_array) { + for delta in deltas { + let Some(delta) = delta.as_object() else { + // The engine only ever records object delta entries and rejects + // anything else on replay; skipping a malformed entry would build + // an inventory that disagrees with what the engine will load, so + // fail closed instead. + return Err(ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest has a non-object delta entry" + ))); + }; + let delta_name = str_field(delta, "file_name"); + ensure_plain_file_name(kind, &delta_name)?; + refs.push(ArtifactRef { + name: format!("{gen_dir}/{delta_dir}/{delta_name}"), + sha256: checked_sha256(kind, delta)?, + size_bytes: u64_field(delta, "file_bytes"), + kind: kind.to_string(), + epoch: store_epoch, + is_base: false, + }); + } + } + Ok(refs) +} + +/// Reconstructs each journaled store's on-disk delta-journal `manifest.json` +/// after a restore. +/// +/// The engine writes one journal manifest per store (in +/// `.-delta/manifest.json`) whose content IS the store's +/// sub-manifest in the commit body — but the journal file itself is engine +/// working state, not an artifact the body pins, so transfers never ship it. +/// The read path doesn't need it (it reads the committed root), but the +/// engine's O(changed) mutation path appends journal deltas through it and +/// fails closed when it is missing — which would make a restored directory +/// readable but not writable. Restores call this to rebuild every journal +/// manifest verbatim from the body, so a pulled/hydrated copy behaves +/// exactly like an engine-authored one. `tvann` is excluded: the ANN sidecar +/// is base-only and the engine keeps no journal for it. +pub(crate) fn write_restored_journal_manifests( + persistence_dir: &Path, + index_key: &str, + body: &Value, +) -> Result<()> { + let base_epoch = body_u64(body, "base_epoch"); + for (kind, dir_suffix) in STORE_KINDS { + if *kind == "tvann" { + continue; + } + let Some(sub_manifest) = body.get(*kind).filter(|value| !value.is_null()) else { + continue; + }; + // Mirror `refs_for_store`'s naming: `tvvf` journals live under the + // sidecar's own epoch (`vf.tvvf.tvvf-delta/`), not the + // generation's base epoch. + let base_name = if *kind == "tvvf" { + let vf_epoch = sub_manifest + .get("vf_epoch") + .and_then(Value::as_u64) + .ok_or_else(|| { + ArtifactStoreError::Integrity( + "tvvf sub-manifest carries no vf_epoch; cannot place its journal \ + manifest" + .to_string(), + ) + })?; + format!("vf{vf_epoch}.tvvf") + } else { + format!("g{base_epoch}.{kind}") + }; + let journal_dir = persistence_dir + .join(format!("{index_key}{GEN_DIR_SUFFIX}")) + .join(format!("{base_name}{dir_suffix}")); + fs::create_dir_all(&journal_dir)?; + let path = journal_dir.join("manifest.json"); + let rendered = serde_json::to_string_pretty(sub_manifest).map_err(|error| { + ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest failed to serialize for the journal manifest: {error}" + )) + })?; + // Atomic replace, like every other pointer/manifest write here. + let scratch = journal_dir.join(".manifest.json.tmp"); + fs::write(&scratch, rendered.as_bytes())?; + fs::rename(&scratch, &path)?; + } + Ok(()) +} + + +/// Reads and validates an artifact digest from a manifest entry. +/// +/// Inventory digests cross the trust boundary twice: a managed pull uses them +/// as staging *file names* (`/`) and the managed layout as +/// object-key segments, so a value that is not exactly 64 lowercase hex +/// characters — an absolute path, a `../` traversal, an empty string — must +/// fail closed here, before it can name a path anywhere downstream. +fn checked_sha256(kind: &str, entry: &Map) -> Result { + let digest = str_field(entry, "sha256"); + crate::blob_layout::validate_sha256(&digest).map_err(|_| { + ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest records a malformed artifact sha256 {digest:?}; \ + expected 64 lowercase hex characters" + )) + })?; + Ok(digest) +} + +/// Rejects a manifest `file_name` that is not a plain path component. +/// +/// Engine artifact names are always single components (`g7.json`, +/// `delta-00000000.jsd`). A name containing a path separator or a `.`/`..` segment +/// would, when joined onto `.gen/`, point outside that directory — still +/// inside the store root (`resolve_within` confines it there), but under another +/// index's key — letting a tampered source manifest plant a file during restore. +/// Fails closed; every legitimate manifest passes. +fn ensure_plain_file_name(kind: &str, file_name: &str) -> Result<()> { + let is_unsafe = file_name.is_empty() + || file_name.contains('/') + || file_name.contains('\\') + || file_name == "." + || file_name == ".."; + if is_unsafe { + return Err(ArtifactStoreError::Integrity(format!( + "{kind} sub-manifest has an unsafe artifact file name {file_name:?}; \ + expected a plain file name" + ))); + } + Ok(()) +} + +/// Reads an unsigned integer field from a manifest body, defaulting to 0 for a +/// missing/non-integer value (matching the engine's own manifest accessors). +fn body_u64(body: &Value, key: &str) -> u64 { + body.get(key).and_then(Value::as_u64).unwrap_or(0) +} + +/// Whether a sub-manifest object flag is present and true. +fn truthy_map(object: &Map, key: &str) -> bool { + object.get(key).and_then(Value::as_bool).unwrap_or(false) +} + +/// Reads a string field from a manifest object, defaulting to empty. +fn str_field(object: &Map, key: &str) -> String { + object + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() +} + +/// Reads an unsigned integer field from a manifest object, defaulting to 0. +fn u64_field(object: &Map, key: &str) -> u64 { + object.get(key).and_then(Value::as_u64).unwrap_or(0) +} diff --git a/crates/lodedb-cloud-core/src/lib.rs b/crates/lodedb-cloud-core/src/lib.rs new file mode 100644 index 00000000..bc20c5c1 --- /dev/null +++ b/crates/lodedb-cloud-core/src/lib.rs @@ -0,0 +1,83 @@ +//! Native Rust core for OreCloud. +//! +//! OreCloud's durable transfer/backup operates on LodeDB's open, generation- +//! addressed commit format. Rather than reimplement (or import the hollowing-out +//! Python implementation of) that format, this crate links `lodedb-core`'s +//! `storage` module directly, so the cloud and the embedded engine share ONE +//! commit-format implementation — schema, on-disk layout, canonical-JSON +//! checksum, and every sub-manifest (`json`/`tvim`/`tvtext`/`tvlex`/`tvmv`/`tvann`/`tvvf`). +//! +//! It provides an [`ArtifactStore`] abstraction with a filesystem default +//! ([`LocalArtifactStore`]), a read-only generation inventory +//! ([`inventory_committed_generation`]/[`diff_inventories`]), committed-generation +//! transfer that ships only committed state ([`export_generation`], filtered by an +//! explicit [`TransferPolicy`]; a restore is the same call with the stores +//! swapped), plus +//! [`verify_generation`] and [`compare_generations`] for integrity checks and +//! push status. This is the Rust *data plane*; auth/catalog/control-plane concerns +//! live in a separate (non-Rust) service. + +pub mod artifact_store; +pub mod blob_layout; +pub mod client_ops; +mod digest; +pub mod error; +pub mod generation_inventory; +pub mod local_artifact_store; +pub mod managed; +pub mod manifest_transfer; +pub mod object_artifact_store; +mod paths; +pub mod snapshot_identity; +pub mod status; +pub mod store_target; +pub mod sync_plan; +pub mod sync_state; +pub mod transfer_policy; +pub mod verify; + +pub use artifact_store::ArtifactStore; +pub use blob_layout::{blob_name, parse_blob_name}; +pub use client_ops::{PullOutcome, SyncForce, SyncOutcome}; +pub use error::{ArtifactStoreError, Result}; +pub use generation_inventory::{ + diff_inventories, inventory_committed_generation, inventory_from_body, list_index_keys, + ArtifactRef, GenerationInventory, InventoryDiff, +}; +pub use local_artifact_store::LocalArtifactStore; +pub use managed::{ + managed_materialize, managed_plan, managed_pull_requirements, managed_record_base, + ManagedLocal, ManagedPlan, ManagedPullOutcome, ManagedSide, +}; +pub use manifest_transfer::{export_generation, TransferResult}; +pub use object_artifact_store::ObjectArtifactStore; +pub use snapshot_identity::{logical_id, snapshot_id}; +pub use status::{compare_generations, status_for_push, StatusReport}; +pub use store_target::artifact_store_from_target; +pub use sync_plan::{classify, SnapRef, SyncClassification}; +pub use sync_state::{read_sync_state, write_sync_state, SidecarRead, SyncState}; +pub use transfer_policy::TransferPolicy; +pub use verify::{verify_generation, verify_local_generation_opens, OpenReport, VerifyReport}; + +/// Returns the commit-manifest schema version understood by the linked +/// `lodedb-core`. +/// +/// OreCloud pins to this value and fails closed on a mismatch: a store written +/// by a newer engine schema must never be silently mis-transferred. Reading it +/// straight from the linked core (rather than hard-coding `1`) means the pin +/// tracks the exact engine OreCloud builds against. +pub fn linked_core_schema_version() -> i64 { + lodedb_core::storage::commit_manifest::COMMIT_MANIFEST_SCHEMA_VERSION +} + +#[cfg(test)] +mod tests { + use super::linked_core_schema_version; + + /// Proves the git dependency on `lodedb-core` resolves, compiles, and that + /// the engine's public commit-format surface is reachable from OreCloud. + #[test] + fn links_lodedb_core_commit_manifest() { + assert_eq!(linked_core_schema_version(), 1); + } +} diff --git a/crates/lodedb-cloud-core/src/local_artifact_store.rs b/crates/lodedb-cloud-core/src/local_artifact_store.rs new file mode 100644 index 00000000..60486e81 --- /dev/null +++ b/crates/lodedb-cloud-core/src/local_artifact_store.rs @@ -0,0 +1,147 @@ +//! Local-filesystem [`ArtifactStore`]: the default backend. +//! +//! A committed generation already lives on disk as immutable `g.*` +//! artifacts under `.gen/` pinned by `.commit.json`, so the local store +//! is a thin wrapper over `lodedb-core`'s commit-manifest primitives — there is +//! no second format. Object-storage backends (S3/GCS/Azure) belong in a later +//! milestone, not here. + +use crate::artifact_store::{body_generation, ArtifactStore}; +use crate::digest::sha256_hex; +use crate::error::{ArtifactStoreError, Result}; +use crate::paths::resolve_within; +use lodedb_core::storage::commit_manifest::{ + commit_manifest_path, read_commit_manifest, write_commit_manifest, +}; +use serde_json::Value; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// Stores artifacts as files under a directory; the pointer is `.commit.json`. +pub struct LocalArtifactStore { + root: PathBuf, + fsync: bool, +} + +impl LocalArtifactStore { + /// Binds the store to a persistence directory (the same directory a `LodeDB` + /// handle persists into). + /// + /// `fsync` mirrors the engine durability flag: when true, each artifact write + /// and the pointer swap are fsynced (file + directory) so a pushed artifact + /// survives power loss; the default (false) keeps the fast + /// atomic-but-not-durable path. + pub fn new(root: impl Into, fsync: bool) -> Self { + Self { + root: root.into(), + fsync, + } + } +} + +/// Atomic publish: write to a sibling `.tmp`, fsync (gated), rename into place, +/// then fsync the parent directory so the rename is durable. This is the engine's +/// `durable_replace` discipline — never write a persisted artifact in place. +fn atomic_write(path: &Path, data: &[u8], fsync: bool) -> Result<()> { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("artifact"); + let tmp = path.with_file_name(format!("{file_name}.tmp")); + let mut handle = File::create(&tmp)?; + handle.write_all(data)?; + if fsync { + handle.sync_all()?; + } + drop(handle); + fs::rename(&tmp, path)?; + if fsync { + if let Some(parent) = path.parent() { + File::open(parent)?.sync_all()?; + } + } + Ok(()) +} + +impl ArtifactStore for LocalArtifactStore { + fn read_bytes(&self, name: &str) -> Result> { + let path = resolve_within(&self.root, &self.root.join(name))?; + fs::read(&path).map_err(|error| match error.kind() { + std::io::ErrorKind::NotFound => ArtifactStoreError::NotFound(name.to_string()), + _ => ArtifactStoreError::Io(error), + }) + } + + fn write_bytes_if_absent(&self, name: &str, data: &[u8], sha256: &str) -> Result<()> { + // Verify the incoming bytes before any write, so corruption can never be + // stored. + let digest = sha256_hex(data); + if digest != sha256 { + return Err(ArtifactStoreError::Integrity(format!( + "artifact {name:?} failed checksum: expected {sha256}, computed {digest}" + ))); + } + let path = resolve_within(&self.root, &self.root.join(name))?; + if path.exists() { + // Names are epoch-addressed, not sha-derived, so two independent + // lineages can collide on a name. Identical bytes are an idempotent + // no-op; different bytes are a genuine conflict we refuse rather than + // clobber (the immutability invariant). + let existing = sha256_hex(&fs::read(&path)?); + if existing == sha256 { + return Ok(()); + } + return Err(ArtifactStoreError::Integrity(format!( + "artifact {name:?} already exists with different content (stored {existing}, \ + incoming {sha256}); refusing to overwrite an immutable artifact" + ))); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + atomic_write(&path, data, self.fsync) + } + + fn contains(&self, name: &str) -> Result { + // A metadata probe, not a read — `try_exists` surfaces genuine I/O + // failures (unlike `Path::exists`, which would mask them as absence). + let path = resolve_within(&self.root, &self.root.join(name))?; + Ok(path.try_exists()?) + } + + fn read_pointer(&self, key: &str) -> Result> { + let pointer = resolve_within(&self.root, &commit_manifest_path(&self.root, key))?; + // `read_commit_manifest` validates the schema version and body checksum, + // failing closed on a garbled pointer. + Ok(read_commit_manifest(&pointer)?.map(|manifest| manifest.body)) + } + + fn compare_and_swap_pointer( + &self, + key: &str, + old_body: Option<&Value>, + new_body: &Value, + ) -> Result<()> { + // On a local filesystem this is a read-check-then-replace: the final + // rename inside `write_commit_manifest` is atomic per file, but the + // read+swap pair is not a true cross-process CAS. That is safe under + // LodeDB's single-writer model and for out-of-band backup use; an + // object-store backend must instead use a real conditional write. + // + // The precondition compares the full committed body, not just its + // generation number: two lineages can share a generation with different + // content, so a numeric check would be ABA-prone. + let pointer = resolve_within(&self.root, &commit_manifest_path(&self.root, key))?; + let current = read_commit_manifest(&pointer)?.map(|manifest| manifest.body); + if current.as_ref() != old_body { + return Err(ArtifactStoreError::PointerConflict { + key: key.to_string(), + expected: old_body.and_then(body_generation), + found: current.as_ref().and_then(body_generation), + }); + } + write_commit_manifest(&pointer, new_body, self.fsync)?; + Ok(()) + } +} diff --git a/crates/lodedb-cloud-core/src/managed.rs b/crates/lodedb-cloud-core/src/managed.rs new file mode 100644 index 00000000..7aa78046 --- /dev/null +++ b/crates/lodedb-cloud-core/src/managed.rs @@ -0,0 +1,431 @@ +//! Client-side composition for the managed (`orecloud://`) transfer plane. +//! +//! Managed transfers split the client edge in two: the Python layer speaks +//! HTTP to the control plane (begin/commit push sessions, pull plans, +//! presigned or proxied blob bytes), while everything that touches the commit +//! format stays here — inventories, canonical identities, the engine-written +//! pointer document, sidecar trust, classification, and the restore path with +//! its verify-open. The seam is deliberate: Python never parses a manifest or +//! re-serialises a body, so the canonical-JSON contract has exactly one +//! implementation (the engine's), exactly as the dumb-target verbs already +//! guarantee. +//! +//! The remote's committed state arrives as the head *body* the control plane +//! returned (stored and served as JSON). Identities recomputed from that body +//! are stable across the trip because the engine's canonical form is +//! key-order-insensitive: `serde_json::Value` objects are sorted maps, so a +//! body that passed through the catalog re-canonicalises to the same digest — +//! pinned by the end-to-end tests. +//! +//! Unlike dumb targets, the remote identity string used for sidecar trust is +//! supplied by the caller verbatim (`orecloud://org/environment/db#host=…`): the +//! Python layer owns the spelling of managed remotes, and the sidecar records +//! and compares exactly that string. + +use crate::client_ops::{ensure_no_pending_wal, snap_ref}; +use crate::error::{ArtifactStoreError, Result}; +use crate::generation_inventory::{ + diff_inventories, inventory_from_body, write_restored_journal_manifests, ArtifactRef, +}; +use crate::local_artifact_store::LocalArtifactStore; +use crate::manifest_transfer::{publish_staged, stage_generation_pinned, TransferResult}; +use crate::snapshot_identity::{identity_from_document, pointer_document}; +use crate::status::{compare_generations, StatusReport}; +use crate::sync_plan::{classify, SnapRef}; +use crate::sync_state::{read_sync_state, write_sync_state, SyncState}; +use crate::transfer_policy::TransferPolicy; +use crate::verify::{verify_candidate_opens, OpenReport}; +use crate::ArtifactStore; +use serde_json::Value; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// One side's identity and payload masks, as the wire contract speaks them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedSide { + pub snapshot_id: String, + pub logical_id: String, + pub generation: u64, + pub has_text: bool, + pub has_lexical: bool, +} + +/// The local half of a push: identity, the exact body/pointer document a push +/// would publish, and the full artifact set backing it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedLocal { + pub side: ManagedSide, + /// The policy-redacted committed body — what a push publishes. + pub body: Value, + /// The engine-canonical pointer document for `body` (UTF-8 JSON bytes, + /// carried as a string). Shipped verbatim at commit so the control plane + /// can maintain the object-store pointer mirror without re-serialising. + pub pointer_document: String, + /// Every artifact the body pins (the begin-push inventory). + pub artifacts: Vec, +} + +/// Everything the Python edge needs to run one managed status/push/sync +/// decision: the byte-level status report (lineage fields filled), both ends' +/// identities, and the trusted base. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedPlan { + pub report: StatusReport, + pub local: Option, + pub remote: Option, + pub base: Option, + /// Whether the trusted base already records the remote's current snapshot + /// — when false after an `in_sync` classification, the caller should + /// refresh the sidecar (mirrors `client_ops::sync`'s stale-base repair). + pub base_is_current: bool, + /// The RAW (unredacted) local pointer's snapshot id at classification + /// time; `None` when the directory holds no committed generation. A + /// pull-direction materialization pins on this so a local commit landing + /// between classification and materialization refuses instead of being + /// silently overwritten — the managed twin of the dumb sync carrying its + /// classified `local_raw` into the pointer CAS. + pub local_raw_snapshot_id: Option, +} + +/// What one managed pull produced (mirrors [`crate::PullOutcome`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedPullOutcome { + pub transfer: TransferResult, + pub open: OpenReport, +} + +fn require_local_dir(dir: &str) -> Result<()> { + if dir.contains("://") { + return Err(ArtifactStoreError::Backend(format!( + "a managed transfer's local end must be a directory path, got {dir:?}" + ))); + } + Ok(()) +} + +fn managed_side(body: &Value, reference: &SnapRef) -> ManagedSide { + ManagedSide { + snapshot_id: reference.snapshot_id.clone(), + logical_id: reference.logical_id.clone(), + generation: reference.generation, + has_text: body.get("tvtext").is_some_and(|value| !value.is_null()), + has_lexical: body.get("tvlex").is_some_and(|value| !value.is_null()), + } +} + +/// Builds the full decision context for one managed index: reads the local +/// committed generation (redacted by `policy`), computes identities and the +/// pointer document, inventories both ends, reads the sidecar (trusted only +/// when recorded against exactly `remote_id`), and classifies. +/// +/// `remote_body` is the branch-head body the control plane returned (`None` +/// when the remote holds nothing). Pure local I/O — no network. +pub fn managed_plan( + dir: &str, + index_key: &str, + remote_id: &str, + remote_body: Option, + policy: TransferPolicy, +) -> Result { + require_local_dir(dir)?; + let local_store = LocalArtifactStore::new(dir, false); + + let local_raw = local_store.read_pointer(index_key)?; + let local_raw_snapshot_id = local_raw + .as_ref() + .map(crate::snapshot_identity::snapshot_id) + .transpose()?; + let local_body = local_raw.map(|body| policy.redact_body(&body)); + let local_inventory = inventory_from_body(index_key, local_body.as_ref())?; + let remote_inventory = inventory_from_body(index_key, remote_body.as_ref())?; + let mut report = compare_generations( + index_key, + local_inventory.as_ref(), + remote_inventory.as_ref(), + ); + + let sidecar = read_sync_state(Path::new(dir), index_key)?; + let base = sidecar + .state + .as_ref() + .filter(|state| state.remote == remote_id) + .map(|state| state.base.clone()); + + let local_ref = local_body.as_ref().map(snap_ref).transpose()?; + let remote_ref = remote_body.as_ref().map(snap_ref).transpose()?; + let classification = classify(local_ref.as_ref(), base.as_ref(), remote_ref.as_ref()); + + report.sidecar_present = base.is_some(); + report.sidecar_corrupt = sidecar.corrupt; + report.base_generation = base.as_ref().map(|base| base.generation); + report.classification = Some(classification.as_str().to_string()); + + let base_is_current = match (&base, &remote_ref) { + (Some(base), Some(remote)) => base.snapshot_id == remote.snapshot_id, + _ => false, + }; + + let local = match (local_body, local_ref, local_inventory) { + (Some(body), Some(reference), Some(inventory)) => { + let document = pointer_document(&body)?; + // Cross-check the document against the recomputed identity: both + // come from the same engine writer, so a mismatch means the body + // changed between the two reads — fail closed rather than begin a + // push whose commit will contradict itself. + let document_id = identity_from_document(&document)?; + if document_id != reference.snapshot_id { + return Err(ArtifactStoreError::Integrity(format!( + "pointer document identity {document_id} does not match the snapshot id \ + {} computed from the same body", + reference.snapshot_id + ))); + } + let document = String::from_utf8(document).map_err(|error| { + ArtifactStoreError::Integrity(format!( + "engine-written pointer document is not UTF-8: {error}" + )) + })?; + Some(ManagedLocal { + side: managed_side(&body, &reference), + body, + pointer_document: document, + artifacts: inventory.artifacts, + }) + } + _ => None, + }; + let remote = match (&remote_body, &remote_ref) { + (Some(body), Some(reference)) => Some(managed_side(body, reference)), + _ => None, + }; + + Ok(ManagedPlan { + report, + local, + remote, + base, + base_is_current, + local_raw_snapshot_id, + }) +} + +/// Records `body` as the sidecar base for `remote_id` after a successful +/// managed transfer. `remote_id` is stored verbatim — the caller owns the +/// canonical spelling of managed remote identities. +pub fn managed_record_base( + dir: &str, + index_key: &str, + remote_id: &str, + body: &Value, +) -> Result<()> { + require_local_dir(dir)?; + let state = SyncState { + index_key: index_key.to_string(), + remote: remote_id.to_string(), + base: snap_ref(body)?, + updated_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs()) + .unwrap_or(0), + }; + write_sync_state(Path::new(dir), &state) +} + +/// The artifacts a pull of `body` must download: everything the remote body +/// pins that the local directory does not already hold byte-identically. +pub fn managed_pull_requirements( + dir: &str, + index_key: &str, + body: &Value, +) -> Result> { + require_local_dir(dir)?; + let local_store = LocalArtifactStore::new(dir, false); + let remote_inventory = inventory_from_body(index_key, Some(body))? + .expect("inventory is Some when the body is Some"); + let local_body = local_store.read_pointer(index_key)?; + let local_inventory = inventory_from_body(index_key, local_body.as_ref())?; + Ok(diff_inventories(&remote_inventory, local_inventory.as_ref()).to_upload) +} + +/// A read-only [`ArtifactStore`] over a staging directory of downloaded +/// blobs, addressed by content: `read_bytes(name)` resolves the engine +/// artifact name to its sha256 via the pull body's inventory and reads +/// `/`. Write operations are unreachable by construction +/// (it is only ever a transfer *source*) and fail closed if reached. +struct StagingBlobStore { + root: std::path::PathBuf, + body: Value, + sha_by_name: HashMap, +} + +impl StagingBlobStore { + fn new(staging_dir: &Path, index_key: &str, body: Value) -> Result { + let inventory = inventory_from_body(index_key, Some(&body))? + .expect("inventory is Some when the body is Some"); + let sha_by_name = inventory + .artifacts + .into_iter() + .map(|artifact| (artifact.name, artifact.sha256)) + .collect(); + Ok(Self { + root: staging_dir.to_path_buf(), + body, + sha_by_name, + }) + } + + fn staged_path(&self, name: &str) -> Result { + let sha = self.sha_by_name.get(name).ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "artifact {name:?} is not referenced by the pull body" + )) + })?; + // The inventory already validated every digest as 64 lowercase hex + // characters; re-assert here (defense in depth) so a digest can never + // traverse out of the staging root even if a future inventory change + // loosens that guarantee. + crate::blob_layout::validate_sha256(sha)?; + Ok(self.root.join(sha)) + } +} + +impl ArtifactStore for StagingBlobStore { + fn read_bytes(&self, name: &str) -> Result> { + let path = self.staged_path(name)?; + std::fs::read(&path).map_err(|error| match error.kind() { + std::io::ErrorKind::NotFound => ArtifactStoreError::NotFound(format!( + "blob for artifact {name:?} was not downloaded into the staging directory" + )), + _ => ArtifactStoreError::Io(error), + }) + } + + fn write_bytes_if_absent(&self, name: &str, _data: &[u8], _sha256: &str) -> Result<()> { + Err(ArtifactStoreError::Backend(format!( + "staging blob store is read-only (attempted write of {name:?})" + ))) + } + + fn contains(&self, name: &str) -> Result { + Ok(self.staged_path(name)?.try_exists()?) + } + + fn read_pointer(&self, _key: &str) -> Result> { + Ok(Some(self.body.clone())) + } + + fn compare_and_swap_pointer( + &self, + _key: &str, + _old_body: Option<&Value>, + _new_body: &Value, + ) -> Result<()> { + Err(ArtifactStoreError::Backend( + "staging blob store is read-only (attempted pointer swap)".to_string(), + )) + } +} + +/// Materialises a managed pull: restores `body`'s generation into `dir` from +/// content-addressed blobs previously downloaded into `staging_dir` +/// (`/` per blob), proves the restored copy opens +/// through the engine BEFORE the pointer moves, and records the sidecar base +/// against `remote_id`. +/// +/// The restore runs under the engine's single-writer lock (a live writer and +/// a restore must never interleave) and refuses when the destination WAL +/// still holds acknowledged operations — pass `discard_pending_wal` (the +/// force-pull semantics) to truncate them along with the local lineage. Every +/// blob is re-hashed against the manifest checksum on write (a corrupt +/// download fails the pull before any pointer moves), the candidate is +/// verify-opened from a scratch layout before publication, and the local +/// pointer swap preconditions on the committed body read at the start (a +/// concurrent engine commit surfaces as a `PointerConflict` instead of being +/// clobbered). +pub fn managed_materialize( + dir: &str, + index_key: &str, + remote_id: &str, + body: Value, + staging_dir: &str, + discard_pending_wal: bool, + expected_local_snapshot_id: Option<&str>, +) -> Result { + require_local_dir(dir)?; + let staging = StagingBlobStore::new(Path::new(staging_dir), index_key, body.clone())?; + let local_store = LocalArtifactStore::new(dir, false); + fs::create_dir_all(dir)?; + let _writer_lock = lodedb_core::engine::acquire_dir_writer_lock(Path::new(dir)) + .map_err(ArtifactStoreError::Core)?; + if !discard_pending_wal { + ensure_no_pending_wal( + dir, + index_key, + "checkpoint them by opening the store once, or re-run the sync with \ + --force-pull to discard them along with the local lineage", + )?; + } + let local_raw = local_store.read_pointer(index_key)?; + // Pin the materialization to the local state the caller CLASSIFIED, not + // the state found now: a commit landing between classification and this + // lock acquisition must refuse (re-run the sync to re-classify) rather + // than be silently overwritten. `None` skips the pin (a plain pull's + // decision IS the state read under this lock); the empty string pins to + // "classified as absent" (a fresh clone), any other value to that exact + // raw snapshot id. + if let Some(expected) = expected_local_snapshot_id { + let current = local_raw + .as_ref() + .map(crate::snapshot_identity::snapshot_id) + .transpose()?; + let unchanged = if expected.is_empty() { + current.is_none() + } else { + current.as_deref() == Some(expected) + }; + if !unchanged { + return Err(ArtifactStoreError::SyncConflict { + classification: "stale".to_string(), + hint: "the local store committed a new generation after this sync \ + classified it; re-run the sync to reconcile" + .to_string(), + }); + } + } + // The managed remote absorbs same-name forks (blobs are content- + // addressed there), but the LOCAL directory still stores artifacts by + // engine name: force-pulling a diverged lineage that reuses a name with + // different bytes fails closed on the immutability invariant, exactly + // like the dumb-target verbs — recover by pulling into a fresh + // directory. The wrapper attaches that recovery hint. + let staged = stage_generation_pinned( + &staging, + &local_store, + index_key, + TransferPolicy::full(), + body, + local_raw, + ) + .map_err(crate::client_ops::explain_fork_collision)?; + // Acceptance checks run against a scratch candidate BEFORE the pointer + // moves (the scratch carries its own journal manifests, so the real + // destination stays untouched until the swap). + let open = verify_candidate_opens(Path::new(dir), index_key, &staged.source_body)?; + let (transfer, restored) = publish_staged(&local_store, index_key, staged)?; + if discard_pending_wal { + lodedb_core::storage::wal::truncate( + &crate::client_ops::contained_wal_path(dir, index_key)?, + false, + )?; + } + // Rebuild the journal manifests (working state the body doesn't pin) so + // the restored copy is writable, not just readable — the cloud writer + // opens hydrated copies through this path. Strictly AFTER the swap: + // writing them first would, on a failed CAS, leave the candidate's + // journals attached to some other writer's committed body. + write_restored_journal_manifests(Path::new(dir), index_key, &restored)?; + managed_record_base(dir, index_key, remote_id, &restored)?; + Ok(ManagedPullOutcome { transfer, open }) +} diff --git a/crates/lodedb-cloud-core/src/manifest_transfer.rs b/crates/lodedb-cloud-core/src/manifest_transfer.rs new file mode 100644 index 00000000..09035b83 --- /dev/null +++ b/crates/lodedb-cloud-core/src/manifest_transfer.rs @@ -0,0 +1,235 @@ +//! Export a committed generation between two artifact stores. +//! +//! Copies exactly one committed generation from one [`ArtifactStore`] to another, +//! then publishes the destination's root pointer. Because the source is read +//! through its committed +//! root pointer (never the `.wal` tail or the on-disk per-store manifests), only +//! committed state ships — an uncommitted WAL write or a torn half-commit is +//! excluded by construction. +//! +//! The copy is O(changed): the destination's existing inventory is diffed against +//! the source's, so only missing artifacts are uploaded. The destination verifies +//! each artifact's checksum on write, and the pointer swap is the single commit +//! point, so a crash mid-transfer leaves the destination on its previous +//! generation with some extra immutable blobs — never a torn generation. +//! +//! Both ends are artifact stores, so a transfer is symmetric: a restore is the +//! same call with the stores swapped — the remote/backup store as `source` and +//! the local directory as `dest` (a source that was pushed redacted already omits +//! text, so [`TransferPolicy::full`] on the way back restores it verbatim). Follow +//! a local restore with +//! [`verify_local_generation_opens`](crate::verify_local_generation_opens) to +//! confirm the engine can open it. +//! +//! Every transfer states its [`TransferPolicy`] explicitly: a redacted push +//! publishes a body whose text/lexical sub-manifests are nulled and skips those +//! artifacts, so only redacted state leaves the machine; a full policy ships the +//! generation verbatim. There is deliberately no default-policy convenience — +//! shipping payload-bearing stores must be a visible choice at the call site. + +use crate::artifact_store::ArtifactStore; +use crate::error::{ArtifactStoreError, Result}; +use crate::generation_inventory::{diff_inventories, inventory_from_body}; +use crate::transfer_policy::TransferPolicy; + +/// Metrics-only summary of one generation transfer. +/// +/// Carries counts and bytes only (no ids, text, or vectors), so it is safe to log +/// or surface in control-plane telemetry. `artifacts_written` are the blobs +/// actually uploaded; `artifacts_skipped` were already present at the +/// destination; `pointer_published` is false when the destination already pointed +/// at this generation (an idempotent no-op transfer). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransferResult { + pub index_key: String, + pub generation: u64, + pub artifacts_written: usize, + pub artifacts_skipped: usize, + pub bytes_written: u64, + pub pointer_published: bool, +} + +/// Copies `index_key`'s committed generation from `source` to `dest` under +/// `policy`. +/// +/// A redacted `policy` publishes a body whose excluded text/lexical sub-manifests +/// are nulled and ships none of their artifacts, so the destination generation +/// genuinely omits that payload (see [`TransferPolicy::redact_body`]); a +/// [`TransferPolicy::full`] policy ships the generation verbatim. Reads the +/// source's committed root, diffs it against whatever the destination already +/// holds, uploads only the missing artifacts (each checksum-verified by the +/// destination), and finally swaps the destination's root pointer — the one +/// commit point. Idempotent: re-running when the destination already holds the +/// exact body this transfer would publish uploads nothing and leaves the pointer +/// untouched. +/// +/// Returns [`ArtifactStoreError::NotFound`] when the source has no committed +/// generation for `index_key`. The pointer swap may return +/// [`ArtifactStoreError::PointerConflict`] if a concurrent writer advanced the +/// destination between the read and the swap. +pub fn export_generation( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, +) -> Result { + export_generation_with_body(source, dest, index_key, policy).map(|(result, _)| result) +} + +/// [`export_generation`], additionally returning the committed body the +/// destination now points at (the policy-redacted source body). +/// +/// The sync layer records that body's identity in the sidecar; returning it +/// from the transfer itself (rather than re-reading the source afterwards) +/// means the recorded base is exactly what was published, even if a concurrent +/// engine commit advances the source mid-call. Crate-internal: the public +/// result type stays metrics-only (safe to log), which a full body is not. +pub(crate) fn export_generation_with_body( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, +) -> Result<(TransferResult, serde_json::Value)> { + let dest_body = dest.read_pointer(index_key)?; + export_generation_against(source, dest, index_key, policy, dest_body) +} + +/// [`export_generation_with_body`] with the destination's committed body +/// supplied by the caller instead of read here. +/// +/// The pointer swap preconditions on exactly `dest_body`, so a caller that +/// already read (and classified) the destination — the sync verb — makes its +/// transfer conditional on the state it decided on: a concurrent advance of +/// the destination between that read and this call surfaces as a +/// [`PointerConflict`](ArtifactStoreError::PointerConflict), with the +/// atomicity the destination backend's `compare_and_swap_pointer` provides +/// (a true conditional write on object stores; read-check-then-replace on a +/// directory, per that impl's documented single-writer contract). +pub(crate) fn export_generation_against( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, + dest_body: Option, +) -> Result<(TransferResult, serde_json::Value)> { + let raw_body = source.read_pointer(index_key)?.ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "no committed generation to export for index key {index_key:?}" + )) + })?; + export_generation_pinned(source, dest, index_key, policy, raw_body, dest_body) +} + +/// [`export_generation_against`] with the *source's* committed body supplied +/// too, instead of re-read here. +/// +/// The sync verb classifies a specific pair of bodies; pinning the source +/// means the transfer ships exactly the classified snapshot — a concurrent +/// source change cannot swap in a body the classifier never saw. The +/// artifact bytes are still read live by name, but every name/checksum comes +/// from the pinned body and the destination re-hashes on write, so a mutated +/// source artifact fails the transfer rather than shipping unclassified +/// bytes. +pub(crate) fn export_generation_pinned( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, + raw_body: serde_json::Value, + dest_body: Option, +) -> Result<(TransferResult, serde_json::Value)> { + let staged = stage_generation_pinned(source, dest, index_key, policy, raw_body, dest_body)?; + publish_staged(dest, index_key, staged) +} + +/// Everything a staged (artifacts-uploaded, pointer-untouched) transfer knows: +/// the body a publish would commit, the destination body the publish must +/// precondition on, and the upload metrics. +/// +/// Splitting staging from publication lets local restores insert their +/// acceptance checks (candidate verify-open, journal reconstruction) between +/// the two, so a failed check leaves the destination on its previous committed +/// generation — the pointer swap stays the single commit point. +pub(crate) struct StagedExport { + pub source_body: serde_json::Value, + pub dest_body: Option, + pub generation: u64, + pub artifacts_total: usize, + pub artifacts_written: usize, + pub bytes_written: u64, +} + +/// Uploads the missing artifact set for a pinned transfer WITHOUT touching the +/// destination pointer. +/// +/// The destination re-hashes each artifact on write, so a corrupt source +/// artifact fails here — before any pointer moves. Redaction happens before +/// inventorying, exactly as in [`export_generation_pinned`]. +pub(crate) fn stage_generation_pinned( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, + raw_body: serde_json::Value, + dest_body: Option, +) -> Result { + // Redact before inventorying so excluded artifacts are neither listed nor + // uploaded, and the published body matches exactly what shipped. + let source_body = policy.redact_body(&raw_body); + let source_inventory = inventory_from_body(index_key, Some(&source_body))? + .expect("inventory is Some when the body is Some"); + + let dest_inventory = inventory_from_body(index_key, dest_body.as_ref())?; + let diff = diff_inventories(&source_inventory, dest_inventory.as_ref()); + + for artifact in &diff.to_upload { + let data = source.read_bytes(&artifact.name)?; + dest.write_bytes_if_absent(&artifact.name, &data, &artifact.sha256)?; + } + + Ok(StagedExport { + source_body, + dest_body, + generation: source_inventory.generation, + artifacts_total: source_inventory.artifacts.len(), + artifacts_written: diff.to_upload.len(), + bytes_written: diff.upload_bytes, + }) +} + +/// Publishes a staged transfer's pointer — the single commit point. +/// +/// Skips the swap only when the destination already holds this exact committed +/// *body* — comparing the full body, not just the generation integer, because +/// two independent lineages can share a generation number with different +/// content. The swap preconditions on the exact body staging read +/// (`dest_body`), so a concurrent change between read and swap is caught as a +/// PointerConflict. +pub(crate) fn publish_staged( + dest: &dyn ArtifactStore, + index_key: &str, + staged: StagedExport, +) -> Result<(TransferResult, serde_json::Value)> { + let StagedExport { + source_body, + dest_body, + generation, + artifacts_total, + artifacts_written, + bytes_written, + } = staged; + let pointer_published = dest_body.as_ref() != Some(&source_body); + if pointer_published { + dest.compare_and_swap_pointer(index_key, dest_body.as_ref(), &source_body)?; + } + + let result = TransferResult { + index_key: index_key.to_string(), + generation, + artifacts_written, + artifacts_skipped: artifacts_total - artifacts_written, + bytes_written, + pointer_published, + }; + Ok((result, source_body)) +} diff --git a/crates/lodedb-cloud-core/src/object_artifact_store.rs b/crates/lodedb-cloud-core/src/object_artifact_store.rs new file mode 100644 index 00000000..eb36b5f1 --- /dev/null +++ b/crates/lodedb-cloud-core/src/object_artifact_store.rs @@ -0,0 +1,253 @@ +//! Object-storage [`ArtifactStore`]: the cloud backend (S3/GCS/Azure). +//! +//! Wraps any [`object_store::ObjectStore`] so a committed generation ships to +//! object storage under the same artifact names it has on disk. Two properties +//! the local filesystem gave for free must be re-established here: +//! +//! - **Atomic root-pointer commit.** Object stores have no `os.replace`, so the +//! pointer swap uses the backend's *conditional write*: `PutMode::Create` (fails +//! if the pointer exists) and `PutMode::Update{e_tag}` (fails unless the pointer +//! still matches the version we read). That is a real strongly-consistent +//! compare-and-swap, never an eventually-consistent list-then-write. +//! - **Per-tenant isolation.** Content addressing deduplicates by checksum, so in a +//! shared bucket every name is prefixed with a per-tenant key. A caller can only +//! reach blobs under its own prefix; a checksum known to one tenant cannot fetch +//! another tenant's object. +//! +//! The [`ArtifactStore`] trait is synchronous, but `object_store` is async, so this +//! type owns a current-thread Tokio runtime and blocks on each operation. Its +//! consumers (transfer/verify) are sequential batch operations, and the managed +//! serving tier hydrates to a local directory before opening the engine, so a sync +//! surface fits every M2 caller. + +use crate::artifact_store::{body_generation, ArtifactStore}; +use crate::digest::sha256_hex; +use crate::error::{ArtifactStoreError, Result}; +use lodedb_core::storage::commit_manifest::read_commit_manifest; +use object_store::path::Path as ObjectPath; +use object_store::{Error as ObjectError, ObjectStore, PutMode, PutOptions, UpdateVersion}; +use serde_json::Value; +use std::io::Write; +use std::sync::Arc; +use tempfile::NamedTempFile; +use tokio::runtime::{Builder, Runtime}; + +/// Stores artifacts as objects under a per-tenant prefix; the pointer is +/// `/.commit.json`, swapped with a conditional write. +pub struct ObjectArtifactStore { + store: Arc, + prefix: String, + runtime: Runtime, +} + +impl ObjectArtifactStore { + /// Binds the store to `object_store` under `prefix` (the per-tenant namespace). + /// + /// `prefix` is prepended to every artifact name and pointer key; pass an empty + /// string for a single-tenant bucket. Fails only if the current-thread Tokio + /// runtime cannot be constructed. + pub fn new(store: Arc, prefix: impl Into) -> Result { + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| { + ArtifactStoreError::Backend(format!("failed to build async runtime: {error}")) + })?; + Ok(Self { + store, + prefix: prefix.into(), + runtime, + }) + } + + /// Joins a store-relative artifact name onto the tenant prefix to form the + /// object key. `object_store`'s `Path` normalises the segments, so a name can + /// never traverse above the prefix. + fn object_path(&self, name: &str) -> ObjectPath { + if self.prefix.is_empty() { + ObjectPath::from(name) + } else { + ObjectPath::from(format!("{}/{}", self.prefix.trim_end_matches('/'), name)) + } + } + + /// The pointer object key for an index key: `/.commit.json`. + fn pointer_path(&self, key: &str) -> ObjectPath { + self.object_path(&format!("{key}.commit.json")) + } + + /// Fetches an object's bytes, mapping a missing object to `None` rather than an + /// error, so callers can distinguish absence from a transport failure. + fn get_optional(&self, path: &ObjectPath) -> Result> { + match self.runtime.block_on(self.store.get(path)) { + Ok(result) => { + let version = UpdateVersion { + e_tag: result.meta.e_tag.clone(), + version: result.meta.version.clone(), + }; + let bytes = self + .runtime + .block_on(result.bytes()) + .map_err(map_backend_error)?; + Ok(Some(GetBytes { + bytes: bytes.to_vec(), + version, + })) + } + Err(ObjectError::NotFound { .. }) => Ok(None), + Err(error) => Err(map_backend_error(error)), + } + } +} + +/// An object's bytes plus the version handle needed for a conditional overwrite. +struct GetBytes { + bytes: Vec, + version: UpdateVersion, +} + +impl ArtifactStore for ObjectArtifactStore { + fn read_bytes(&self, name: &str) -> Result> { + let path = self.object_path(name); + self.get_optional(&path)? + .map(|got| got.bytes) + .ok_or_else(|| ArtifactStoreError::NotFound(name.to_string())) + } + + fn write_bytes_if_absent(&self, name: &str, data: &[u8], sha256: &str) -> Result<()> { + // Verify before any upload so corruption is never stored. + let digest = sha256_hex(data); + if digest != sha256 { + return Err(ArtifactStoreError::Integrity(format!( + "artifact {name:?} failed checksum: expected {sha256}, computed {digest}" + ))); + } + let path = self.object_path(name); + let options = PutOptions { + mode: PutMode::Create, + ..PutOptions::default() + }; + match self + .runtime + .block_on(self.store.put_opts(&path, data.to_vec().into(), options)) + { + Ok(_) => Ok(()), + // The name already exists. Identical bytes are an idempotent no-op; + // different bytes are a genuine conflict we refuse rather than clobber + // (artifacts are immutable/content-addressed). + Err(ObjectError::AlreadyExists { .. }) => { + let existing = self.read_bytes(name)?; + if sha256_hex(&existing) == sha256 { + Ok(()) + } else { + Err(ArtifactStoreError::Integrity(format!( + "artifact {name:?} already exists with different content; refusing to \ + overwrite an immutable artifact" + ))) + } + } + Err(error) => Err(map_backend_error(error)), + } + } + + fn contains(&self, name: &str) -> Result { + // A HEAD request: presence without fetching the object's bytes. + let path = self.object_path(name); + match self.runtime.block_on(self.store.head(&path)) { + Ok(_) => Ok(true), + Err(ObjectError::NotFound { .. }) => Ok(false), + Err(error) => Err(map_backend_error(error)), + } + } + + fn read_pointer(&self, key: &str) -> Result> { + let path = self.pointer_path(key); + match self.get_optional(&path)? { + Some(got) => Ok(Some(validate_pointer_document(&got.bytes)?)), + None => Ok(None), + } + } + + fn compare_and_swap_pointer( + &self, + key: &str, + old_body: Option<&Value>, + new_body: &Value, + ) -> Result<()> { + let path = self.pointer_path(key); + let document = serialize_pointer_document(new_body)?; + let current = self.get_optional(&path)?; + + // Precondition on the full committed body (a generation number is not a + // unique version token). The e_tag from the same read then arms the + // conditional put below as a second, race-tight guard. + let current_body = current + .as_ref() + .map(|got| validate_pointer_document(&got.bytes)) + .transpose()?; + if current_body.as_ref() != old_body { + return Err(ArtifactStoreError::PointerConflict { + key: key.to_string(), + expected: old_body.and_then(body_generation), + found: current_body.as_ref().and_then(body_generation), + }); + } + + // Conditional write: create when the pointer must not yet exist, else + // overwrite only if it still matches the version we just read. Either + // precondition failure means a concurrent writer moved the pointer between + // our read and this write. + let mode = match ¤t { + Some(got) => PutMode::Update(got.version.clone()), + None => PutMode::Create, + }; + let options = PutOptions { + mode, + ..PutOptions::default() + }; + match self + .runtime + .block_on(self.store.put_opts(&path, document.into(), options)) + { + Ok(_) => Ok(()), + Err(ObjectError::AlreadyExists { .. } | ObjectError::Precondition { .. }) => { + Err(ArtifactStoreError::PointerConflict { + key: key.to_string(), + expected: old_body.and_then(body_generation), + // The read-side body is stale by now; report the generation we + // expected and let the caller re-read to learn the new value. + found: current_body.as_ref().and_then(body_generation), + }) + } + Err(error) => Err(map_backend_error(error)), + } + } +} + +/// Serialises a committed body into pointer-document bytes — the shared +/// engine-writer round-trip in [`snapshot_identity`](crate::snapshot_identity). +fn serialize_pointer_document(body: &Value) -> Result> { + crate::snapshot_identity::pointer_document(body) +} + +/// Validates pointer-document bytes and returns the committed body, reusing the +/// engine's schema + `body_sha256` validation via a scratch file. +/// +/// Fails closed on a garbled pointer (the engine's `read_commit_manifest` rejects +/// a bad schema version or body checksum), so a corrupted object never loads as a +/// valid generation. +fn validate_pointer_document(bytes: &[u8]) -> Result { + let mut scratch = NamedTempFile::new()?; + scratch.write_all(bytes)?; + scratch.flush()?; + let manifest = read_commit_manifest(scratch.path())?.ok_or_else(|| { + ArtifactStoreError::Integrity("pointer document is empty or absent".to_string()) + })?; + Ok(manifest.body) +} + +/// Maps a non-`NotFound`, non-precondition object-store error to a backend error, +/// preserving the backend's own diagnostic. +fn map_backend_error(error: ObjectError) -> ArtifactStoreError { + ArtifactStoreError::Backend(error.to_string()) +} diff --git a/crates/lodedb-cloud-core/src/paths.rs b/crates/lodedb-cloud-core/src/paths.rs new file mode 100644 index 00000000..149e0a02 --- /dev/null +++ b/crates/lodedb-cloud-core/src/paths.rs @@ -0,0 +1,153 @@ +//! Path-containment guard for filesystem-backed artifact stores. +//! +//! A committed generation's artifact *names* and pointer *keys* are joined onto a +//! store root to form on-disk paths. Once a name or key can originate from CLI or +//! remote/control-plane input (the cloud trust boundary), a crafted `..` segment +//! or absolute path could otherwise read or write outside the store root — a +//! cross-tenant disclosure in a multi-tenant store. [`resolve_within`] is the +//! single primitive that confines every such join to its root. + +use crate::error::{ArtifactStoreError, Result}; +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; + +/// Canonicalises `candidate` and proves it stays within `root`. +/// +/// Follows symlinks through the candidate's existing prefix (matching Python's +/// `Path.resolve`) and normalises `.`/`..` in any not-yet-created tail, then +/// asserts the result is `root` itself or beneath it. Resolving *first* is what +/// makes this sound: a purely lexical `starts_with` would read `root/../etc` as +/// "under root", and a symlink inside the tree could redirect outside it. An +/// absolute `candidate` discards `root` and resolves outside it, so it is +/// rejected too. Returns an `Integrity` error on any escape (defence against +/// CWE-22 path traversal). +/// +/// `root` need not exist yet: a fresh backup destination resolves to the path it +/// will occupy (its deepest existing ancestor, canonicalised, plus the pending +/// tail), so a not-yet-created store directory is accepted and created on first +/// write rather than being rejected with `ENOENT`. +pub fn resolve_within(root: &Path, candidate: &Path) -> Result { + let root_real = canonical_resolve(root)?; + let resolved = canonical_resolve(candidate)?; + if resolved == root_real || resolved.starts_with(&root_real) { + Ok(resolved) + } else { + Err(ArtifactStoreError::Integrity(format!( + "path {candidate:?} escapes store root {root_real:?}" + ))) + } +} + +/// A path's canonical identity string: absolute, symlink-resolved (through the +/// existing prefix), `.`/`..`-normalised — the same resolution +/// [`resolve_within`] uses. Two spellings of one directory (relative vs +/// absolute, redundant `.` segments) map to one identity. On a resolution +/// failure the raw spelling is returned, which for the sidecar-trust caller can +/// only produce a false *mismatch* (fails toward force, never toward trusting +/// the wrong remote). +pub(crate) fn canonical_identity(candidate: &Path) -> String { + // A bare relative path must anchor to the invocation cwd explicitly, or a + // fully-nonexistent path would stay relative through the lexical fallback. + let absolute = if candidate.is_absolute() { + candidate.to_path_buf() + } else { + match std::env::current_dir() { + Ok(cwd) => cwd.join(candidate), + Err(_) => return candidate.to_string_lossy().into_owned(), + } + }; + match canonical_resolve(&absolute) { + Ok(resolved) => resolved.to_string_lossy().into_owned(), + Err(_) => candidate.to_string_lossy().into_owned(), + } +} + +/// Resolves `candidate` to an absolute, normalised path. +/// +/// `std::fs::canonicalize` requires the whole path to exist, but artifact and +/// pointer paths are routinely resolved before they are created. So we +/// canonicalise the deepest existing ancestor (resolving symlinks like Python's +/// `resolve`) and re-append the not-yet-created tail, then collapse any `.`/`..` +/// lexically. +fn canonical_resolve(candidate: &Path) -> Result { + let mut existing = candidate.to_path_buf(); + let mut tail: Vec = Vec::new(); + loop { + if existing.exists() { + let mut resolved = existing.canonicalize()?; + for part in tail.iter().rev() { + resolved.push(part); + } + return Ok(lexical_normalize(&resolved)); + } + match existing.file_name() { + Some(name) => { + tail.push(name.to_os_string()); + if !existing.pop() { + break; + } + } + None => break, + } + } + Ok(lexical_normalize(candidate)) +} + +/// Collapses `.` and `..` components purely lexically (no filesystem access), so +/// a `..` in the not-yet-created tail cannot lift the path above the resolved +/// prefix without the containment check catching it. +fn lexical_normalize(path: &Path) -> PathBuf { + let mut out = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + other => out.push(other.as_os_str()), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::resolve_within; + use crate::error::ArtifactStoreError; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + + static COUNTER: AtomicU64 = AtomicU64::new(0); + + /// A unique, created temp directory (std-only, no dev-dependency). + fn temp_dir(label: &str) -> PathBuf { + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("orecloud-paths-{label}-{}-{n}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn accepts_a_path_inside_the_root() { + let root = temp_dir("inside"); + let resolved = resolve_within(&root, &root.join("idx.gen/g0.json")).unwrap(); + assert!(resolved.starts_with(root.canonicalize().unwrap())); + } + + #[test] + fn rejects_a_parent_traversal_escape() { + let root = temp_dir("escape"); + let err = resolve_within(&root, &root.join("../secrets")).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + } + + #[test] + fn accepts_a_not_yet_created_root() { + // A fresh backup destination that does not exist yet must resolve, not + // fail with ENOENT — the store creates it on first write. + let root = temp_dir("fresh-parent").join("new-backup"); + let resolved = resolve_within(&root, &root.join("idx.commit.json")).unwrap(); + assert!(resolved.ends_with("new-backup/idx.commit.json")); + } +} diff --git a/crates/lodedb-cloud-core/src/snapshot_identity.rs b/crates/lodedb-cloud-core/src/snapshot_identity.rs new file mode 100644 index 00000000..139beb5f --- /dev/null +++ b/crates/lodedb-cloud-core/src/snapshot_identity.rs @@ -0,0 +1,85 @@ +//! Content identity for a committed generation body. +//! +//! Sync needs to ask "are these two committed generations the same?" across +//! machines and stores, where generation *numbers* are not unique version +//! tokens (two independent lineages can share one). The answer is the engine's +//! own canonical body digest: +//! +//! - [`snapshot_id`] — the `body_sha256` the engine records in +//! `.commit.json` for exactly this body. Two stores hold the same bytes +//! iff their snapshot ids match; storage and authorization decisions key on +//! it. +//! - [`logical_id`] — the snapshot id of the body's fully *redacted* form. A +//! full push and a redacted push of one engine commit share a `logical_id` +//! while their `snapshot_id`s differ, so lineage comparison ("is this the +//! same commit?") runs on `logical_id` and never mistakes a redaction for a +//! divergence. +//! +//! The digest is over the engine's canonical JSON, which this crate must never +//! reimplement. `body_sha256` itself is `pub(crate)` in `lodedb-core`, but +//! `write_commit_manifest` emits `{"body":…,"body_sha256":"…",…}`, so a +//! scratch-file round-trip through the engine's own writer yields the exact +//! canonical digest — the same trick `ObjectArtifactStore` uses for pointer +//! documents. +//! +//! TODO(upstream): ask lodedb-core to export `body_sha256(body)` directly and +//! drop the scratch-file round-trip. + +use crate::error::{ArtifactStoreError, Result}; +use crate::transfer_policy::TransferPolicy; +use lodedb_core::storage::commit_manifest::write_commit_manifest; +use serde_json::Value; +use tempfile::NamedTempFile; + +/// Serialises `body` into the engine's canonical pointer-document bytes — +/// the exact `{"body":…,"body_sha256":…,"schema_version":…}` file a +/// `.commit.json` pointer carrying this body holds on disk. +/// +/// Routing through the engine's own `write_commit_manifest` (via a scratch +/// file) guarantees the body checksum and schema envelope are byte-identical +/// to an engine-written pointer, so `read_commit_manifest` validates the +/// result the same way. The managed transfer plane ships these bytes to the +/// control plane, which stores them verbatim for the object-store pointer +/// mirror — the server never re-serialises a pointer itself. +pub fn pointer_document(body: &Value) -> Result> { + let scratch = NamedTempFile::new()?; + write_commit_manifest(scratch.path(), body, false)?; + Ok(std::fs::read(scratch.path())?) +} + +/// Extracts the canonical `body_sha256` from pointer-document bytes produced +/// by [`pointer_document`]. +pub fn identity_from_document(document: &[u8]) -> Result { + let document: Value = serde_json::from_slice(document).map_err(|error| { + ArtifactStoreError::Integrity(format!( + "engine-written pointer document failed to parse: {error}" + )) + })?; + document + .get("body_sha256") + .and_then(Value::as_str) + .filter(|digest| !digest.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + ArtifactStoreError::Integrity( + "engine-written pointer document carries no body_sha256".to_string(), + ) + }) +} + +/// Returns the engine's canonical `body_sha256` for `body` — the digest a +/// `.commit.json` pointer carrying this exact body records. +pub fn snapshot_id(body: &Value) -> Result { + identity_from_document(&pointer_document(body)?) +} + +/// Returns the snapshot id of `body`'s fully redacted form. +/// +/// Redaction ([`TransferPolicy::redacted`]) nulls the payload-bearing stores, +/// and redacting an already-redacted body is a no-op, so every push of one +/// engine commit — full, redacted, or partial — maps to one `logical_id`. A +/// corollary used by the sync classifier: a body is fully redacted iff its +/// `snapshot_id` equals its `logical_id`. +pub fn logical_id(body: &Value) -> Result { + snapshot_id(&TransferPolicy::redacted().redact_body(body)) +} diff --git a/crates/lodedb-cloud-core/src/status.rs b/crates/lodedb-cloud-core/src/status.rs new file mode 100644 index 00000000..0bb08b60 --- /dev/null +++ b/crates/lodedb-cloud-core/src/status.rs @@ -0,0 +1,143 @@ +//! Compare a local generation against a remote one. +//! +//! [`compare_generations`] answers "what would a push move, and are the two ends +//! already in sync?" purely from two inventories — no bytes are read. It composes +//! [`diff_inventories`](crate::diff_inventories), so the O(changed) upload set and +//! the base-vs-delta signal are computed the same way the transfer computes them. +//! +//! The comparison is direction-aware (push: local -> remote) and compares the two +//! inventories exactly as given. A caller applying a [`TransferPolicy`] should +//! build the local inventory from the *redacted* body, so `in_sync` reflects the +//! redacted push it intends to make — [`status_for_push`] does exactly that over +//! two stores. `in_sync` means a push would move nothing at +//! all: no artifacts to upload *and* the remote's committed body already equals the +//! body the push would publish. The body check matters because a push republishes +//! the pointer whenever the bodies differ even when no bytes move — e.g. a redacted +//! push against a previously-full remote uploads nothing yet must still swap the +//! pointer to drop the text reference, so it is correctly reported as not in sync. + +use crate::artifact_store::ArtifactStore; +use crate::error::Result; +use crate::generation_inventory::{diff_inventories, inventory_from_body, GenerationInventory}; +use crate::transfer_policy::TransferPolicy; + +/// Metrics-only comparison of a local and a remote committed generation. +/// +/// `local_*`/`remote_*` are `None` when that side holds no committed generation. +/// `artifacts_to_upload`/`bytes_to_upload`/`ships_base` describe a push from local +/// to remote (the O(changed) upload set). `in_sync` is true when a push would move +/// nothing — the remote already holds every artifact the local side would ship. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatusReport { + pub index_key: String, + pub local_generation: Option, + pub remote_generation: Option, + pub local_document_count: Option, + pub remote_document_count: Option, + pub local_chunk_count: Option, + pub remote_chunk_count: Option, + pub artifacts_to_upload: usize, + pub bytes_to_upload: u64, + pub ships_base: bool, + pub in_sync: bool, + /// Whether a *trusted* sync sidecar for this remote was found next to the + /// local index (valid checksum, recorded against the same remote target). + /// Always false from [`compare_generations`]/[`status_for_push`] — the + /// sidecar lives on the local filesystem, so only the client edge + /// ([`client_ops::status`](crate::client_ops::status)) can fill lineage. + pub sidecar_present: bool, + /// Whether a sidecar file was found but failed validation (torn or + /// tampered) and was therefore ignored. Distinct from a sidecar recorded + /// against a different remote, which is valid — just not trusted here. + pub sidecar_corrupt: bool, + /// The recorded base generation from the sidecar, when one was trusted. + pub base_generation: Option, + /// The three-pointer [`SyncClassification`](crate::sync_plan::SyncClassification) + /// name (`in_sync`/`local_ahead`/…), when lineage was evaluated. `None` + /// from the store-level comparison, which has no sidecar to consult. + pub classification: Option, +} + +/// Reads both ends and compares them for a push of `index_key` under `policy`. +/// +/// The local body is redacted by `policy` *before* inventorying, so the report +/// describes the push the caller would actually make (a redacted push neither +/// counts nor ships text/lexical artifacts, and `in_sync` compares against the +/// redacted body the push would publish). Read-only on both stores. +pub fn status_for_push( + source: &dyn ArtifactStore, + dest: &dyn ArtifactStore, + index_key: &str, + policy: TransferPolicy, +) -> Result { + let local_body = source + .read_pointer(index_key)? + .map(|body| policy.redact_body(&body)); + let local = inventory_from_body(index_key, local_body.as_ref())?; + let remote_body = dest.read_pointer(index_key)?; + let remote = inventory_from_body(index_key, remote_body.as_ref())?; + Ok(compare_generations( + index_key, + local.as_ref(), + remote.as_ref(), + )) +} + +/// Compares `local` against `remote` for a push (local -> remote). +/// +/// `index_key` labels the report (the inventories carry it too, but either side +/// may be `None`). When `local` is `None` there is nothing to push, so the report +/// is empty and `in_sync` is true. When `local` is `Some`, the upload set is +/// `diff_inventories(local, remote)` and `in_sync` is true iff that set is empty +/// *and* the remote already holds the exact body the push would publish (so the +/// pointer would not be re-swapped). +pub fn compare_generations( + index_key: &str, + local: Option<&GenerationInventory>, + remote: Option<&GenerationInventory>, +) -> StatusReport { + let Some(local) = local else { + // Nothing local to push: the report reflects only the remote side. + return StatusReport { + index_key: index_key.to_string(), + local_generation: None, + remote_generation: remote.map(|inventory| inventory.generation), + local_document_count: None, + remote_document_count: remote.map(|inventory| inventory.document_count), + local_chunk_count: None, + remote_chunk_count: remote.map(|inventory| inventory.chunk_count), + artifacts_to_upload: 0, + bytes_to_upload: 0, + ships_base: false, + in_sync: true, + sidecar_present: false, + sidecar_corrupt: false, + base_generation: None, + classification: None, + }; + }; + + let diff = diff_inventories(local, remote); + // A push moves nothing only when there are no artifacts to upload AND the + // remote already holds the exact body the push would publish — otherwise the + // pointer is still re-swapped (e.g. a redaction that drops a store reference + // without uploading any bytes). + let body_matches_remote = remote.is_some_and(|remote| remote.root_body == local.root_body); + StatusReport { + index_key: index_key.to_string(), + local_generation: Some(local.generation), + remote_generation: remote.map(|inventory| inventory.generation), + local_document_count: Some(local.document_count), + remote_document_count: remote.map(|inventory| inventory.document_count), + local_chunk_count: Some(local.chunk_count), + remote_chunk_count: remote.map(|inventory| inventory.chunk_count), + artifacts_to_upload: diff.to_upload.len(), + bytes_to_upload: diff.upload_bytes, + ships_base: diff.ships_base, + in_sync: diff.to_upload.is_empty() && body_matches_remote, + sidecar_present: false, + sidecar_corrupt: false, + base_generation: None, + classification: None, + } +} diff --git a/crates/lodedb-cloud-core/src/store_target.rs b/crates/lodedb-cloud-core/src/store_target.rs new file mode 100644 index 00000000..510e72ce --- /dev/null +++ b/crates/lodedb-cloud-core/src/store_target.rs @@ -0,0 +1,55 @@ +//! Resolve a user-facing transfer target into an [`ArtifactStore`]. +//! +//! The client edge (CLI / Python binding) names each end of a transfer with one +//! string — a local directory path or an object-store URL — so the mapping from +//! that string to a store lives here, once, rather than in every frontend: +//! +//! - `s3://bucket/prefix` → [`ObjectArtifactStore`] over Amazon S3 or any +//! S3-compatible endpoint (MinIO, R2, …). Credentials, region, and endpoint come +//! from the standard `AWS_*` environment variables (`AWS_ACCESS_KEY_ID`, +//! `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, `AWS_ENDPOINT`, `AWS_ALLOW_HTTP`), +//! read by `AmazonS3Builder::from_env`. The URL path becomes the per-tenant key +//! prefix. +//! - anything without a `://` scheme → [`LocalArtifactStore`] on that directory. +//! +//! Other object-store schemes (`gs://`, `az://`) are rejected until there is a +//! deployment that needs them — adding one is a new match arm plus a Cargo +//! feature, never a change to the transfer code. + +use crate::artifact_store::ArtifactStore; +use crate::error::{ArtifactStoreError, Result}; +use crate::local_artifact_store::LocalArtifactStore; +use crate::object_artifact_store::ObjectArtifactStore; +use object_store::aws::AmazonS3Builder; +use std::sync::Arc; + +/// Resolves `target` (a local directory path or an `s3://bucket/prefix` URL) +/// into an artifact store. +/// +/// Local stores are opened without fsync, matching the engine's default +/// durability. Returns a `Backend` error for an unsupported URL scheme or an S3 +/// URL with no bucket, and surfaces the S3 builder's own error when the `AWS_*` +/// environment is incomplete (e.g. a missing region). +pub fn artifact_store_from_target(target: &str) -> Result> { + if let Some(rest) = target.strip_prefix("s3://") { + let (bucket, prefix) = rest.split_once('/').unwrap_or((rest, "")); + if bucket.is_empty() { + return Err(ArtifactStoreError::Backend(format!( + "s3 target {target:?} has no bucket; expected s3://bucket[/prefix]" + ))); + } + let s3 = AmazonS3Builder::from_env() + .with_bucket_name(bucket) + .build() + .map_err(|error| ArtifactStoreError::Backend(error.to_string()))?; + let store = ObjectArtifactStore::new(Arc::new(s3), prefix.trim_matches('/'))?; + return Ok(Box::new(store)); + } + if target.contains("://") { + return Err(ArtifactStoreError::Backend(format!( + "unsupported target scheme in {target:?}; expected a local directory path or \ + s3://bucket/prefix" + ))); + } + Ok(Box::new(LocalArtifactStore::new(target, false))) +} diff --git a/crates/lodedb-cloud-core/src/sync_plan.rs b/crates/lodedb-cloud-core/src/sync_plan.rs new file mode 100644 index 00000000..a029bf72 --- /dev/null +++ b/crates/lodedb-cloud-core/src/sync_plan.rs @@ -0,0 +1,225 @@ +//! Pure three-pointer sync classification — no I/O. +//! +//! Sync compares three snapshots of one index: the **local** committed +//! generation (as the caller's transfer policy would publish it), the **base** +//! recorded by the sidecar (the last state both ends agreed on), and the +//! **remote** committed generation. [`classify`] reduces those to one +//! [`SyncClassification`] that says which transfer — if any — is a +//! fast-forward, and which situations require an explicit force flag. +//! +//! Comparison runs on [`logical_id`](crate::snapshot_identity::logical_id) +//! (redaction-invariant content identity), never on generation numbers alone: +//! two independent lineages can share a generation number. Generations serve +//! only as a monotonicity sanity check, because single-base ancestry is +//! trust-based — the sidecar is a *claim* the local machine makes about +//! history, so a generation that moved backwards relative to the recorded base +//! is treated as rollback/tamper-suspect ([`Unknown`]) rather than +//! fast-forwarded. +//! +//! [`Unknown`]: SyncClassification::Unknown + +/// One side's identity: which exact bytes ([`snapshot_id`]), which engine +/// commit regardless of redaction ([`logical_id`]), the committed generation +/// number (monotonicity check only), and a per-store identity for each +/// payload-bearing store the body carries. +/// +/// The payload identities (`text_id`/`lexical_id`, `None` when the store is +/// absent) are what let the classifier compare two snapshots of the *same* +/// commit: the two stores are independent redaction choices +/// ([`TransferPolicy`]), so store-set inclusion decides between no-op and +/// republish — and because `logical_id` is computed with both stores nulled, +/// a store present on *both* ends must also match by identity, or the two +/// "same" snapshots disagree about payload content (different encodings, or +/// tampering) and no untransfer-free reconciliation exists. +/// +/// [`snapshot_id`]: crate::snapshot_identity::snapshot_id +/// [`logical_id`]: crate::snapshot_identity::logical_id +/// [`TransferPolicy`]: crate::TransferPolicy +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapRef { + pub snapshot_id: String, + pub logical_id: String, + pub generation: u64, + /// Identity of the raw-text store's sub-manifest; `None` when absent. + pub text_id: Option, + /// Identity of the lexical store's sub-manifest; `None` when absent. + pub lexical_id: Option, +} + +impl SnapRef { + /// Whether a store present on both snapshots differs by identity — the + /// "same commit, conflicting payload" case that must never be reconciled + /// silently. + fn shared_store_conflict(&self, other: &Self) -> bool { + let conflicts = |mine: &Option, theirs: &Option| matches!((mine, theirs), (Some(a), Some(b)) if a != b); + conflicts(&self.text_id, &other.text_id) || conflicts(&self.lexical_id, &other.lexical_id) + } + + /// Whether this snapshot's payload stores are a strict superset of + /// `other`'s: it carries at least one store `other` lacks, and lacks none + /// `other` carries. + fn carries_more_than(&self, other: &Self) -> bool { + let gains = (self.text_id.is_some() && other.text_id.is_none()) + || (self.lexical_id.is_some() && other.lexical_id.is_none()); + gains && other.carries_no_more_than(self) + } + + /// Whether this snapshot's payload stores are a (non-strict) subset of + /// `other`'s. + fn carries_no_more_than(&self, other: &Self) -> bool { + (self.text_id.is_none() || other.text_id.is_some()) + && (self.lexical_id.is_none() || other.lexical_id.is_some()) + } + + /// Whether discarding this snapshot in favor of something derived from + /// `base` loses nothing: the same commit, no payload store `base` lacks, + /// and no shared store whose content disagrees with `base`'s. + /// + /// This — not bare logical equality — is what makes a side "unchanged" + /// for fast-forward purposes: a side that enriched the base with payload + /// (an unpublished `--include-text` upgrade) has moved, even though its + /// logical id has not. + fn adds_nothing_over(&self, base: &Self) -> bool { + self.logical_id == base.logical_id + && !self.shared_store_conflict(base) + && self.carries_no_more_than(base) + } +} + +/// What a sync of one index would have to do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncClassification { + /// Local and remote already hold the same content; nothing to transfer. + InSync, + /// Remote still equals the recorded base and local moved forward: a push + /// is a fast-forward. + LocalAhead, + /// Local still equals the recorded base and remote moved forward: a pull + /// is a fast-forward. + RemoteAhead, + /// Both sides moved past the base independently; a transfer in either + /// direction discards the other side's commit, so force is required. + Diverged, + /// Local and remote are the same engine commit but the local copy carries + /// payload stores the remote lacks (and none the other way): a push + /// republishes the pointer (upgrading the remote) without any lineage + /// conflict. + Republish, + /// History cannot be trusted or the transfer cannot be lossless: no + /// recorded base while both sides hold different content, a generation + /// moved backwards relative to the base, or the two ends carry + /// incomparable payload-store sets of one commit. Force is required. + Unknown, +} + +impl SyncClassification { + /// The stable lowercase name reports and error messages use. + pub fn as_str(&self) -> &'static str { + match self { + Self::InSync => "in_sync", + Self::LocalAhead => "local_ahead", + Self::RemoteAhead => "remote_ahead", + Self::Diverged => "diverged", + Self::Republish => "republish", + Self::Unknown => "unknown", + } + } +} + +/// Classifies one index's sync state from its three pointers. +/// +/// `local` is the generation the caller *would publish* (i.e. already redacted +/// by its transfer policy), `base` is the sidecar's recorded last-synced state +/// (`None` when the sidecar is absent or corrupt), and `remote` is the remote's +/// committed generation. All lineage equality runs on `logical_id`. +/// +/// Only [`Diverged`](SyncClassification::Diverged) and +/// [`Unknown`](SyncClassification::Unknown) require force; every other +/// classification maps to at most one fast-forward transfer. +pub fn classify( + local: Option<&SnapRef>, + base: Option<&SnapRef>, + remote: Option<&SnapRef>, +) -> SyncClassification { + // Nothing anywhere (a recorded base with both ends gone included): there + // is no transfer to make. + if local.is_none() && remote.is_none() { + return SyncClassification::InSync; + } + + // Generation regression against the recorded base on any present side is + // rollback/tamper-suspect: never proceed silently over it — not for ends + // that agree with each other (a coordinated rollback must be loud, and a + // republish would publish payload onto a suspect remote), and not toward + // an absent side either (auto-pushing a rolled-back generation to a fresh + // remote would silently re-establish it as current). Checked before every + // other comparison. + if let Some(base) = base { + let regressed = + |side: Option<&SnapRef>| side.is_some_and(|side| side.generation < base.generation); + if regressed(local) || regressed(remote) { + return SyncClassification::Unknown; + } + } + + let (local, remote) = match (local, remote) { + // One side absent (and no regression above): the transfer direction + // is unambiguous — there is nothing on the other end to discard. + (Some(_), None) => return SyncClassification::LocalAhead, + (None, Some(_)) => return SyncClassification::RemoteAhead, + (Some(local), Some(remote)) => (local, remote), + (None, None) => unreachable!("handled above"), + }; + + if local.logical_id == remote.logical_id { + // Same engine commit; the ends should only differ by which payload + // stores they carry (the logical id nulls exactly those stores). + // First, a store present on BOTH ends must match by identity — a + // mismatch means the two "same" snapshots disagree about payload + // content, which no direction of transfer reconciles losslessly. + // Then compare the store sets by inclusion: + // - local carries stores the remote lacks (and nothing less): a push + // republishes the pointer, upgrading the remote without discarding + // anything; + // - local carries no more than the remote: nothing to publish, no-op; + // - incomparable (each carries a store the other lacks — text-only vs + // lexical-only): publishing either way drops a store the other end + // has, so force is required. + return if local.snapshot_id == remote.snapshot_id { + SyncClassification::InSync + } else if local.shared_store_conflict(remote) { + SyncClassification::Unknown + } else if local.carries_more_than(remote) { + SyncClassification::Republish + } else if local.carries_no_more_than(remote) { + SyncClassification::InSync + } else { + SyncClassification::Unknown + }; + } + + // Local and remote differ. Without a base there is no way to tell which + // side moved: fail toward force rather than guess a direction. + let Some(base) = base else { + return SyncClassification::Unknown; + }; + + // A fast-forward discards the "unchanged" side, so that side must add + // nothing over the base — same commit AND no payload the base lacks. A + // side that merely enriched the base with payload stores (an unpublished + // `--include-text` upgrade of the last-synced commit) has moved too: + // discarding it would silently drop that payload, so both-moved is + // Diverged even when one side's logical id still equals the base's. + match ( + local.adds_nothing_over(base), + remote.adds_nothing_over(base), + ) { + (false, true) => SyncClassification::LocalAhead, + (true, false) => SyncClassification::RemoteAhead, + (false, false) => SyncClassification::Diverged, + // Both ends equal the base while differing from each other is + // impossible (the same-logical case returned above); keep the + // conservative answer for completeness. + (true, true) => SyncClassification::InSync, + } +} diff --git a/crates/lodedb-cloud-core/src/sync_state.rs b/crates/lodedb-cloud-core/src/sync_state.rs new file mode 100644 index 00000000..46dfa2fb --- /dev/null +++ b/crates/lodedb-cloud-core/src/sync_state.rs @@ -0,0 +1,211 @@ +//! The sync sidecar: `/.orecloud`, the recorded base. +//! +//! Three-pointer sync needs a durable record of the last state local and +//! remote agreed on. That record lives *next to* the index it describes — a +//! small JSON sidecar beside `.commit.json` — because it is a claim about +//! this particular local copy's history, not about the remote (two clones of +//! one remote each carry their own base). +//! +//! Naming and placement are deliberate. The file deliberately does NOT end in +//! `.json`: the engine's load path globs `*.json` in the persistence directory +//! and treats anything that is not a commit manifest or a known metadata file +//! as a *legacy index snapshot* (verified against lodedb rev `f242d3a`, +//! `engine/core.py _load_persisted_indexes`), so a `.orecloud.json` +//! sidecar would break reopening the database. `.orecloud` is invisible +//! to that glob, and the engine's epoch GC removes files only under +//! `.gen/`, so a sidecar survives any number of commits. Both properties +//! are pinned by tests (the reopen-safety one from Python, where the real +//! engine runs). +//! +//! The sidecar is written atomically (temp file + rename, the same discipline +//! as [`LocalArtifactStore`](crate::LocalArtifactStore)'s pointer writes) and +//! carries a self-checksum over its payload. A torn, tampered, or +//! half-migrated sidecar therefore reads as *absent-but-corrupt* — which the +//! classifier maps to [`Unknown`](crate::sync_plan::SyncClassification::Unknown), +//! requiring an explicit force — rather than as a trusted base. + +use crate::digest::sha256_hex; +use crate::error::{ArtifactStoreError, Result}; +use crate::paths::resolve_within; +use crate::sync_plan::SnapRef; +use serde_json::{json, Value}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// The sidecar file suffix (`.orecloud`). Must never end in `.json` — +/// see the module docs on the engine's legacy-snapshot glob. +pub const SYNC_STATE_SUFFIX: &str = ".orecloud"; + +/// The sidecar schema this crate writes and accepts. +const SYNC_STATE_SCHEMA_VERSION: u64 = 1; + +/// The recorded base: what local and remote last agreed on, and where. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyncState { + pub index_key: String, + /// The remote target string the base was established against. + pub remote: String, + /// Identity of the last-synced generation. + pub base: SnapRef, + /// Unix seconds of the last sidecar write (informational). + pub updated_unix: u64, +} + +/// The result of reading a sidecar: the state if one was trustworthy, plus a +/// flag distinguishing "no sidecar" from "a sidecar was present but corrupt". +/// +/// Both cases classify the same way (no trusted base -> `Unknown` when the +/// ends differ), but a frontend should *warn* on corruption — it usually means +/// a torn write or manual tampering, not a fresh directory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SidecarRead { + pub state: Option, + pub corrupt: bool, +} + +/// The sidecar path for `key` under `dir`: `/.orecloud`. +pub fn sync_state_path(dir: &Path, key: &str) -> PathBuf { + dir.join(format!("{key}{SYNC_STATE_SUFFIX}")) +} + +/// Reads the sidecar for `key` under `dir`. +/// +/// An absent file is `{state: None, corrupt: false}`. Any parse failure, +/// schema mismatch, checksum mismatch, or an `index_key` that disagrees with +/// `key` is `{state: None, corrupt: true}` — the base is untrusted, never +/// partially honored. Only genuine I/O failures (permissions etc.) are errors. +pub fn read_sync_state(dir: &Path, key: &str) -> Result { + let path = resolve_within(dir, &sync_state_path(dir, key))?; + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(SidecarRead { + state: None, + corrupt: false, + }) + } + Err(error) => return Err(ArtifactStoreError::Io(error)), + }; + Ok(match parse_sidecar(&bytes, key) { + Some(state) => SidecarRead { + state: Some(state), + corrupt: false, + }, + None => SidecarRead { + state: None, + corrupt: true, + }, + }) +} + +/// Writes the sidecar for `state.index_key` under `dir`, atomically. +/// +/// Temp file + rename in the same directory, so a crash mid-write leaves +/// either the previous sidecar or a stray `.tmp` — never a torn document (and +/// a torn document would fail its self-checksum anyway). +pub fn write_sync_state(dir: &Path, state: &SyncState) -> Result<()> { + let path = resolve_within(dir, &sync_state_path(dir, &state.index_key))?; + let document = serde_json::to_vec_pretty(&document_json(state)).map_err(|error| { + ArtifactStoreError::Integrity(format!("failed to serialize sync sidecar: {error}")) + })?; + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + // A uniquely-named temp file (not a fixed `.tmp` sibling), so two + // concurrent syncs of one key cannot collide mid-write; the rename keeps + // the publish atomic either way, last writer wins. + let dir = path.parent().unwrap_or(Path::new(".")); + let mut scratch = tempfile::NamedTempFile::new_in(dir)?; + scratch.write_all(&document)?; + scratch + .persist(&path) + .map_err(|error| ArtifactStoreError::Io(error.error))?; + Ok(()) +} + +/// The checksummed payload — everything except the `sha256` field itself. +/// +/// Built with one fixed construction order shared by the write path and the +/// read-side verification, so the serialized bytes (and therefore the digest) +/// are identical regardless of how the document was produced. +fn payload_json(state: &SyncState) -> Value { + json!({ + "schema_version": SYNC_STATE_SCHEMA_VERSION, + "index_key": state.index_key, + "remote": state.remote, + "base": { + "snapshot_id": state.base.snapshot_id, + "logical_id": state.base.logical_id, + "generation": state.base.generation, + "text_id": state.base.text_id, + "lexical_id": state.base.lexical_id, + }, + "updated_unix": state.updated_unix, + }) +} + +/// The full on-disk document: the payload plus its self-checksum. +fn document_json(state: &SyncState) -> Value { + let payload = payload_json(state); + let digest = sha256_hex(&serde_json::to_vec(&payload).unwrap_or_default()); + let mut document = payload; + if let Some(object) = document.as_object_mut() { + object.insert("sha256".to_string(), Value::String(digest)); + } + document +} + +/// Parses and validates sidecar bytes; `None` means "do not trust this file". +fn parse_sidecar(bytes: &[u8], key: &str) -> Option { + let document: Value = serde_json::from_slice(bytes).ok()?; + let object = document.as_object()?; + if object.get("schema_version")?.as_u64()? != SYNC_STATE_SCHEMA_VERSION { + return None; + } + let base = object.get("base")?.as_object()?; + let state = SyncState { + index_key: object.get("index_key")?.as_str()?.to_string(), + remote: object.get("remote")?.as_str()?.to_string(), + base: SnapRef { + snapshot_id: nonempty(base.get("snapshot_id")?.as_str()?)?, + logical_id: nonempty(base.get("logical_id")?.as_str()?)?, + generation: base.get("generation")?.as_u64()?, + text_id: optional_id(base.get("text_id")?)?, + lexical_id: optional_id(base.get("lexical_id")?)?, + }, + updated_unix: object.get("updated_unix")?.as_u64()?, + }; + if state.index_key != key { + return None; + } + // Recompute the digest from the parsed fields via the same construction + // the writer used; a mismatch means torn/tampered bytes. + let recorded = object.get("sha256")?.as_str()?; + let expected = sha256_hex(&serde_json::to_vec(&payload_json(&state)).ok()?); + if recorded != expected { + return None; + } + Some(state) +} + +/// Rejects an empty identity string (a base with no id is no base). +fn nonempty(value: &str) -> Option { + if value.is_empty() { + None + } else { + Some(value.to_string()) + } +} + +/// Parses a nullable store-identity field: JSON `null` is an absent store +/// (`Some(None)`), a non-empty string is its identity, anything else is +/// corrupt (`None`, propagated by `?`). +#[allow(clippy::option_option)] +fn optional_id(value: &serde_json::Value) -> Option> { + match value { + serde_json::Value::Null => Some(None), + serde_json::Value::String(id) => Some(Some(nonempty(id)?)), + _ => None, + } +} diff --git a/crates/lodedb-cloud-core/src/transfer_policy.rs b/crates/lodedb-cloud-core/src/transfer_policy.rs new file mode 100644 index 00000000..39dc1d58 --- /dev/null +++ b/crates/lodedb-cloud-core/src/transfer_policy.rs @@ -0,0 +1,81 @@ +//! Which payload-bearing stores a transfer is allowed to ship. +//! +//! The redacted stores (`json`/`tvim`/`tvmv`/`tvann`/`tvvf`) carry no raw text and always ship — +//! they are the metadata and the vector/late-interaction index a restored copy +//! needs to answer searches. Two stores are payload-bearing and opt-in: +//! +//! - `tvtext` — the raw document text (`db.get(id)` content); +//! - `tvlex` — lexical terms, which are tokenised text and so payload-derived. +//! +//! A [`TransferPolicy`] gates those two. `tvmv` (late-interaction patch matrices) +//! is embedding data, not text, so it ships by default like `tvim` — as does +//! `tvvf`, the rescore original-vector sidecar (vectors, never text). +//! +//! Redaction rewrites the *committed body* rather than merely skipping bytes: a +//! redacted push publishes a body whose excluded sub-manifests are null, so the +//! remote generation genuinely has no text — a restore of it cannot resurrect +//! text that was never uploaded. + +use serde_json::Value; + +/// Whether a transfer ships the payload-bearing text and lexical stores. +/// +/// [`Default`] is the redacted posture (both off), matching the roadmap's default +/// that only redacted artifacts leave the machine. Every transfer states its +/// policy explicitly at the call site; pass [`TransferPolicy::full`] to ship a +/// generation verbatim (e.g. when restoring a backup). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TransferPolicy { + /// Ship the raw-text store (`tvtext` base + `.txd` deltas). + pub include_text: bool, + /// Ship the lexical-index store (`tvlex` base + `.lxd` deltas). + pub include_lexical: bool, +} + +impl TransferPolicy { + /// Ships every store, including text and lexical — a verbatim copy of the + /// committed generation. + pub fn full() -> Self { + Self { + include_text: true, + include_lexical: true, + } + } + + /// Ships only the redacted stores (no text, no lexical) — the default posture. + pub fn redacted() -> Self { + Self { + include_text: false, + include_lexical: false, + } + } + + /// Returns a copy of a committed body with the excluded stores nulled. + /// + /// Nulling a top-level store key reproduces exactly what + /// `build_commit_body` emits for an absent store, and the body checksum is + /// recomputed when the pointer is written, so the result is a valid, + /// self-consistent committed body describing a generation that omits those + /// stores. Cloning-and-nulling (rather than rebuilding via `build_commit_body`) + /// preserves every other field the engine put in the body, even ones this + /// crate does not model. A [`full`](Self::full) policy returns an unchanged + /// clone. + pub fn redact_body(&self, body: &Value) -> Value { + let mut body = body.clone(); + if let Some(object) = body.as_object_mut() { + if !self.include_text { + object.insert("tvtext".to_string(), Value::Null); + } + if !self.include_lexical { + object.insert("tvlex".to_string(), Value::Null); + } + } + body + } +} + +impl Default for TransferPolicy { + fn default() -> Self { + Self::redacted() + } +} diff --git a/crates/lodedb-cloud-core/src/verify.rs b/crates/lodedb-cloud-core/src/verify.rs new file mode 100644 index 00000000..d7f30b6e --- /dev/null +++ b/crates/lodedb-cloud-core/src/verify.rs @@ -0,0 +1,159 @@ +//! Verify a committed generation before trusting it. +//! +//! Two levels of assurance, both read-only: +//! +//! - [`verify_generation`] is store-agnostic (works on a remote/object store as +//! well as a local directory): it re-hashes every artifact the committed root +//! pins and compares against the recorded checksum, failing closed on the first +//! mismatch. Reading the pointer through the engine's `read_commit_manifest` +//! already validates the body checksum, so a corrupt root fails before any +//! artifact is read. +//! - [`verify_local_generation_opens`] is the strongest check for a restored +//! *local* copy: it opens the store read-only through `lodedb-core`'s own load +//! path, proving the committed manifest and its artifacts actually parse and +//! load as the engine would read them. + +use crate::artifact_store::ArtifactStore; +use crate::digest::sha256_hex; +use crate::error::{ArtifactStoreError, Result}; +use crate::generation_inventory::inventory_from_body; +use lodedb_core::storage::commit_manifest::{commit_manifest_path, write_commit_manifest}; +use lodedb_core::storage::{load_store, LoadOptions}; +use std::path::Path; + +/// Metrics-only summary of a checksum verification. +/// +/// Carries the generation and counts/bytes only (safe to log). A returned report +/// means every artifact matched its recorded checksum; a mismatch is an error, not +/// a report field. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifyReport { + pub index_key: String, + pub generation: u64, + pub artifacts_verified: usize, + pub bytes_verified: u64, +} + +/// Re-hashes every artifact `index_key`'s committed generation pins and compares +/// against the manifest's recorded checksum. +/// +/// Returns [`ArtifactStoreError::NotFound`] when the store holds no committed +/// generation for `index_key`, and [`ArtifactStoreError::Integrity`] on the first +/// artifact whose bytes do not match — failing closed rather than reporting a +/// partial success. Reading the pointer validates the body checksum as a +/// side effect (the engine's `read_commit_manifest` fails closed on a garbled +/// root), so this checks the whole chain: root body, then every referenced blob. +pub fn verify_generation(store: &dyn ArtifactStore, index_key: &str) -> Result { + let body = store.read_pointer(index_key)?.ok_or_else(|| { + ArtifactStoreError::NotFound(format!( + "no committed generation to verify for index key {index_key:?}" + )) + })?; + let inventory = inventory_from_body(index_key, Some(&body))? + .expect("inventory is Some when the body is Some"); + + let mut bytes_verified = 0u64; + for artifact in &inventory.artifacts { + let data = store.read_bytes(&artifact.name)?; + let digest = sha256_hex(&data); + if digest != artifact.sha256 { + return Err(ArtifactStoreError::Integrity(format!( + "artifact {:?} failed checksum: manifest records {}, computed {}", + artifact.name, artifact.sha256, digest + ))); + } + bytes_verified += data.len() as u64; + } + + Ok(VerifyReport { + index_key: index_key.to_string(), + generation: inventory.generation, + artifacts_verified: inventory.artifacts.len(), + bytes_verified, + }) +} + +/// Metrics-only summary of a successful read-only open. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpenReport { + pub index_key: String, + pub document_count: usize, + pub chunk_count: usize, +} + +/// Confirms a restored local generation opens read-only through the engine. +/// +/// This reuses `lodedb-core`'s `load_store` with `read_only` (which takes no writer +/// lock and reads the exact committed manifest, never the `.wal` tail), so it +/// proves a restored directory is loadable exactly as the embedded engine would +/// read it — the acceptance check the roadmap calls for after a restore. Returns +/// the loaded document/chunk counts. `persistence_dir` is the local directory the +/// generation was restored into. +pub fn verify_local_generation_opens( + persistence_dir: &Path, + index_key: &str, +) -> Result { + let store = load_store( + persistence_dir, + index_key, + LoadOptions { + read_only: true, + read_wal: false, + }, + )?; + Ok(OpenReport { + index_key: index_key.to_string(), + document_count: store.document_count(), + chunk_count: store.chunk_count(), + }) +} + +/// Proves a not-yet-committed candidate body opens through the engine WITHOUT +/// touching the destination's committed pointer. +/// +/// The restore flow stages the candidate's artifacts into `dir` first (they +/// are immutable and additive, so this is safe), then calls this: every +/// artifact the candidate pins is hardlinked (copied where linking fails) +/// into a scratch directory alongside reconstructed journal manifests and the +/// candidate pointer, and the scratch copy is verify-opened read-only. A +/// checksum-consistent but semantically invalid artifact therefore fails the +/// restore while the destination still points at its previous generation — +/// without this, the pointer swap would publish the broken generation before +/// the open check could reject it. +pub(crate) fn verify_candidate_opens( + dir: &Path, + index_key: &str, + body: &serde_json::Value, +) -> Result { + let inventory = inventory_from_body(index_key, Some(body))? + .expect("inventory is Some when the body is Some"); + // The scratch lives inside the destination so hardlinks stay on one + // filesystem; the dotted prefix keeps it invisible to the engine's + // top-level `*.json` legacy-snapshot glob, and TempDir removes it on drop. + let scratch = tempfile::Builder::new() + .prefix(".orecloud-verify-") + .tempdir_in(dir)?; + for artifact in &inventory.artifacts { + let source = crate::paths::resolve_within(dir, &dir.join(&artifact.name))?; + let target = scratch.path().join(&artifact.name); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent)?; + } + if std::fs::hard_link(&source, &target).is_err() { + // Filesystems without hardlinks (or cross-device edge cases): + // fall back to a byte copy — correctness over speed. + std::fs::copy(&source, &target)?; + } + } + crate::generation_inventory::write_restored_journal_manifests( + scratch.path(), + index_key, + body, + )?; + write_commit_manifest( + &commit_manifest_path(scratch.path(), index_key), + body, + false, + )?; + verify_local_generation_opens(scratch.path(), index_key) +} diff --git a/crates/lodedb-cloud-core/tests/artifact_store.rs b/crates/lodedb-cloud-core/tests/artifact_store.rs new file mode 100644 index 00000000..487bde73 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/artifact_store.rs @@ -0,0 +1,162 @@ +//! Tests for `LocalArtifactStore`: immutable byte I/O and root-pointer CAS. + +mod common; + +use common::*; +use lodedb_cloud_core::{ArtifactStore, ArtifactStoreError, LocalArtifactStore}; +use serde_json::Value; + +#[test] +fn write_and_read_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let data = b"generation-artifact-bytes"; + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha_hex(data)) + .unwrap(); + assert_eq!(store.read_bytes("idx.gen/g0.json").unwrap(), data.to_vec()); +} + +#[test] +fn checksum_mismatch_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let err = store + .write_bytes_if_absent("idx.gen/g0.json", b"data", &sha_hex(b"other")) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn rewrite_identical_bytes_is_noop() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let data = b"immutable"; + let sha = sha_hex(data); + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha) + .unwrap(); + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha) + .unwrap(); + assert_eq!(store.read_bytes("idx.gen/g0.json").unwrap(), data.to_vec()); +} + +#[test] +fn rewrite_different_bytes_conflicts() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + store + .write_bytes_if_absent("idx.gen/g0.json", b"first", &sha_hex(b"first")) + .unwrap(); + let err = store + .write_bytes_if_absent("idx.gen/g0.json", b"second", &sha_hex(b"second")) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + // The original bytes are preserved — never overwritten in place. + assert_eq!( + store.read_bytes("idx.gen/g0.json").unwrap(), + b"first".to_vec() + ); +} + +#[test] +fn read_missing_artifact_is_not_found() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let err = store.read_bytes("idx.gen/absent.json").unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); +} + +#[test] +fn read_pointer_returns_committed_body_or_none() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + assert!(store.read_pointer("idx").unwrap().is_none()); + + let json = store_sub(dir.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let body = write_json_commit(dir.path(), "idx", 1, 0, json); + assert_eq!(store.read_pointer("idx").unwrap(), Some(body)); +} + +#[test] +fn compare_and_swap_enforces_body_precondition() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let json = store_sub(dir.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let current = write_json_commit(dir.path(), "idx", 1, 0, json.clone()); + let next = commit_body("idx", 2, 0, json.clone()); + + // Expecting-absent-but-present -> conflict. + let err = store + .compare_and_swap_pointer("idx", None, &next) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::PointerConflict { .. })); + + // Expecting a body that is not the committed one -> conflict (even though a + // number-only check on its generation could have been made to pass). + let wrong = commit_body("idx", 9, 0, json); + let err = store + .compare_and_swap_pointer("idx", Some(&wrong), &next) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::PointerConflict { .. })); + + // Expecting the exact committed body -> swap succeeds and advances the pointer. + store + .compare_and_swap_pointer("idx", Some(¤t), &next) + .unwrap(); + assert_eq!( + store + .read_pointer("idx") + .unwrap() + .unwrap() + .get("generation") + .and_then(Value::as_u64), + Some(2) + ); +} + +#[test] +fn compare_and_swap_rejects_same_generation_different_body() { + // The ABA guard: a body sharing the committed generation *number* but carrying + // different content must NOT satisfy the precondition — a number is not a + // version token. A generation-only check would wrongly let this swap through. + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let json = store_sub(dir.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + write_json_commit(dir.path(), "idx", 1, 0, json); + + // A divergent lineage's body that also calls itself generation 1. + let other_gen1 = commit_body( + "idx", + 1, + 0, + serde_json::json!({ + "base": { "file_name": "g0.json", "sha256": "deadbeef", "file_bytes": 5 }, + "deltas": [], + }), + ); + let next = commit_body( + "idx", + 2, + 0, + serde_json::json!({ + "base": { "file_name": "g0.json", "sha256": "cafe", "file_bytes": 4 }, + "deltas": [], + }), + ); + let err = store + .compare_and_swap_pointer("idx", Some(&other_gen1), &next) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::PointerConflict { .. })); +} + +#[test] +fn read_pointer_on_nonexistent_root_is_none() { + // A store bound to a not-yet-created directory reads as empty rather than + // erroring — the precondition for exporting into a fresh backup target. + let parent = tempfile::tempdir().unwrap(); + let root = parent.path().join("not-created-yet"); + let store = LocalArtifactStore::new(&root, false); + assert!(store.read_pointer("idx").unwrap().is_none()); +} diff --git a/crates/lodedb-cloud-core/tests/client_ops.rs b/crates/lodedb-cloud-core/tests/client_ops.rs new file mode 100644 index 00000000..ae7cca26 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/client_ops.rs @@ -0,0 +1,105 @@ +//! Tests for the string-target client operations (`client_ops`): the facade the +//! CLI/binding calls. The typed primitives are covered by their own suites; +//! these tests pin the composition — target resolution, policy pass-through, +//! and pull's built-in open-verification. + +mod common; + +use common::*; +use lodedb_core::storage::{ + write_generation_commit, GenerationCommitInput, GenerationWriteOptions, +}; +use lodedb_cloud_core::client_ops::{keys, pull, push, status, verify}; +use lodedb_cloud_core::{ArtifactStoreError, TransferPolicy}; + +/// One engine-written committed generation in `dir` under `key`. +fn commit_engine_generation(dir: &std::path::Path, key: &str) { + let state = engine_state(key); + write_generation_commit( + dir, + GenerationCommitInput { + index_key: key, + generation: 1, + applied_lsn: 0, + base_epoch: 1, + state: &state, + tvim: None, + raw_text: None, + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: true, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); +} + +#[test] +fn push_status_pull_round_trip_by_target_strings() { + const KEY: &str = "1111dec251fa5e544784ac1af95b0ae6530cad714a2d34f8c4615740ecbf8205"; + let src = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + let restored = tempfile::tempdir().unwrap(); + commit_engine_generation(src.path(), KEY); + let src_s = src.path().to_str().unwrap(); + let remote_s = remote.path().to_str().unwrap(); + let restored_s = restored.path().to_str().unwrap(); + + assert_eq!(keys(src_s).unwrap(), vec![KEY.to_string()]); + + let before = status(src_s, remote_s, KEY, TransferPolicy::redacted()).unwrap(); + assert!(!before.in_sync); + + let pushed = push(src_s, remote_s, KEY, TransferPolicy::redacted()).unwrap(); + assert!(pushed.pointer_published); + + let after = status(src_s, remote_s, KEY, TransferPolicy::redacted()).unwrap(); + assert!(after.in_sync); + + verify(remote_s, KEY).unwrap(); + + // Pull restores AND proves the copy opens — one operation. + let outcome = pull(remote_s, restored_s, KEY).unwrap(); + assert!(outcome.transfer.pointer_published); + assert_eq!(outcome.open.index_key, KEY); +} + +#[test] +fn pull_into_an_unopenable_destination_fails_after_transfer() { + // A remote holding a pointer whose base artifact is missing: the transfer + // of the pointer alone "succeeds" at the store level, but pull must fail + // closed because the restored copy cannot actually open. This pins that the + // open-verification is genuinely part of pull, not a separate step. + const KEY: &str = "idx"; + let remote = tempfile::tempdir().unwrap(); + let restored = tempfile::tempdir().unwrap(); + // Hand-build a committed pointer referencing a base, but delete the base + // file so the artifact copy fails checksum/absence checks during pull. + let json = store_sub(remote.path(), KEY, "g0.json", b"state", ".json-delta", &[]); + write_json_commit(remote.path(), KEY, 1, 0, json); + std::fs::remove_file(remote.path().join("idx.gen/g0.json")).unwrap(); + + let err = pull( + remote.path().to_str().unwrap(), + restored.path().to_str().unwrap(), + KEY, + ) + .unwrap_err(); + assert!(matches!( + err, + ArtifactStoreError::NotFound(_) | ArtifactStoreError::Io(_) + )); + // The destination pointer was never published, so the failed pull left no + // half-restored generation behind. + let dest = lodedb_cloud_core::LocalArtifactStore::new(restored.path(), false); + use lodedb_cloud_core::ArtifactStore; + assert!(dest.read_pointer(KEY).unwrap().is_none()); +} + +#[test] +fn a_bad_target_scheme_is_rejected_up_front() { + let err = verify("ftp://nope/x", "idx").unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Backend(_))); +} diff --git a/crates/lodedb-cloud-core/tests/common/mod.rs b/crates/lodedb-cloud-core/tests/common/mod.rs new file mode 100644 index 00000000..db1b95cb --- /dev/null +++ b/crates/lodedb-cloud-core/tests/common/mod.rs @@ -0,0 +1,276 @@ +//! Shared fixtures for the artifact-store tests. +//! +//! Committed generations are built with `lodedb-core`'s own commit-writer, so the +//! fixtures match the exact on-disk format the engine produces. + +#![allow(dead_code)] + +use lodedb_core::storage::commit_manifest::{ + build_commit_body, commit_manifest_path, write_commit_manifest, CommitBodyInput, +}; +use lodedb_cloud_core::{ArtifactRef, GenerationInventory}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::fs; +use std::path::Path; + +/// Lowercase-hex SHA-256, matching what the engine records per artifact. +pub fn sha_hex(data: &[u8]) -> String { + Sha256::digest(data) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// Writes a store's base file (and any delta segments) under `/.gen/`, +/// returning the `{base, deltas}` sub-manifest that names them — the exact shape +/// the engine records and the inventory reads. +pub fn store_sub( + dir: &Path, + key: &str, + base_name: &str, + base_bytes: &[u8], + delta_dir_suffix: &str, + deltas: &[(&str, &[u8])], +) -> Value { + let gen_dir = dir.join(format!("{key}.gen")); + fs::create_dir_all(&gen_dir).unwrap(); + fs::write(gen_dir.join(base_name), base_bytes).unwrap(); + let mut delta_entries = Vec::new(); + if !deltas.is_empty() { + let delta_dir = gen_dir.join(format!("{base_name}{delta_dir_suffix}")); + fs::create_dir_all(&delta_dir).unwrap(); + for (seq, (name, bytes)) in deltas.iter().enumerate() { + fs::write(delta_dir.join(name), bytes).unwrap(); + delta_entries.push(json!({ + "file_name": name, + "sha256": sha_hex(bytes), + "file_bytes": bytes.len(), + "seq": seq, + })); + } + } + json!({ + "base": { + "file_name": base_name, + "sha256": sha_hex(base_bytes), + "file_bytes": base_bytes.len(), + }, + "deltas": delta_entries, + }) +} + +/// Builds a committed root body (via the engine's own `build_commit_body`) and +/// writes the `.commit.json` pointer. +#[allow(clippy::too_many_arguments)] +pub fn write_commit( + dir: &Path, + key: &str, + generation: u64, + base_epoch: u64, + json_manifest: Option, + tvim_manifest: Option, + tvtext_manifest: Option, + tvlex_manifest: Option, + tvmv_manifest: Option, +) -> Value { + let body = build_commit_body(CommitBodyInput { + index_key: key, + generation, + applied_lsn: generation, + base_epoch, + native_dim: None, + document_count: 0, + chunk_count: 0, + json_manifest, + tvim_manifest, + tvtext_manifest, + tvlex_manifest, + tvmv_manifest, + tvann_manifest: None, + tvvf_manifest: None, + }); + write_commit_manifest(&commit_manifest_path(dir, key), &body, false).unwrap(); + body +} + +/// Convenience: commit a generation whose only store is `json`. +pub fn write_json_commit( + dir: &Path, + key: &str, + generation: u64, + base_epoch: u64, + json_manifest: Value, +) -> Value { + write_commit( + dir, + key, + generation, + base_epoch, + Some(json_manifest), + None, + None, + None, + None, + ) +} + +/// Builds a commit body without touching the filesystem — for CAS tests that only +/// need a well-formed pointer payload. +pub fn commit_body(key: &str, generation: u64, base_epoch: u64, json_manifest: Value) -> Value { + build_commit_body(CommitBodyInput { + index_key: key, + generation, + applied_lsn: generation, + base_epoch, + native_dim: None, + document_count: 0, + chunk_count: 0, + json_manifest: Some(json_manifest), + tvim_manifest: None, + tvtext_manifest: None, + tvlex_manifest: None, + tvmv_manifest: None, + tvann_manifest: None, + tvvf_manifest: None, + }) +} + +/// A minimal but valid engine `state` object for a ready, empty index. +/// +/// Enough for `write_generation_commit` to produce a generation that +/// `load_store` reads back. `key` is used for both `index_key` and +/// `client_id_hash`, matching what the engine records. +pub fn engine_state(key: &str) -> Value { + json!({ + "cache_reuse_count": 0, + "chunks": [], + "client_id_hash": key, + "columnar_generation": 1, + "created_at": "2026-06-26T00:00:00+00:00", + "delete_count": 0, + "deleted_chunk_count": 0, + "document_chunk_ids": {}, + "document_hashes": {}, + "document_metadata": {}, + "embedded_chunk_count": 0, + "fallback_count": 0, + "fallback_reasons": {}, + "index_id": "default", + "index_key": key, + "metadata": {}, + "model": "sentence-transformers/all-MiniLM-L6-v2", + "name": "lodedb-local", + "native_dim": 384, + "provider": "local_open", + "query_count": 0, + "route_profile": "minilm-turbovec", + "schema_version": 1, + "status": "ready", + "storage_profile": "turbovec_direct", + "task": "direct-turbovec", + "turbovec_bit_width": 4, + "updated_at": "2026-06-26T00:00:00+00:00" + }) +} + +/// Commits one engine-written generation into `dir` through the engine's own +/// `write_generation_commit`, returning the committed body. +/// +/// `salt` lands in the state's metadata so two commits with different salts +/// have different content (different snapshot/logical ids); `raw_text` makes +/// the generation payload-bearing (a non-null `tvtext` sub-manifest), which is +/// what distinguishes a full push from a redacted one. +pub fn commit_engine_generation( + dir: &Path, + key: &str, + generation: u64, + base_epoch: u64, + salt: &str, + raw_text: Option<&[(&str, &str)]>, +) -> Value { + commit_engine_generation_with_lexical(dir, key, generation, base_epoch, salt, raw_text, false) +} + +/// [`commit_engine_generation`], optionally also writing a lexical store +/// (`tvlex`) — for tests that need both payload-bearing stores. +pub fn commit_engine_generation_with_lexical( + dir: &Path, + key: &str, + generation: u64, + base_epoch: u64, + salt: &str, + raw_text: Option<&[(&str, &str)]>, + lexical: bool, +) -> Value { + use lodedb_core::storage::lexical_store::TokenLists; + use lodedb_core::storage::{ + write_generation_commit, GenerationCommitInput, GenerationWriteOptions, + }; + let mut state = engine_state(key); + state["metadata"] = json!({ "salt": salt }); + let text: std::collections::BTreeMap = raw_text + .unwrap_or_default() + .iter() + .map(|(id, body)| (id.to_string(), body.to_string())) + .collect(); + let tokens: std::collections::BTreeMap = if lexical { + raw_text + .unwrap_or_default() + .iter() + .map(|(id, body)| { + let token_lists = vec![body + .split_whitespace() + .map(str::to_string) + .collect::>()]; + (id.to_string(), token_lists) + }) + .collect() + } else { + Default::default() + }; + write_generation_commit( + dir, + GenerationCommitInput { + index_key: key, + generation, + applied_lsn: 0, + base_epoch, + state: &state, + tvim: None, + raw_text: raw_text.map(|_| &text), + lexical_tokens: lexical.then_some(&tokens), + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: false, + }, + GenerationWriteOptions::default(), + ) + .unwrap() +} + +/// A hand-built [`ArtifactRef`] for the pure `diff_inventories` tests. +pub fn artifact(name: &str, sha256: &str, is_base: bool) -> ArtifactRef { + ArtifactRef { + name: name.into(), + sha256: sha256.into(), + size_bytes: 8, + kind: "json".into(), + epoch: 0, + is_base, + } +} + +/// A minimal [`GenerationInventory`] wrapping the given artifacts. +pub fn inventory(artifacts: Vec) -> GenerationInventory { + GenerationInventory { + index_key: "idx".into(), + generation: 1, + base_epoch: 0, + document_count: 0, + chunk_count: 0, + root_body: json!({}), + artifacts, + } +} diff --git a/crates/lodedb-cloud-core/tests/generation_inventory.rs b/crates/lodedb-cloud-core/tests/generation_inventory.rs new file mode 100644 index 00000000..0ef133e8 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/generation_inventory.rs @@ -0,0 +1,413 @@ +//! Tests for the read-only generation inventory and its O(changed) diff. + +mod common; + +use common::*; +use lodedb_cloud_core::{ + diff_inventories, inventory_committed_generation, inventory_from_body, list_index_keys, + ArtifactStoreError, +}; +use serde_json::json; +use std::fs; + +/// A syntactically valid (64 lowercase hex chars) placeholder digest: the +/// inventory validates every digest at the trust boundary, so handcrafted +/// bodies must spell theirs correctly even when the bytes never exist. +fn hex64(nibble: u8) -> String { + format!("{nibble:x}").repeat(64) +} + +#[test] +fn inventory_lists_base_and_delta_segments() { + let dir = tempfile::tempdir().unwrap(); + let json = store_sub( + dir.path(), + "idx", + "g0.json", + b"base-state", + ".json-delta", + &[ + ("delta-00000000.jsd", b"delta-a"), + ("delta-00000001.jsd", b"delta-b"), + ], + ); + write_json_commit(dir.path(), "idx", 1, 0, json); + + let inv = inventory_committed_generation(dir.path(), "idx") + .unwrap() + .unwrap(); + assert_eq!(inv.generation, 1); + assert_eq!(inv.artifacts.len(), 3); + + let base = &inv.artifacts[0]; + assert_eq!(base.name, "idx.gen/g0.json"); + assert!(base.is_base); + assert_eq!(base.kind, "json"); + assert_eq!( + inv.artifacts[1].name, + "idx.gen/g0.json.json-delta/delta-00000000.jsd" + ); + assert!(!inv.artifacts[1].is_base); + + // Every referenced artifact exists on disk with the recorded checksum + size. + for reference in &inv.artifacts { + let bytes = fs::read(dir.path().join(&reference.name)).unwrap(); + assert_eq!(sha_hex(&bytes), reference.sha256); + assert_eq!(bytes.len() as u64, reference.size_bytes); + } +} + +#[test] +fn inventory_covers_multivector_tvmv_store() { + let dir = tempfile::tempdir().unwrap(); + let json = store_sub(dir.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let tvmv = store_sub( + dir.path(), + "idx", + "g0.tvmv", + b"multivec-base", + ".tvmv-delta", + &[], + ); + write_commit( + dir.path(), + "idx", + 1, + 0, + Some(json), + None, + None, + None, + Some(tvmv), + ); + + let inv = inventory_committed_generation(dir.path(), "idx") + .unwrap() + .unwrap(); + assert!( + inv.artifacts + .iter() + .any(|a| a.kind == "tvmv" && a.is_base && a.name == "idx.gen/g0.tvmv"), + "multi-vector base must be inventoried so late-interaction indexes back up completely" + ); +} + +#[test] +fn inventory_covers_the_ann_tvann_store() { + // `tvann` (the persisted ANN cluster partition) is base-only and, to the + // engine, a rebuildable cache — but a body referencing a tvann base the + // transfer never shipped would fail byte-verification on pull, so the + // inventory must cover it like any other store. + let body = json!({ + "index_key": "idx", + "generation": 1, + "base_epoch": 1, + "document_count": 0, + "chunk_count": 0, + "json": { "base": { "file_name": "g1.json", "sha256": hex64(0xa), "file_bytes": 0 }, "deltas": [] }, + "tvim": null, + "tvtext": null, + "tvlex": null, + "tvmv": null, + "tvann": { "base": { "file_name": "g1.tvann", "sha256": hex64(0xb), "file_bytes": 3 } }, + }); + let inv = inventory_from_body("idx", Some(&body)).unwrap().unwrap(); + assert!(inv.artifacts.iter().any(|artifact| artifact.kind == "tvann" + && artifact.is_base + && artifact.name == "idx.gen/g1.tvann")); +} + +#[test] +fn inventory_covers_the_rescore_tvvf_store() { + // `tvvf` (the rescore original-vector sidecar, engine 1.3.2+) is a journaled + // {base, deltas} store: the engine refuses to open a rescore store without + // its sidecar, so a push must ship the base AND any delta segments — a + // pulled copy missing either would be unopenable, not merely degraded. + // Its base is NOT `g.tvvf`: the sidecar keeps its own epoch + // counter (`vf_epoch`, here deliberately different from base_epoch) and + // lives at `vf.tvvf` (`tvvf_store::base_path`) — this body + // mirrors exactly what `write_base_manifest` records. + let body = json!({ + "index_key": "idx", + "generation": 2, + "base_epoch": 1, + "document_count": 0, + "chunk_count": 0, + "json": { "base": { "file_name": "g1.json", "sha256": hex64(0xa), "file_bytes": 0 }, "deltas": [] }, + "tvim": null, + "tvtext": null, + "tvlex": null, + "tvmv": null, + "tvann": null, + "tvvf": { + "schema_version": 1, + "index_key": "idx", + "vf_epoch": 3, + "base": { "file_name": "vf3.tvvf", "sha256": hex64(0xb), "file_bytes": 8 }, + "deltas": [ + { "file_name": "delta-00000000.vfd", "sha256": hex64(0xc), "file_bytes": 4 }, + ], + "next_seq": 1, + }, + }); + let inv = inventory_from_body("idx", Some(&body)).unwrap().unwrap(); + assert!(inv.artifacts.iter().any(|artifact| artifact.kind == "tvvf" + && artifact.is_base + && artifact.epoch == 3 + && artifact.name == "idx.gen/vf3.tvvf")); + assert!(inv.artifacts.iter().any(|artifact| artifact.kind == "tvvf" + && !artifact.is_base + && artifact.name == "idx.gen/vf3.tvvf.tvvf-delta/delta-00000000.vfd")); +} + +#[test] +fn inventory_rejects_a_tvvf_manifest_without_vf_epoch() { + // Without the sidecar's own epoch the base path cannot be derived; guessing + // `g.tvvf` would name a file the engine never writes. + let body = json!({ + "index_key": "idx", + "generation": 2, + "base_epoch": 1, + "json": { "base": { "file_name": "g1.json", "sha256": hex64(0xa), "file_bytes": 0 }, "deltas": [] }, + "tvvf": { + "base": { "file_name": "vf3.tvvf", "sha256": hex64(0xb), "file_bytes": 8 }, + "deltas": [], + }, + }); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert!(err.to_string().contains("vf_epoch"), "{err}"); +} + +#[test] +fn inventory_rejects_a_malformed_artifact_digest() { + // Digests become staging file names and object keys downstream, so a + // "digest" carrying a path (or anything that is not 64 lowercase hex + // chars) must fail closed at the inventory — before it can name a path. + for bad in ["", "abc", "../../../etc/passwd", "/tmp/evil"] { + let sub = json!({ + "base": { "file_name": "g0.json", "sha256": bad, "file_bytes": 0 }, + "deltas": [], + }); + let body = commit_body("idx", 1, 0, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert!(err.to_string().contains("sha256"), "{err}"); + } +} + +#[test] +fn inventory_rejects_an_unknown_store_sub_manifest() { + // A future engine store this build does not know must refuse the transfer, + // not silently drop its artifacts — an understated inventory ships a + // generation whose referenced blobs were never uploaded. + let body = json!({ + "index_key": "idx", + "generation": 1, + "base_epoch": 0, + "json": { "base": { "file_name": "g0.json", "sha256": hex64(0xa), "file_bytes": 0 }, "deltas": [] }, + "tvfuture": { "base": { "file_name": "g0.tvfuture", "sha256": hex64(0xb), "file_bytes": 0 }, "deltas": [] }, + }); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert!(err.to_string().contains("tvfuture"), "{err}"); +} + +#[test] +fn inventory_absent_generation_is_none() { + let dir = tempfile::tempdir().unwrap(); + assert!(inventory_committed_generation(dir.path(), "idx") + .unwrap() + .is_none()); +} + +#[test] +fn list_index_keys_finds_committed_pointers() { + let dir = tempfile::tempdir().unwrap(); + for key in ["beta", "alpha"] { + let json = store_sub(dir.path(), key, "g0.json", b"state", ".json-delta", &[]); + write_json_commit(dir.path(), key, 1, 0, json); + } + assert_eq!( + list_index_keys(dir.path()).unwrap(), + vec!["alpha".to_string(), "beta".to_string()] + ); +} + +#[test] +fn diff_uploads_everything_when_remote_absent() { + let local = inventory(vec![ + artifact("idx.gen/g0.json", "aaa", true), + artifact("idx.gen/g0.json.json-delta/d0", "bbb", false), + ]); + let diff = diff_inventories(&local, None); + assert_eq!(diff.to_upload.len(), 2); + assert!(diff.ships_base); +} + +#[test] +fn diff_uploads_nothing_when_identical() { + let local = inventory(vec![artifact("idx.gen/g0.json", "aaa", true)]); + let remote = local.clone(); + let diff = diff_inventories(&local, Some(&remote)); + assert!(diff.to_upload.is_empty()); + assert!(!diff.ships_base); +} + +#[test] +fn diff_delta_only_onto_shared_base_does_not_ship_base() { + let base = artifact("idx.gen/g0.json", "aaa", true); + let remote = inventory(vec![ + base.clone(), + artifact("idx.gen/g0.json.json-delta/d0", "bbb", false), + ]); + let local = inventory(vec![ + base, + artifact("idx.gen/g0.json.json-delta/d0", "bbb", false), + artifact("idx.gen/g0.json.json-delta/d1", "ccc", false), + ]); + let diff = diff_inventories(&local, Some(&remote)); + assert_eq!(diff.to_upload.len(), 1); + assert_eq!(diff.to_upload[0].name, "idx.gen/g0.json.json-delta/d1"); + assert!(!diff.ships_base); +} + +#[test] +fn diff_new_base_epoch_ships_base() { + let remote = inventory(vec![artifact("idx.gen/g0.json", "aaa", true)]); + let local = inventory(vec![artifact("idx.gen/g1.json", "zzz", true)]); + let diff = diff_inventories(&local, Some(&remote)); + assert!(diff.ships_base); + assert!(diff.to_upload.iter().any(|a| a.is_base)); +} + +#[test] +fn diff_ships_base_when_same_epoch_base_differs_by_checksum() { + let remote = inventory(vec![artifact("idx.gen/g0.json", "aaa", true)]); + let local = inventory(vec![artifact("idx.gen/g0.json", "DIFFERENT", true)]); + let diff = diff_inventories(&local, Some(&remote)); + assert_eq!(diff.to_upload.len(), 1); + assert!(diff.ships_base); +} + +#[test] +fn inventory_rejects_a_traversing_delta_file_name() { + // A tampered manifest whose delta file name climbs out of `.gen` must be + // rejected before any artifact path is built, so a restore cannot plant a file + // under another index's key. + let sub = json!({ + "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, + "deltas": [{ "file_name": "../../victim.commit.json", "sha256": hex64(0xe), "file_bytes": 0, "seq": 0 }], + }); + let body = commit_body("idx", 1, 0, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_traversing_base_file_name() { + let sub = json!({ + "base": { "file_name": "../escape", "sha256": hex64(0xd), "file_bytes": 0 }, + "deltas": [], + }); + let body = commit_body("idx", 1, 0, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_baseless_store_manifest() { + // A non-null store manifest must carry a journaled base; a base-less object + // (e.g. from corruption) would otherwise silently drop the base from a backup. + let body = commit_body("idx", 1, 0, json!({ "deltas": [] })); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_non_object_delta_entry() { + let sub = json!({ + "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, + "deltas": [null], + }); + let body = commit_body("idx", 1, 0, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_includes_non_null_tvim_regardless_of_present_flag() { + // The engine loads a non-null tvim manifest whatever `tvim_present` says, so the + // inventory must too — otherwise a restore drops the tvim base it will open. + let body = json!({ + "index_key": "idx", + "generation": 1, + "base_epoch": 1, + "document_count": 0, + "chunk_count": 0, + "json": { "base": { "file_name": "g1.json", "sha256": hex64(0xa), "file_bytes": 0 }, "deltas": [] }, + "tvim": { "base": { "file_name": "g1.tvim", "sha256": hex64(0xb), "file_bytes": 0 }, "deltas": [] }, + "tvim_present": false, + "tvtext": null, + "tvlex": null, + "tvmv": null, + }); + let inv = inventory_from_body("idx", Some(&body)).unwrap().unwrap(); + assert!(inv.artifacts.iter().any(|artifact| artifact.kind == "tvim" + && artifact.is_base + && artifact.name == "idx.gen/g1.tvim")); +} + +#[test] +fn inventory_rejects_a_base_file_name_that_disagrees_with_the_epoch() { + // A valid basename that is not `g.` is a tampered/inconsistent + // pointer: the engine would open `g1.json` while the manifest names `g2.json`. + let sub = json!({ + "base": { "file_name": "g2.json", "sha256": hex64(0xd), "file_bytes": 0 }, + "deltas": [], + }); + let body = commit_body("idx", 1, 1, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_missing_body_index_key() { + // The engine rejects an empty/missing body index_key as corrupt, so inventory + // must too rather than fall back to the requested key. + let body = json!({ + "generation": 1, + "base_epoch": 0, + "json": { "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, "deltas": [] }, + }); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_non_object_store_manifest() { + // A non-null, non-object store value is present-and-corrupt to the engine, not + // absent, so it must fail closed instead of being silently skipped. + let body = json!({ + "index_key": "idx", + "generation": 1, + "base_epoch": 0, + "json": { "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, "deltas": [] }, + "tvtext": "bad", + }); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn inventory_rejects_a_body_key_mismatch() { + // The pointer file-name key ("idx") disagrees with the body's own index_key + // ("other") — only possible via tampering, since the body checksum is valid. + let sub = json!({ + "base": { "file_name": "g0.json", "sha256": hex64(0xd), "file_bytes": 0 }, + "deltas": [], + }); + let body = commit_body("other", 1, 0, sub); + let err = inventory_from_body("idx", Some(&body)).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} diff --git a/crates/lodedb-cloud-core/tests/managed.rs b/crates/lodedb-cloud-core/tests/managed.rs new file mode 100644 index 00000000..30988e2a --- /dev/null +++ b/crates/lodedb-cloud-core/tests/managed.rs @@ -0,0 +1,268 @@ +//! Tests for the managed (`orecloud://`) transfer helpers: plan/classify, +//! sidecar trust against a caller-supplied remote identity, pull +//! requirements, and staging-directory materialisation. + +mod common; + +use common::*; +use lodedb_cloud_core::{ + managed_materialize, managed_plan, managed_pull_requirements, managed_record_base, snapshot_id, + ArtifactStore, ArtifactStoreError, TransferPolicy, +}; +use std::fs; +use std::path::Path; + +const KEY: &str = "idx"; +const REMOTE: &str = "orecloud://acme/support/default#host=https://example.test"; + +fn dir_str(path: &Path) -> &str { + path.to_str().unwrap() +} + +/// Copies every artifact a committed generation pins into a staging directory +/// under its content digest — what the Python edge does with downloaded blobs. +fn stage_generation(source: &Path, body: &serde_json::Value, staging: &Path) { + let inventory = lodedb_cloud_core::inventory_from_body(KEY, Some(body)) + .unwrap() + .unwrap(); + fs::create_dir_all(staging).unwrap(); + for artifact in inventory.artifacts { + let bytes = fs::read(source.join(&artifact.name)).unwrap(); + fs::write(staging.join(&artifact.sha256), bytes).unwrap(); + } +} + +#[test] +fn plan_for_a_fresh_local_generation_is_local_ahead_with_full_inventory() { + let local = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(local.path(), KEY, 1, 1, "a", None); + + let plan = managed_plan( + dir_str(local.path()), + KEY, + REMOTE, + None, + TransferPolicy::redacted(), + ) + .unwrap(); + + assert_eq!(plan.report.classification.as_deref(), Some("local_ahead")); + let local_part = plan.local.expect("local generation is committed"); + // The redacted policy nulls tvtext/tvlex, so the plan's identity is the + // *redacted* body's — and for a text-free commit that equals the raw one. + assert_eq!(local_part.side.snapshot_id, snapshot_id(&body).unwrap()); + assert!(!local_part.side.has_text); + assert!(!local_part.artifacts.is_empty()); + // The pointer document round-trips to the same identity. + let document: serde_json::Value = serde_json::from_str(&local_part.pointer_document).unwrap(); + assert_eq!( + document.get("body_sha256").and_then(|v| v.as_str()), + Some(local_part.side.snapshot_id.as_str()) + ); + assert_eq!(document.get("body").unwrap(), &local_part.body); + assert!(plan.remote.is_none()); + assert!(plan.base.is_none()); +} + +#[test] +fn plan_trusts_the_sidecar_only_for_the_exact_remote_identity() { + let local = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(local.path(), KEY, 1, 1, "a", None); + managed_record_base(dir_str(local.path()), KEY, REMOTE, &body).unwrap(); + + // Same identity: base trusted, and with an equal remote head the pair is + // in sync with a current base. + let plan = managed_plan( + dir_str(local.path()), + KEY, + REMOTE, + Some(body.clone()), + TransferPolicy::redacted(), + ) + .unwrap(); + assert_eq!(plan.report.classification.as_deref(), Some("in_sync")); + assert!(plan.base.is_some()); + assert!(plan.base_is_current); + + // A different remote identity (another org, another host) must not + // inherit that base: a remote holding different content classifies as + // unknown (force required), never as a fast-forward. + let other = tempfile::tempdir().unwrap(); + let other_body = commit_engine_generation(other.path(), KEY, 2, 1, "b", None); + let plan = managed_plan( + dir_str(local.path()), + KEY, + "orecloud://other/testing/default#host=https://example.test", + Some(other_body), + TransferPolicy::redacted(), + ) + .unwrap(); + assert_eq!(plan.report.classification.as_deref(), Some("unknown")); + assert!(plan.base.is_none()); +} + +#[test] +fn plan_classifies_a_remote_advance_as_remote_ahead() { + // One lineage, two checkouts: the "remote" is the second checkout's + // commit on top of the shared base. + let local = tempfile::tempdir().unwrap(); + let base_body = commit_engine_generation(local.path(), KEY, 1, 1, "a", None); + managed_record_base(dir_str(local.path()), KEY, REMOTE, &base_body).unwrap(); + + let ahead = tempfile::tempdir().unwrap(); + let ahead_body = commit_engine_generation(ahead.path(), KEY, 2, 1, "b", None); + + let plan = managed_plan( + dir_str(local.path()), + KEY, + REMOTE, + Some(ahead_body), + TransferPolicy::redacted(), + ) + .unwrap(); + assert_eq!(plan.report.classification.as_deref(), Some("remote_ahead")); + assert!(!plan.base_is_current); +} + +#[test] +fn pull_requirements_shrink_to_nothing_after_materialise() { + let source = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(source.path(), KEY, 1, 1, "a", Some(&[("d1", "text")])); + + let fresh = tempfile::tempdir().unwrap(); + let needed = managed_pull_requirements(dir_str(fresh.path()), KEY, &body).unwrap(); + assert!(!needed.is_empty()); + + let staging = tempfile::tempdir().unwrap(); + stage_generation(source.path(), &body, staging.path()); + let outcome = managed_materialize( + dir_str(fresh.path()), + KEY, + REMOTE, + body.clone(), + dir_str(staging.path()), + false, + None, + ) + .unwrap(); + assert!(outcome.transfer.pointer_published); + assert_eq!(outcome.transfer.generation, 1); + assert_eq!(outcome.open.index_key, KEY); + + // Everything staged is now local: nothing left to download, and the + // sidecar records the base so a re-plan against the same head is in sync. + let needed_after = managed_pull_requirements(dir_str(fresh.path()), KEY, &body).unwrap(); + assert!(needed_after.is_empty()); + let plan = managed_plan( + dir_str(fresh.path()), + KEY, + REMOTE, + Some(body), + TransferPolicy::full(), + ) + .unwrap(); + assert_eq!(plan.report.classification.as_deref(), Some("in_sync")); + assert!(plan.base_is_current); +} + +#[test] +fn materialise_with_a_missing_staged_blob_fails_before_any_pointer_moves() { + let source = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(source.path(), KEY, 1, 1, "a", None); + + let fresh = tempfile::tempdir().unwrap(); + let staging = tempfile::tempdir().unwrap(); // deliberately empty + let err = managed_materialize( + dir_str(fresh.path()), + KEY, + REMOTE, + body, + dir_str(staging.path()), + false, + None, + ) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); + assert!(!fresh.path().join(format!("{KEY}.commit.json")).exists()); +} + +#[test] +fn materialise_with_a_corrupt_staged_blob_fails_closed() { + let source = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(source.path(), KEY, 1, 1, "a", None); + + let fresh = tempfile::tempdir().unwrap(); + let staging = tempfile::tempdir().unwrap(); + stage_generation(source.path(), &body, staging.path()); + // Corrupt one staged blob: the restore re-hashes on write and must refuse. + let victim = fs::read_dir(staging.path()) + .unwrap() + .next() + .unwrap() + .unwrap(); + fs::write(victim.path(), b"corrupted-download").unwrap(); + + let err = managed_materialize( + dir_str(fresh.path()), + KEY, + REMOTE, + body, + dir_str(staging.path()), + false, + None, + ) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert!(!fresh.path().join(format!("{KEY}.commit.json")).exists()); +} + +#[test] +fn materialise_refuses_when_the_local_store_moved_after_classification() { + // The sync caller pins the local snapshot it classified: a commit landing + // between classification and materialization must refuse (re-run the + // sync) instead of being silently overwritten by the pull. + let source = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(source.path(), KEY, 2, 2, "remote-v2", None); + + let local = tempfile::tempdir().unwrap(); + let classified = commit_engine_generation(local.path(), KEY, 1, 1, "local-v1", None); + let classified_id = lodedb_cloud_core::snapshot_id(&classified).unwrap(); + // The local store commits again after "classification". + let newer = commit_engine_generation(local.path(), KEY, 3, 3, "local-v3", None); + + let staging = tempfile::tempdir().unwrap(); + stage_generation(source.path(), &body, staging.path()); + let err = managed_materialize( + dir_str(local.path()), + KEY, + REMOTE, + body.clone(), + dir_str(staging.path()), + false, + Some(&classified_id), + ) + .unwrap_err(); + assert!( + matches!(err, ArtifactStoreError::SyncConflict { .. }), + "expected the stale-classification refusal, got: {err}" + ); + // The newer local commit survives untouched. + let current = lodedb_cloud_core::LocalArtifactStore::new(local.path(), false) + .read_pointer(KEY) + .unwrap() + .unwrap(); + assert_eq!(current, newer); + + // Pinning to "classified as absent" ("") refuses over any commit too. + let err = managed_materialize( + dir_str(local.path()), + KEY, + REMOTE, + body, + dir_str(staging.path()), + false, + Some(""), + ) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::SyncConflict { .. })); +} diff --git a/crates/lodedb-cloud-core/tests/manifest_transfer.rs b/crates/lodedb-cloud-core/tests/manifest_transfer.rs new file mode 100644 index 00000000..7464a173 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/manifest_transfer.rs @@ -0,0 +1,317 @@ +//! Tests for `export_generation`: O(changed) copy + all-or-nothing pointer swap, +//! exercised against both hand-built and genuine engine-written generations. + +mod common; + +use common::*; +use lodedb_core::storage::{ + wal, write_generation_commit, GenerationCommitInput, GenerationWriteOptions, +}; +use lodedb_cloud_core::{ + export_generation, inventory_committed_generation, verify_local_generation_opens, + ArtifactStore, ArtifactStoreError, LocalArtifactStore, TransferPolicy, +}; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::fs; + +#[test] +fn export_copies_generation_and_publishes_pointer() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let json = store_sub( + src.path(), + "idx", + "g0.json", + b"base-state", + ".json-delta", + &[("delta-00000000.jsd", b"delta-a")], + ); + write_json_commit(src.path(), "idx", 1, 0, json); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let result = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + + assert_eq!(result.artifacts_written, 2); + assert_eq!(result.artifacts_skipped, 0); + assert!(result.pointer_published); + assert_eq!( + dest.read_pointer("idx").unwrap(), + source.read_pointer("idx").unwrap() + ); + for name in [ + "idx.gen/g0.json", + "idx.gen/g0.json.json-delta/delta-00000000.jsd", + ] { + assert_eq!( + dest.read_bytes(name).unwrap(), + source.read_bytes(name).unwrap() + ); + } +} + +#[test] +fn export_is_idempotent() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let json = store_sub(src.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + write_json_commit(src.path(), "idx", 1, 0, json); + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + + export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + let again = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + assert_eq!(again.artifacts_written, 0); + assert_eq!(again.bytes_written, 0); + assert!(!again.pointer_published); +} + +#[test] +fn export_missing_source_generation_is_not_found() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let err = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); +} + +#[test] +fn export_rejects_corrupt_source_artifact_before_publishing() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + // Write a base file whose recorded checksum does not match its bytes. + let gen_dir = src.path().join("idx.gen"); + fs::create_dir_all(&gen_dir).unwrap(); + fs::write(gen_dir.join("g0.json"), b"actual-bytes").unwrap(); + let corrupt_sub = json!({ + "base": { + "file_name": "g0.json", + "sha256": sha_hex(b"claimed-different-bytes"), + "file_bytes": "actual-bytes".len(), + }, + "deltas": [], + }); + write_json_commit(src.path(), "idx", 1, 0, corrupt_sub); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let err = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + // The destination pointer was never published. + assert!(dest.read_pointer("idx").unwrap().is_none()); +} + +#[test] +fn exports_a_generation_written_by_the_engine() { + const KEY: &str = "6f78dec251fa5e544784ac1af95b0ae6530cad714a2d34f8c4615740ecbf8205"; + let src = tempfile::tempdir().unwrap(); + let state = engine_state(KEY); + write_generation_commit( + src.path(), + GenerationCommitInput { + index_key: KEY, + generation: 1, + applied_lsn: 0, + base_epoch: 1, + state: &state, + tvim: None, + raw_text: None, + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: true, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); + + let inv = inventory_committed_generation(src.path(), KEY) + .unwrap() + .unwrap(); + assert!(inv.artifacts.iter().any(|a| a.kind == "json" && a.is_base)); + for reference in &inv.artifacts { + let bytes = fs::read(src.path().join(&reference.name)).unwrap(); + assert_eq!( + sha_hex(&bytes), + reference.sha256, + "checksum matches engine record" + ); + } + + let dst = tempfile::tempdir().unwrap(); + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let result = export_generation(&source, &dest, KEY, TransferPolicy::full()).unwrap(); + assert!(result.pointer_published); + assert_eq!( + dest.read_pointer(KEY).unwrap(), + source.read_pointer(KEY).unwrap() + ); +} + +#[test] +fn redacted_push_nulls_text_and_skips_its_artifacts() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let json = store_sub(src.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let text = store_sub( + src.path(), + "idx", + "g0.tvtext", + b"secret document text", + ".tvtext-delta", + &[], + ); + // Source commits both a redacted (json) and a payload-bearing (tvtext) store. + write_commit( + src.path(), + "idx", + 1, + 0, + Some(json), + None, + Some(text), + None, + None, + ); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let result = export_generation(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); + + // Only the json base shipped; the text base was neither counted nor uploaded. + assert_eq!(result.artifacts_written, 1); + assert!(matches!( + dest.read_bytes("idx.gen/g0.tvtext").unwrap_err(), + ArtifactStoreError::NotFound(_) + )); + assert!(dest.read_bytes("idx.gen/g0.json").is_ok()); + // The published body genuinely omits the text store. + let body = dest.read_pointer("idx").unwrap().unwrap(); + assert_eq!(body.get("tvtext"), Some(&Value::Null)); + assert!(body.get("json").is_some_and(|json| !json.is_null())); +} + +#[test] +fn full_push_ships_text_that_redacted_push_omits() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let json = store_sub(src.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let text = store_sub( + src.path(), + "idx", + "g0.tvtext", + b"text", + ".tvtext-delta", + &[], + ); + write_commit( + src.path(), + "idx", + 1, + 0, + Some(json), + None, + Some(text), + None, + None, + ); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + + assert!(dest.read_bytes("idx.gen/g0.tvtext").is_ok()); + assert_eq!( + dest.read_pointer("idx").unwrap(), + source.read_pointer("idx").unwrap() + ); +} + +#[test] +fn redacted_push_of_engine_generation_opens_without_text() { + const KEY: &str = "aa11bb22cc33dd44ee55ff6600112233445566778899aabbccddeeff00112233"; + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let mut raw_text = BTreeMap::new(); + raw_text.insert("doc-1".to_string(), "sensitive body text".to_string()); + write_generation_commit( + src.path(), + GenerationCommitInput { + index_key: KEY, + generation: 1, + applied_lsn: 0, + base_epoch: 1, + state: &engine_state(KEY), + tvim: None, + raw_text: Some(&raw_text), + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: false, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + export_generation(&source, &dest, KEY, TransferPolicy::redacted()).unwrap(); + + // The restored copy carries no text base at all, yet opens read-only through + // the engine's own load path. + assert!(!dst.path().join(format!("{KEY}.gen/g1.tvtext")).exists()); + let report = verify_local_generation_opens(dst.path(), KEY).unwrap(); + assert_eq!(report.index_key, KEY); +} + +#[test] +fn export_excludes_uncommitted_wal_writes() { + const KEY: &str = "bb22cc33dd44ee55ff6600112233445566778899aabbccddeeff001122334455"; + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + write_generation_commit( + src.path(), + GenerationCommitInput { + index_key: KEY, + generation: 3, + applied_lsn: 0, + base_epoch: 1, + state: &engine_state(KEY), + tvim: None, + raw_text: None, + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: false, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); + // An uncommitted WAL write sits alongside the committed generation. + let wal_file = wal::wal_path(src.path(), KEY); + wal::append_record(&wal_file, 4, "add", json!({"id": "pending"}), false).unwrap(); + assert!(wal::scan_stats(&wal_file).unwrap().op_count > 0); + + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let result = export_generation(&source, &dest, KEY, TransferPolicy::full()).unwrap(); + + // Only the committed generation shipped: no `.wal` reached the destination, + // and the published generation is the committed one. + assert_eq!(result.generation, 3); + assert!(!wal::wal_path(dst.path(), KEY).exists()); + assert_eq!( + dest.read_pointer(KEY) + .unwrap() + .unwrap() + .get("generation") + .and_then(Value::as_u64), + Some(3) + ); +} diff --git a/crates/lodedb-cloud-core/tests/object_artifact_store.rs b/crates/lodedb-cloud-core/tests/object_artifact_store.rs new file mode 100644 index 00000000..37a0c3e5 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/object_artifact_store.rs @@ -0,0 +1,202 @@ +//! Tests for `ObjectArtifactStore` over `object_store`'s in-memory backend: +//! immutable byte I/O, conditional-write pointer CAS, per-tenant isolation, and a +//! full export/restore round trip through an "object store". + +mod common; + +use common::*; +use lodedb_core::storage::{ + write_generation_commit, GenerationCommitInput, GenerationWriteOptions, +}; +use object_store::memory::InMemory; +use object_store::ObjectStore; +use lodedb_cloud_core::{ + export_generation, verify_generation, verify_local_generation_opens, ArtifactStore, + ArtifactStoreError, LocalArtifactStore, ObjectArtifactStore, TransferPolicy, +}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// A base-only json sub-manifest for a pointer body (no artifacts need exist for +/// pointer-CAS tests). +fn json_sub(base_name: &str) -> Value { + json!({ + "base": { "file_name": base_name, "sha256": sha_hex(b"x"), "file_bytes": 0 }, + "deltas": [], + }) +} + +#[test] +fn write_and_read_round_trips() { + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + let data = b"generation-artifact-bytes"; + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha_hex(data)) + .unwrap(); + assert_eq!(store.read_bytes("idx.gen/g0.json").unwrap(), data.to_vec()); +} + +#[test] +fn read_missing_artifact_is_not_found() { + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + let err = store.read_bytes("idx.gen/absent.json").unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); +} + +#[test] +fn checksum_mismatch_is_rejected() { + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + let err = store + .write_bytes_if_absent("idx.gen/g0.json", b"data", &sha_hex(b"other")) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn immutable_rewrite_is_noop_or_conflict() { + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + let data = b"immutable"; + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha_hex(data)) + .unwrap(); + // Identical bytes: idempotent no-op. + store + .write_bytes_if_absent("idx.gen/g0.json", data, &sha_hex(data)) + .unwrap(); + // Different bytes under the same name: refused, original preserved. + let err = store + .write_bytes_if_absent("idx.gen/g0.json", b"different", &sha_hex(b"different")) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert_eq!(store.read_bytes("idx.gen/g0.json").unwrap(), data.to_vec()); +} + +#[test] +fn pointer_cas_creates_updates_and_conflicts() { + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + assert!(store.read_pointer("idx").unwrap().is_none()); + + // Create (expect-absent) publishes generation 1. + let gen1 = commit_body("idx", 1, 0, json_sub("g0.json")); + store.compare_and_swap_pointer("idx", None, &gen1).unwrap(); + + // Expect-absent now conflicts (the pointer exists). + let gen2 = commit_body("idx", 2, 0, json_sub("g0.json")); + let err = store + .compare_and_swap_pointer("idx", None, &gen2) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::PointerConflict { .. })); + + // Matching the exact committed body advances the pointer to 2. + store + .compare_and_swap_pointer("idx", Some(&gen1), &gen2) + .unwrap(); + assert_eq!( + store + .read_pointer("idx") + .unwrap() + .unwrap() + .get("generation") + .and_then(Value::as_u64), + Some(2) + ); + + // A stale expected body (a would-be concurrent writer still holding gen 1) + // conflicts — the pointer is now gen 2. + let gen3 = commit_body("idx", 3, 0, json_sub("g0.json")); + let err = store + .compare_and_swap_pointer("idx", Some(&gen1), &gen3) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::PointerConflict { .. })); +} + +#[test] +fn read_pointer_rejects_a_corrupt_document() { + // A pointer object whose bytes are not a valid commit document must fail + // closed, not parse as an empty/zero generation. + let backend: Arc = Arc::new(InMemory::new()); + let store = ObjectArtifactStore::new(backend, "tenant").unwrap(); + store + .write_bytes_if_absent("idx.commit.json", b"not-json", &sha_hex(b"not-json")) + .unwrap(); + assert!(store.read_pointer("idx").is_err()); +} + +#[test] +fn tenants_are_isolated_by_prefix() { + // Two stores share one bucket under different tenant prefixes. + let backend: Arc = Arc::new(InMemory::new()); + let tenant_a = ObjectArtifactStore::new(backend.clone(), "tenant-a").unwrap(); + let tenant_b = ObjectArtifactStore::new(backend, "tenant-b").unwrap(); + + let data = b"tenant-a-secret"; + tenant_a + .write_bytes_if_absent("idx.gen/g0.json", data, &sha_hex(data)) + .unwrap(); + tenant_a + .compare_and_swap_pointer("idx", None, &commit_body("idx", 1, 0, json_sub("g0.json"))) + .unwrap(); + + // Tenant B, knowing the exact name and checksum, still cannot reach A's blob + // or pointer — the prefix namespaces content addressing per tenant. + assert!(matches!( + tenant_b.read_bytes("idx.gen/g0.json").unwrap_err(), + ArtifactStoreError::NotFound(_) + )); + assert!(tenant_b.read_pointer("idx").unwrap().is_none()); +} + +#[test] +fn exports_and_restores_through_an_object_store() { + const KEY: &str = "dd44ee55ff6600112233445566778899aabbccddeeff00112233445566778899"; + let src = tempfile::tempdir().unwrap(); + write_generation_commit( + src.path(), + GenerationCommitInput { + index_key: KEY, + generation: 1, + applied_lsn: 0, + base_epoch: 1, + state: &engine_state(KEY), + tvim: None, + raw_text: None, + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: false, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); + + // Push: local directory -> object store. + let source = LocalArtifactStore::new(src.path(), false); + let backend: Arc = Arc::new(InMemory::new()); + let remote = ObjectArtifactStore::new(backend, "tenant-x").unwrap(); + let pushed = export_generation(&source, &remote, KEY, TransferPolicy::full()).unwrap(); + assert!(pushed.pointer_published); + assert!(pushed.artifacts_written > 0); + + // Idempotent re-push moves nothing. + let again = export_generation(&source, &remote, KEY, TransferPolicy::full()).unwrap(); + assert_eq!(again.artifacts_written, 0); + assert!(!again.pointer_published); + + // Every artifact in the object store re-hashes to its recorded checksum. + let verified = verify_generation(&remote, KEY).unwrap(); + assert_eq!(verified.generation, 1); + assert!(verified.artifacts_verified > 0); + + // Pull: object store -> a fresh local directory that opens read-only. + let restored = tempfile::tempdir().unwrap(); + let dest = LocalArtifactStore::new(restored.path(), false); + export_generation(&remote, &dest, KEY, TransferPolicy::full()).unwrap(); + let report = verify_local_generation_opens(restored.path(), KEY).unwrap(); + assert_eq!(report.index_key, KEY); +} diff --git a/crates/lodedb-cloud-core/tests/restore.rs b/crates/lodedb-cloud-core/tests/restore.rs new file mode 100644 index 00000000..a6a7b7aa --- /dev/null +++ b/crates/lodedb-cloud-core/tests/restore.rs @@ -0,0 +1,283 @@ +//! Tests for the pull-direction transfer (restore = `export_generation` with the +//! stores swapped) and confirming a restored local copy opens read-only through +//! the engine. + +mod common; + +use common::*; +use lodedb_core::storage::{ + write_generation_commit, GenerationCommitInput, GenerationWriteOptions, +}; +use lodedb_cloud_core::{ + export_generation, verify_local_generation_opens, ArtifactStore, ArtifactStoreError, + LocalArtifactStore, TransferPolicy, +}; +use serde_json::json; +use std::fs; + +#[test] +fn restore_copies_a_generation_that_opens_read_only() { + const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; + // The "remote" backup is just another artifact store holding a committed + // generation the engine wrote. + let remote = tempfile::tempdir().unwrap(); + write_generation_commit( + remote.path(), + GenerationCommitInput { + index_key: KEY, + generation: 1, + applied_lsn: 0, + base_epoch: 1, + state: &engine_state(KEY), + tvim: None, + raw_text: None, + lexical_tokens: None, + multivec: None, + ann: None, + tvvf_manifest: None, + compress_text: false, + }, + GenerationWriteOptions::default(), + ) + .unwrap(); + + let local = tempfile::tempdir().unwrap(); + let source = LocalArtifactStore::new(remote.path(), false); + let dest = LocalArtifactStore::new(local.path(), false); + let result = export_generation(&source, &dest, KEY, TransferPolicy::full()).unwrap(); + assert!(result.pointer_published); + assert_eq!(result.generation, 1); + + // The strongest acceptance check: the engine's own load path opens the + // restored directory read-only. + let report = verify_local_generation_opens(local.path(), KEY).unwrap(); + assert_eq!(report.index_key, KEY); +} + +#[test] +fn pull_rebuilds_the_delta_journal_manifests() { + // The engine's per-store journal manifest is working state the body never + // pins, so the transfer doesn't ship it — but the O(changed) mutation + // path requires it, so a restored copy must be WRITABLE, not just + // readable. `pull` reconstructs each journal manifest verbatim from the + // body's sub-manifest. + const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation( + remote.path(), + KEY, + 1, + 1, + "journal-manifests", + Some(&[("doc-1", "raw text one")]), + ); + let remote_body = LocalArtifactStore::new(remote.path(), false) + .read_pointer(KEY) + .unwrap() + .unwrap(); + + let local = tempfile::tempdir().unwrap(); + lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + KEY, + ) + .unwrap(); + + let gen_dir = local.path().join(format!("{KEY}.gen")); + for (kind, suffix) in [("json", ".json-delta"), ("tvtext", ".tvtext-delta")] { + let manifest_path = gen_dir.join(format!("g1.{kind}{suffix}")).join("manifest.json"); + let rebuilt: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + assert_eq!( + &rebuilt, + remote_body.get(kind).unwrap(), + "journal manifest for {kind} must equal the body sub-manifest" + ); + } +} + +#[test] +fn torn_restore_leaves_the_destination_pointer_unpublished() { + // A corrupt source artifact must fail the restore before any pointer is + // published, so the destination stays on its previous (here: absent) state. + let remote = tempfile::tempdir().unwrap(); + let local = tempfile::tempdir().unwrap(); + let gen_dir = remote.path().join("idx.gen"); + fs::create_dir_all(&gen_dir).unwrap(); + fs::write(gen_dir.join("g0.json"), b"actual-bytes").unwrap(); + let corrupt_sub = json!({ + "base": { + "file_name": "g0.json", + "sha256": sha_hex(b"claimed-other-bytes"), + "file_bytes": "actual-bytes".len(), + }, + "deltas": [], + }); + write_json_commit(remote.path(), "idx", 1, 0, corrupt_sub); + + let source = LocalArtifactStore::new(remote.path(), false); + let dest = LocalArtifactStore::new(local.path(), false); + let err = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); + assert!(dest.read_pointer("idx").unwrap().is_none()); +} + +#[test] +fn restore_missing_source_generation_is_not_found() { + let remote = tempfile::tempdir().unwrap(); + let local = tempfile::tempdir().unwrap(); + let source = LocalArtifactStore::new(remote.path(), false); + let dest = LocalArtifactStore::new(local.path(), false); + let err = export_generation(&source, &dest, "idx", TransferPolicy::full()).unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); +} + +#[test] +fn pull_refuses_while_an_engine_writer_holds_the_directory_lock() { + // A restore is a writer of the database directory: it must contend on the + // engine's own single-writer lock rather than interleave with a live + // writer (whose in-memory state is based on the pointer the restore + // replaces). The env var keeps the contention check immediate. + const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; + std::env::set_var("LODEDB_PERSIST_LOCK_TIMEOUT", "0"); + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(remote.path(), KEY, 1, 1, "locked", None); + let local = tempfile::tempdir().unwrap(); + + let held = lodedb_core::engine::acquire_dir_writer_lock(local.path()).unwrap(); + let err = lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + KEY, + ) + .unwrap_err(); + assert!( + err.to_string().contains("lodedb lock"), + "expected writer-lock contention, got: {err}" + ); + drop(held); + + // With the writer gone the same pull succeeds. + lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + KEY, + ) + .unwrap(); + std::env::remove_var("LODEDB_PERSIST_LOCK_TIMEOUT"); +} + +#[test] +fn pull_refuses_over_a_destination_wal_with_pending_records() { + // The destination WAL's records were acknowledged against the OLD lineage; + // replaying them onto a pulled snapshot (or silently truncating them) + // corrupts or loses acked writes. Pull must refuse until the caller + // checkpoints — force-pull (a sync flag) is the explicit discard. + const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(remote.path(), KEY, 2, 1, "remote-side", None); + + let local = tempfile::tempdir().unwrap(); + commit_engine_generation(local.path(), KEY, 1, 1, "local-side", None); + let wal = lodedb_core::storage::wal::wal_path(local.path(), KEY); + lodedb_core::storage::wal::append_record( + &wal, + 2, + "add", + serde_json::json!({"id": "pending-doc"}), + false, + ) + .unwrap(); + + let before = LocalArtifactStore::new(local.path(), false) + .read_pointer(KEY) + .unwrap(); + let err = lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + KEY, + ) + .unwrap_err(); + assert!( + matches!(err, ArtifactStoreError::PendingWal { .. }), + "expected the pending-WAL refusal, got: {err}" + ); + // Nothing moved: pointer unchanged, WAL records intact. + let after = LocalArtifactStore::new(local.path(), false) + .read_pointer(KEY) + .unwrap(); + assert_eq!(before, after); + assert_eq!( + lodedb_core::storage::wal::scan_stats(&wal).unwrap().op_count, + 1 + ); +} + +#[test] +fn semantically_invalid_generation_never_replaces_the_destination_pointer() { + // Byte checksums can be internally consistent while the artifact is + // gibberish to the engine (a forged or corrupted-at-source remote): the + // recorded digest MATCHES the broken bytes. The restore must verify the + // candidate opens BEFORE the pointer swap, so the destination keeps its + // previous, valid generation when the check fails. + const KEY: &str = "cc33dd44ee55ff6600112233445566778899aabbccddeeff0011223344556677"; + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(remote.path(), KEY, 2, 2, "poisoned", None); + // Corrupt the remote's state artifact and re-sign the manifest so every + // byte checksum still validates. + let base = remote.path().join(format!("{KEY}.gen/g2.json")); + let garbage = b"not-a-state-journal"; + fs::write(&base, garbage).unwrap(); + let store = LocalArtifactStore::new(remote.path(), false); + let mut body = store.read_pointer(KEY).unwrap().unwrap(); + body["json"]["base"]["sha256"] = serde_json::Value::String(sha_hex(garbage)); + body["json"]["base"]["file_bytes"] = serde_json::Value::from(garbage.len() as u64); + lodedb_core::storage::commit_manifest::write_commit_manifest( + &lodedb_core::storage::commit_manifest::commit_manifest_path(remote.path(), KEY), + &body, + false, + ) + .unwrap(); + + // The destination already holds a valid generation of its own. + let local = tempfile::tempdir().unwrap(); + let valid = commit_engine_generation(local.path(), KEY, 1, 1, "still-good", None); + + let err = lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + KEY, + ) + .unwrap_err(); + assert!( + !matches!(err, ArtifactStoreError::NotFound(_)), + "the failure must come from the acceptance check, got: {err}" + ); + // The destination still points at its previous generation and still opens. + let current = LocalArtifactStore::new(local.path(), false) + .read_pointer(KEY) + .unwrap() + .unwrap(); + assert_eq!(current, valid); + verify_local_generation_opens(local.path(), KEY).unwrap(); +} + +#[test] +fn pull_confines_the_wal_probe_to_the_destination() { + // The index key can arrive from CLI/remote input; a traversing spelling + // must fail closed before the WAL check (or anything else) touches a + // path outside the destination directory. + let remote = tempfile::tempdir().unwrap(); + let local = tempfile::tempdir().unwrap(); + let err = lodedb_cloud_core::client_ops::pull( + remote.path().to_str().unwrap(), + local.path().to_str().unwrap(), + "../escape", + ) + .unwrap_err(); + assert!( + matches!(err, ArtifactStoreError::Integrity(_)), + "expected path containment to reject the key, got: {err}" + ); +} diff --git a/crates/lodedb-cloud-core/tests/snapshot_identity.rs b/crates/lodedb-cloud-core/tests/snapshot_identity.rs new file mode 100644 index 00000000..cf3cf337 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/snapshot_identity.rs @@ -0,0 +1,67 @@ +//! Tests for `snapshot_identity`: the ids must be the engine's own canonical +//! digests, and redaction must collapse to one logical id without collapsing +//! the snapshot id. + +mod common; + +use common::commit_engine_generation; +use lodedb_cloud_core::{logical_id, snapshot_id, TransferPolicy}; +use serde_json::Value; + +/// The id IS the `body_sha256` the engine writes in `.commit.json` — not +/// a lookalike digest. Pinned against a real engine-committed store by reading +/// the pointer document raw. +#[test] +fn snapshot_id_equals_the_engine_recorded_body_sha256() { + let dir = tempfile::tempdir().unwrap(); + let body = commit_engine_generation(dir.path(), "idx", 1, 1, "v1", None); + + let pointer_raw = std::fs::read(dir.path().join("idx.commit.json")).unwrap(); + let document: Value = serde_json::from_slice(&pointer_raw).unwrap(); + let recorded = document["body_sha256"].as_str().unwrap(); + + assert_eq!(snapshot_id(&body).unwrap(), recorded); +} + +#[test] +fn redaction_shares_the_logical_id_but_not_the_snapshot_id() { + let dir = tempfile::tempdir().unwrap(); + // A payload-bearing commit: redacting it genuinely changes the body. + let full = commit_engine_generation( + dir.path(), + "idx", + 1, + 1, + "v1", + Some(&[("doc-1", "the raw text")]), + ); + let redacted = TransferPolicy::redacted().redact_body(&full); + + assert_eq!( + logical_id(&full).unwrap(), + logical_id(&redacted).unwrap(), + "one engine commit must map to one logical id regardless of redaction" + ); + assert_ne!( + snapshot_id(&full).unwrap(), + snapshot_id(&redacted).unwrap(), + "different bytes must have different snapshot ids" + ); + // A fully redacted body is its own logical form — the property the sync + // classifier's Republish detection rests on. + assert_eq!( + snapshot_id(&redacted).unwrap(), + logical_id(&redacted).unwrap() + ); +} + +#[test] +fn different_content_has_different_ids() { + let dir_a = tempfile::tempdir().unwrap(); + let dir_b = tempfile::tempdir().unwrap(); + let a = commit_engine_generation(dir_a.path(), "idx", 1, 1, "v1", None); + let b = commit_engine_generation(dir_b.path(), "idx", 1, 1, "v2", None); + + assert_ne!(snapshot_id(&a).unwrap(), snapshot_id(&b).unwrap()); + assert_ne!(logical_id(&a).unwrap(), logical_id(&b).unwrap()); +} diff --git a/crates/lodedb-cloud-core/tests/store_target.rs b/crates/lodedb-cloud-core/tests/store_target.rs new file mode 100644 index 00000000..3b8f55c8 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/store_target.rs @@ -0,0 +1,35 @@ +//! Tests for `artifact_store_from_target`: mapping a target string (local path +//! or object-store URL) onto the right `ArtifactStore` backend. + +use lodedb_cloud_core::{artifact_store_from_target, ArtifactStoreError}; + +#[test] +fn a_plain_path_resolves_to_a_local_store() { + // Resolution is lazy (no filesystem access), so any path shape works here; + // reads/writes are what touch disk. + let store = artifact_store_from_target("/tmp/some-backup-dir").unwrap(); + // An empty directory has no committed pointer. + assert!(matches!( + store.read_bytes("idx.gen/absent"), + Err(ArtifactStoreError::NotFound(_) | ArtifactStoreError::Io(_)) + )); +} + +/// `unwrap_err` would need the store to be `Debug`, so unpack by hand. +fn expect_backend_error(target: &str) { + match artifact_store_from_target(target) { + Err(ArtifactStoreError::Backend(_)) => {} + Err(other) => panic!("expected a Backend error, got {other:?}"), + Ok(_) => panic!("expected {target:?} to be rejected"), + } +} + +#[test] +fn an_s3_url_without_a_bucket_is_rejected() { + expect_backend_error("s3://"); +} + +#[test] +fn an_unknown_scheme_is_rejected() { + expect_backend_error("ftp://bucket/prefix"); +} diff --git a/crates/lodedb-cloud-core/tests/sync_ops.rs b/crates/lodedb-cloud-core/tests/sync_ops.rs new file mode 100644 index 00000000..3e73a460 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/sync_ops.rs @@ -0,0 +1,663 @@ +//! End-to-end tests for the `sync` verb: two working directories sharing one +//! remote, exercising every classification the way real use produces them — +//! plus `contains` on both store backends. + +mod common; + +use common::commit_engine_generation; +use object_store::memory::InMemory; +use lodedb_cloud_core::client_ops::{status, sync, SyncForce}; +use lodedb_cloud_core::{ArtifactStore, ArtifactStoreError, ObjectArtifactStore, TransferPolicy}; +use std::path::Path; +use std::sync::Arc; + +const KEY: &str = "idx"; + +fn run_sync(dir: &Path, remote: &Path, force: SyncForce) -> lodedb_cloud_core::SyncOutcome { + sync( + dir.to_str().unwrap(), + remote.to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + force, + ) + .unwrap() +} + +#[test] +fn the_two_clone_lifecycle() { + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + // Fresh push: no base, remote absent -> local_ahead -> push. + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("local_ahead", "push") + ); + + // Nothing changed: in sync, no transfer. + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("in_sync", "none") + ); + assert_eq!(outcome.transfer, None); + + // A new local commit fast-forwards the remote. + commit_engine_generation(dir1.path(), KEY, 2, 2, "v2", None); + let report = status( + dir1.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + ) + .unwrap(); + assert_eq!(report.classification.as_deref(), Some("local_ahead")); + assert!(report.sidecar_present); + assert_eq!(report.base_generation, Some(1)); + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "push"); + + // A second clone pulls: local absent -> remote_ahead -> pull + verify-open. + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("remote_ahead", "pull") + ); + assert!( + outcome.open.is_some(), + "a pull proves the restored copy opens" + ); + + // The second clone commits and pushes; the first is now behind and pulls. + commit_engine_generation(dir2.path(), KEY, 3, 3, "v3", None); + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "push"); + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("remote_ahead", "pull") + ); + + // Both commit independently: diverged, sync refuses without force. (The + // two lineages use different base epochs here: colliding artifact *names* + // across divergent lineages is the fork-collision case the Phase-2 + // content-addressed remote layout absorbs; a dumb remote refuses it at the + // immutable-artifact check instead.) + commit_engine_generation(dir1.path(), KEY, 4, 4, "v4-dir1", None); + commit_engine_generation(dir2.path(), KEY, 4, 5, "v4-dir2", None); + run_sync(dir2.path(), remote.path(), SyncForce::None); // dir2 pushes first + let err = sync( + dir1.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::SyncConflict { + classification, + hint, + } => { + assert_eq!(classification, "diverged"); + assert!(hint.contains("--force-push") && hint.contains("--force-pull")); + } + other => panic!("expected SyncConflict, got {other:?}"), + } + + // Force-push keeps dir1's copy; dir2 is then a plain fast-forward pull. + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::Push); + assert_eq!((outcome.action.as_str(), outcome.forced), ("push", true)); + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("remote_ahead", "pull") + ); +} + +#[test] +fn a_full_push_after_a_redacted_one_is_a_republish_not_a_divergence() { + let dir = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", Some(&[("doc-1", "raw text")])); + + // Redacted push first (the default posture). + let outcome = run_sync(dir.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "push"); + let remote_store = lodedb_cloud_core::LocalArtifactStore::new(remote.path(), false); + let published = remote_store.read_pointer(KEY).unwrap().unwrap(); + assert!( + published["tvtext"].is_null(), + "redacted push must not ship text" + ); + + // The same commit pushed with text is an upgrade of the same lineage. + let outcome = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::full(), + SyncForce::None, + ) + .unwrap(); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("republish", "push") + ); + let published = remote_store.read_pointer(KEY).unwrap().unwrap(); + assert!( + !published["tvtext"].is_null(), + "republish must upgrade the remote to full" + ); + + // And it converges: the full policy now matches the full remote. + let outcome = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::full(), + SyncForce::None, + ) + .unwrap(); + assert_eq!(outcome.action, "none"); +} + +#[test] +fn an_untrusted_base_requires_force() { + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + // Two ends that never synced (no sidecar) holding different content. + // (Different base epochs — see the fork-collision note above.) + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); + commit_engine_generation(dir2.path(), KEY, 1, 2, "different", None); + + let err = sync( + dir2.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::SyncConflict { classification, .. } => { + assert_eq!(classification, "unknown"); + } + other => panic!("expected SyncConflict, got {other:?}"), + } + + // Force-pull resolves toward the remote, opens it, and records a base. + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::Pull); + assert_eq!((outcome.action.as_str(), outcome.forced), ("pull", true)); + assert!(outcome.open.is_some()); + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "none"); +} + +/// The sidecar base is a claim about one specific pair of ends: pointing the +/// same directory at a *different* remote must not inherit the old remote's +/// history (that would classify unrelated content as a fast-forward and pull +/// it over the local copy without force). +#[test] +fn a_base_recorded_against_another_remote_is_not_trusted() { + let dir = tempfile::tempdir().unwrap(); + let other_dir = tempfile::tempdir().unwrap(); + let remote_a = tempfile::tempdir().unwrap(); + let remote_b = tempfile::tempdir().unwrap(); + + // dir syncs with remote A; remote B holds an unrelated lineage. + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", None); + run_sync(dir.path(), remote_a.path(), SyncForce::None); + commit_engine_generation(other_dir.path(), KEY, 2, 2, "unrelated", None); + run_sync(other_dir.path(), remote_b.path(), SyncForce::None); + + // Against remote B, dir's local copy still equals remote A's base — but + // that base must not be trusted here: no fast-forward pull of B's content. + let err = sync( + dir.path().to_str().unwrap(), + remote_b.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::SyncConflict { classification, .. } => { + assert_eq!(classification, "unknown"); + } + other => panic!("expected SyncConflict, got {other:?}"), + } + + // Status agrees: the sidecar exists but is not trusted for this remote. + let report = status( + dir.path().to_str().unwrap(), + remote_b.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + ) + .unwrap(); + assert!(!report.sidecar_present); + assert!(!report.sidecar_corrupt); + assert_eq!(report.classification.as_deref(), Some("unknown")); + + // An explicit force re-homes the directory onto remote B. + let outcome = run_sync(dir.path(), remote_b.path(), SyncForce::Pull); + assert_eq!(outcome.action, "pull"); + let outcome = run_sync(dir.path(), remote_b.path(), SyncForce::None); + assert_eq!(outcome.action, "none"); +} + +/// A remote that carries text but not lexical is *partially* redacted: a sync +/// whose policy adds the missing store must republish, not report in-sync +/// (regression: `Republish` originally fired only for fully-redacted remotes). +#[test] +fn a_partial_redaction_upgrade_republishes() { + let dir = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + common::commit_engine_generation_with_lexical( + dir.path(), + KEY, + 1, + 1, + "v1", + Some(&[("doc-1", "raw text")]), + true, + ); + + // First sync ships text but not lexical. + let text_only = TransferPolicy { + include_text: true, + include_lexical: false, + }; + let outcome = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + text_only, + SyncForce::None, + ) + .unwrap(); + assert_eq!(outcome.action, "push"); + let remote_store = lodedb_cloud_core::LocalArtifactStore::new(remote.path(), false); + let published = remote_store.read_pointer(KEY).unwrap().unwrap(); + assert!(!published["tvtext"].is_null()); + assert!(published["tvlex"].is_null()); + + // Full policy on the same commit: the missing lexical store must ship. + let outcome = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::full(), + SyncForce::None, + ) + .unwrap(); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("republish", "push") + ); + let published = remote_store.read_pointer(KEY).unwrap().unwrap(); + assert!(!published["tvtext"].is_null()); + assert!(!published["tvlex"].is_null()); + + // And the reverse (policy narrower than the remote) stays a no-op. + let outcome = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + text_only, + SyncForce::None, + ) + .unwrap(); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("in_sync", "none") + ); +} + +/// The Phase-0 fork-collision limit, pinned: two lineages diverging from one +/// base reuse the same artifact file names with different bytes, and force +/// cannot resolve that against a name-addressed remote — sync fails closed +/// with a recovery hint instead of overwriting an immutable artifact. (The +/// managed content-addressed layout of a later milestone absorbs this shape.) +#[test] +fn a_same_name_fork_fails_closed_with_a_recovery_hint_even_when_forced() { + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + // Both clones share base gen 1, then each commits gen 2 at epoch 2 — the + // natural fork shape: same artifact names, different bytes. + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); + run_sync(dir2.path(), remote.path(), SyncForce::None); + commit_engine_generation(dir1.path(), KEY, 2, 2, "v2-dir1", None); + let dir2_body = commit_engine_generation(dir2.path(), KEY, 2, 2, "v2-dir2", None); + run_sync(dir2.path(), remote.path(), SyncForce::None); // dir2 publishes first + + // Unforced sync refuses as diverged; forced push hits the immutability + // invariant and fails closed with guidance, leaving the remote intact. + let err = sync( + dir1.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::SyncConflict { .. })); + + let err = sync( + dir1.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::Push, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::Integrity(message) => { + assert!(message.contains("divergent lineages"), "got: {message}"); + assert!(message.contains("fresh"), "got: {message}"); + } + other => panic!("expected Integrity with recovery hint, got {other:?}"), + } + // The failed force moved no pointer: the remote still holds dir2's fork + // (as its redacted push published it). + let remote_store = lodedb_cloud_core::LocalArtifactStore::new(remote.path(), false); + let published = remote_store.read_pointer(KEY).unwrap().unwrap(); + assert_eq!( + published, + TransferPolicy::redacted().redact_body(&dir2_body) + ); +} + +/// An unpublished payload upgrade is not discarded by a fast-forward: if the +/// base was recorded redacted, the local copy now syncs with text, and the +/// remote has meanwhile advanced, both sides have moved — force required. +#[test] +fn an_unpublished_payload_upgrade_blocks_a_fast_forward_pull() { + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + // dir1 publishes commit A redacted (the base records the redacted view). + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", Some(&[("doc-1", "raw text")])); + run_sync(dir1.path(), remote.path(), SyncForce::None); + + // A second clone advances the remote to commit B. + run_sync(dir2.path(), remote.path(), SyncForce::None); + commit_engine_generation(dir2.path(), KEY, 2, 2, "v2", None); + run_sync(dir2.path(), remote.path(), SyncForce::None); + + // dir1 now asks to sync WITH text: its policy view carries payload the + // base never published. Pulling B would silently drop that text. + let err = sync( + dir1.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::full(), + SyncForce::None, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::SyncConflict { classification, .. } => { + assert_eq!(classification, "diverged"); + } + other => panic!("expected SyncConflict, got {other:?}"), + } + + // The redacted view, by contrast, adds nothing over the base: plain pull. + let outcome = run_sync(dir1.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("remote_ahead", "pull") + ); +} + +/// The remote is trusted by canonical identity, not raw spelling: two +/// spellings of one directory share a sidecar base, so a re-spelled target +/// does not spuriously demand force. +#[test] +fn equivalent_remote_spellings_share_the_base() { + let dir = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", None); + run_sync(dir.path(), remote.path(), SyncForce::None); + // Advance locally so a mistrusted base would classify unknown (both ends + // present and different), not merely in_sync. + commit_engine_generation(dir.path(), KEY, 2, 2, "v2", None); + + // Same directory, different spelling: a redundant `.` segment. + let respelled = remote + .path() + .parent() + .unwrap() + .join(".") + .join(remote.path().file_name().unwrap()); + let outcome = sync( + dir.path().to_str().unwrap(), + respelled.to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap(); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("local_ahead", "push") + ); +} + +/// An in-sync no-op repairs a stale base (e.g. a prior transfer whose sidecar +/// write failed), so lineage recovers without a force. +#[test] +fn an_in_sync_no_op_repairs_a_stale_base() { + let dir = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", None); + run_sync(dir.path(), remote.path(), SyncForce::None); + let good = lodedb_cloud_core::read_sync_state(dir.path(), KEY) + .unwrap() + .state + .unwrap(); + + // Advance both ends together, then restore the OLD sidecar — the state a + // crashed post-transfer sidecar write leaves behind. + commit_engine_generation(dir.path(), KEY, 2, 2, "v2", None); + run_sync(dir.path(), remote.path(), SyncForce::None); + lodedb_cloud_core::write_sync_state(dir.path(), &good).unwrap(); + + // In sync with a stale base: the no-op refreshes it... + let outcome = run_sync(dir.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "none"); + let refreshed = lodedb_cloud_core::read_sync_state(dir.path(), KEY) + .unwrap() + .state + .unwrap(); + assert_eq!(refreshed.base.generation, 2); +} + +/// An in-sync no-op against a remote with no trusted base records one, so +/// later runs classify as fast-forwards instead of unknown. +#[test] +fn an_in_sync_no_op_establishes_a_base() { + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + // dir1 publishes; dir2 obtains the identical content via pull (which + // records a base), then loses its sidecar (e.g. a copy that dropped it). + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); + run_sync(dir2.path(), remote.path(), SyncForce::None); + std::fs::remove_file(lodedb_cloud_core::sync_state::sync_state_path(dir2.path(), KEY)).unwrap(); + + // In sync, no trusted base: the no-op records one... + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "none"); + + // ...so a remote advance is now a plain fast-forward, not unknown. + commit_engine_generation(dir1.path(), KEY, 2, 2, "v2", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::None); + assert_eq!( + (outcome.classification.as_str(), outcome.action.as_str()), + ("remote_ahead", "pull") + ); +} + +#[test] +fn a_corrupt_sidecar_is_surfaced_and_treated_as_no_base() { + let dir = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", None); + run_sync(dir.path(), remote.path(), SyncForce::None); + + // Corrupt the sidecar; the two ends are still identical, so sync is a + // harmless no-op — but the corruption is reported, by status too. + let sidecar = lodedb_cloud_core::sync_state::sync_state_path(dir.path(), KEY); + std::fs::write(&sidecar, b"{ not json").unwrap(); + let report = status( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + ) + .unwrap(); + assert!(!report.sidecar_present); + assert!(report.sidecar_corrupt); + let outcome = run_sync(dir.path(), remote.path(), SyncForce::None); + assert_eq!(outcome.action, "none"); + assert!(outcome.sidecar_corrupt); + + // The in-sync no-op re-established a trusted base over the corrupt file. + let report = status( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + ) + .unwrap(); + assert!(report.sidecar_present); + assert!(!report.sidecar_corrupt); + + // When the corruption actually causes a refusal (ends differ, base + // untrusted), the error itself carries the diagnosis — the CLI's warning + // path never runs on an error. + commit_engine_generation(dir.path(), KEY, 2, 2, "v2", None); + run_sync(dir.path(), remote.path(), SyncForce::None); // remote at v2 + commit_engine_generation(dir.path(), KEY, 3, 3, "v3", None); // local ahead + std::fs::write(&sidecar, b"{ not json").unwrap(); + let err = sync( + dir.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + match &err { + ArtifactStoreError::SyncConflict { + classification, + hint, + } => { + assert_eq!(classification, "unknown"); + assert!(hint.contains("corrupt"), "got hint: {hint}"); + } + other => panic!("expected SyncConflict, got {other:?}"), + } +} + +#[test] +fn sync_rejects_a_url_local_end() { + let err = sync( + "s3://bucket/prefix", + "/tmp/whatever", + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Backend(_))); +} + +#[test] +fn contains_on_both_backends() { + // Local store. + let dir = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), KEY, 1, 1, "v1", None); + let local = lodedb_cloud_core::LocalArtifactStore::new(dir.path(), false); + assert!(local.contains("idx.gen/g1.json").unwrap()); + assert!(!local.contains("idx.gen/no-such-artifact").unwrap()); + + // Object store (HEAD-based override). + let object = ObjectArtifactStore::new(Arc::new(InMemory::new()), "tenant").unwrap(); + let payload = b"payload"; + let digest = common::sha_hex(payload); + object + .write_bytes_if_absent("blob", payload, &digest) + .unwrap(); + assert!(object.contains("blob").unwrap()); + assert!(!object.contains("missing").unwrap()); +} + +#[test] +fn sync_pull_refuses_a_pending_wal_and_force_pull_discards_it() { + // dir2 is behind the remote AND holds acknowledged-but-uncheckpointed WAL + // records: a plain sync must refuse (those records were acked against the + // old lineage), and --force-pull — the explicit "keep the remote copy" + // decision — discards them along with the local lineage. + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let remote = tempfile::tempdir().unwrap(); + + commit_engine_generation(dir1.path(), KEY, 1, 1, "v1", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); + run_sync(dir2.path(), remote.path(), SyncForce::None); // dir2 clones v1 + commit_engine_generation(dir1.path(), KEY, 2, 2, "v2", None); + run_sync(dir1.path(), remote.path(), SyncForce::None); // remote moves ahead + + let wal = lodedb_core::storage::wal::wal_path(dir2.path(), KEY); + lodedb_core::storage::wal::append_record( + &wal, + 2, + "add", + serde_json::json!({"id": "acked-but-uncheckpointed"}), + false, + ) + .unwrap(); + + let err = sync( + dir2.path().to_str().unwrap(), + remote.path().to_str().unwrap(), + KEY, + TransferPolicy::redacted(), + SyncForce::None, + ) + .unwrap_err(); + assert!( + matches!(err, ArtifactStoreError::PendingWal { .. }), + "expected the pending-WAL refusal, got: {err}" + ); + assert_eq!( + lodedb_core::storage::wal::scan_stats(&wal).unwrap().op_count, + 1, + "a refusal must not touch the WAL" + ); + + let outcome = run_sync(dir2.path(), remote.path(), SyncForce::Pull); + assert_eq!((outcome.action.as_str(), outcome.forced), ("pull", true)); + assert_eq!( + lodedb_core::storage::wal::scan_stats(&wal).unwrap().op_count, + 0, + "force-pull discards the pending records with the local lineage" + ); +} diff --git a/crates/lodedb-cloud-core/tests/sync_plan.rs b/crates/lodedb-cloud-core/tests/sync_plan.rs new file mode 100644 index 00000000..1c51c7ba --- /dev/null +++ b/crates/lodedb-cloud-core/tests/sync_plan.rs @@ -0,0 +1,273 @@ +//! Tests for the pure sync classifier: the explicit decision table, plus +//! invariants asserted over an exhaustive enumeration of a small domain. + +use lodedb_cloud_core::{classify, SnapRef, SyncClassification}; + +/// A snapshot of commit `tag` carrying the given payload stores. Snapshots of +/// one commit share a logical id and per-store identities; each distinct +/// store combination is a distinct snapshot id (matching how redaction +/// changes the body bytes). +fn snap(tag: &str, generation: u64, has_text: bool, has_lexical: bool) -> SnapRef { + let snapshot_id = if has_text || has_lexical { + format!( + "snap-{tag}-t{}l{}", + u8::from(has_text), + u8::from(has_lexical) + ) + } else { + // The fully redacted form is its own logical identity. + format!("logical-{tag}") + }; + SnapRef { + snapshot_id, + logical_id: format!("logical-{tag}"), + generation, + text_id: has_text.then(|| format!("text-{tag}")), + lexical_id: has_lexical.then(|| format!("lex-{tag}")), + } +} + +/// A snapshot of commit `tag` whose *text store identity* disagrees with +/// [`snap`]'s — same commit, same store set, different payload bytes (a +/// re-encoded or tampered text store). +fn conflicting_text(tag: &str, generation: u64, has_lexical: bool) -> SnapRef { + let mut reference = snap(tag, generation, true, has_lexical); + reference.snapshot_id = format!("{}-alttext", reference.snapshot_id); + reference.text_id = Some(format!("text-{tag}-alt")); + reference +} + +/// A full (text + lexical) snapshot of commit `tag`. +fn full(tag: &str, generation: u64) -> SnapRef { + snap(tag, generation, true, true) +} + +/// The fully redacted form of `full(tag, _)`: snapshot id == logical id. +fn redacted(tag: &str, generation: u64) -> SnapRef { + snap(tag, generation, false, false) +} + +use SyncClassification::*; + +/// One decision-table row: (local, base, remote) -> expected classification. +type Case<'a> = ( + Option<&'a SnapRef>, + Option<&'a SnapRef>, + Option<&'a SnapRef>, + SyncClassification, +); + +#[test] +fn the_decision_table() { + let a = full("a", 2); + let a_red = redacted("a", 2); + let a_text = snap("a", 2, true, false); + let a_lex = snap("a", 2, false, true); + let a_text_alt = conflicting_text("a", 2, false); + let b = full("b", 3); + let c = full("c", 3); + let rollback = full("r", 1); + + let cases: &[Case] = &[ + // Nothing anywhere / one side absent. + (None, None, None, InSync), + (None, Some(&a), None, InSync), + (Some(&a), None, None, LocalAhead), + (Some(&a), Some(&a), None, LocalAhead), + (None, None, Some(&a), RemoteAhead), + (None, Some(&a), Some(&a), RemoteAhead), + // Same content both sides: in sync (absent or matching base). + (Some(&a), None, Some(&a), InSync), + (Some(&a), Some(&a), Some(&a), InSync), + // Fast-forwards: the side equal to the base did not move. + (Some(&b), Some(&a), Some(&a), LocalAhead), + (Some(&b), Some(&a), Some(&a_red), LocalAhead), + (Some(&a), Some(&a), Some(&b), RemoteAhead), + (Some(&a_red), Some(&a), Some(&b), RemoteAhead), + // Both moved past the base: force required. + (Some(&b), Some(&a), Some(&c), Diverged), + // No base while the ends differ: no way to pick a direction. + (Some(&a), None, Some(&b), Unknown), + // Generation regression against the base: rollback/tamper suspect — + // even when the two ends agree with each other (a coordinated + // rollback must be loud, not a silent no-op or republish). + (Some(&rollback), Some(&a), Some(&a), Unknown), + (Some(&a), Some(&a), Some(&rollback), Unknown), + (Some(&a), Some(&b), Some(&a), Unknown), + (Some(&rollback), Some(&a), Some(&rollback), Unknown), + // ...including toward an absent side: auto-pushing/pulling a + // rolled-back generation would silently re-establish it as current. + (Some(&rollback), Some(&a), None, Unknown), + (None, Some(&a), Some(&rollback), Unknown), + ( + Some(&rollback), + Some(&a), + Some(&snap("r", 1, false, false)), + Unknown, + ), + // Same engine commit, local carrying strictly more payload stores: a + // push republishes the pointer without any lineage conflict. + (Some(&a), Some(&a), Some(&a_red), Republish), + (Some(&a), None, Some(&a_red), Republish), + (Some(&a), Some(&a), Some(&a_text), Republish), + (Some(&a_text), Some(&a), Some(&a_red), Republish), + // The mirror is NOT a republish: pushing a store-subset local over the + // remote would drop stores, so it stays a no-op. + (Some(&a_red), Some(&a), Some(&a), InSync), + (Some(&a_text), Some(&a), Some(&a), InSync), + // Incomparable store sets of one commit: publishing either way drops + // a store the other has — force required. + (Some(&a_text), Some(&a), Some(&a_lex), Unknown), + (Some(&a_lex), Some(&a), Some(&a_text), Unknown), + // Same commit, same store set, but a shared store's payload identity + // differs (re-encoded or tampered text): never silently "in sync", + // and never a republish even when local carries more stores. + (Some(&a_text), Some(&a), Some(&a_text_alt), Unknown), + (Some(&a), Some(&a), Some(&a_text_alt), Unknown), + // A side that enriched the base with payload has moved even though + // its logical id has not: an unpublished `--include-text` upgrade of + // the last-synced commit must not be discarded by a fast-forward + // when the other end advanced. + (Some(&a), Some(&a_red), Some(&b), Diverged), + (Some(&b), Some(&a_red), Some(&a), Diverged), + (Some(&a_text_alt), Some(&a_text), Some(&b), Diverged), + // Whereas a side whose payload merely narrowed (policy view smaller + // than the recorded base) has nothing to lose: still a fast-forward. + (Some(&a_red), Some(&a), Some(&b), RemoteAhead), + (Some(&b), Some(&a), Some(&a_red), LocalAhead), + ]; + + for (index, (local, base, remote, expected)) in cases.iter().enumerate() { + assert_eq!( + classify(*local, *base, *remote), + *expected, + "case #{index}: local={local:?} base={base:?} remote={remote:?}" + ); + } +} + +/// Every combination of local/base/remote drawn from a small domain (absent, +/// two full commits at different generations, the redacted form of the first, +/// and a low-generation commit) — the classifier must uphold its global +/// invariants everywhere, not just on the table above. +#[test] +fn invariants_hold_over_the_exhaustive_domain() { + let a = full("a", 2); + let a_red = redacted("a", 2); + let a_text = snap("a", 2, true, false); + let a_lex = snap("a", 2, false, true); + let a_text_alt = conflicting_text("a", 2, false); + let b = full("b", 3); + let low = full("low", 0); + let domain: Vec> = vec![ + None, + Some(&a), + Some(&a_red), + Some(&a_text), + Some(&a_lex), + Some(&a_text_alt), + Some(&b), + Some(&low), + ]; + + for local in &domain { + for base in &domain { + for remote in &domain { + let classification = classify(*local, *base, *remote); + + // Force is required exactly for Diverged/Unknown; every other + // outcome is a no-op or a single fast-forward. + let requires_force = matches!(classification, Diverged | Unknown); + + // Force is never demanded when there is nothing anywhere, and + // divergence specifically needs both ends (Unknown can arise + // with one absent end: a rollback below the trusted base). + if requires_force { + assert!( + local.is_some() || remote.is_some(), + "force demanded with nothing on either end: \ + base={base:?}" + ); + } + if classification == Diverged { + assert!( + local.is_some() && remote.is_some(), + "divergence needs both ends: local={local:?} \ + base={base:?} remote={remote:?}" + ); + } + + // Identical logical content never classifies as ahead or + // diverged: a generation regression against the base or a + // shared-store identity conflict is Unknown; otherwise + // comparable store sets are InSync/Republish and incomparable + // ones (each end carrying a store the other lacks) demand + // force as Unknown. + if let (Some(local), Some(remote)) = (local, remote) { + if local.logical_id == remote.logical_id { + let regressed = base.is_some_and(|base| { + local.generation < base.generation + || remote.generation < base.generation + }); + let conflicting = local.snapshot_id != remote.snapshot_id + && ((local.text_id.is_some() + && remote.text_id.is_some() + && local.text_id != remote.text_id) + || (local.lexical_id.is_some() + && remote.lexical_id.is_some() + && local.lexical_id != remote.lexical_id)); + let comparable = ((local.text_id.is_none() || remote.text_id.is_some()) + && (local.lexical_id.is_none() || remote.lexical_id.is_some())) + || ((remote.text_id.is_none() || local.text_id.is_some()) + && (remote.lexical_id.is_none() || local.lexical_id.is_some())); + if regressed || conflicting { + assert_eq!( + classification, Unknown, + "same-commit rollback/payload-conflict must demand force: \ + local={local:?} base={base:?} remote={remote:?}" + ); + } else if comparable { + assert!( + matches!(classification, InSync | Republish), + "same commit with comparable stores must be \ + InSync/Republish, got {classification:?} for \ + local={local:?} remote={remote:?}" + ); + } else { + assert_eq!( + classification, Unknown, + "incomparable store sets must demand force: \ + local={local:?} remote={remote:?}" + ); + } + } + } + + // Push/pull duality: swapping the ends mirrors the direction. + // Republish mirrors to InSync (a redacted local never + // "upgrades" a full remote). + let mirrored = classify(*remote, *base, *local); + let expected_mirror = match classification { + LocalAhead => RemoteAhead, + RemoteAhead => LocalAhead, + Republish => InSync, + InSync => { + // InSync may mirror to Republish (the redaction + // asymmetry); both directions must still be + // conflict-free. + assert!( + matches!(mirrored, InSync | Republish), + "InSync mirrored to {mirrored:?}" + ); + continue; + } + other => other, + }; + assert_eq!( + mirrored, expected_mirror, + "duality broken for local={local:?} base={base:?} remote={remote:?}" + ); + } + } + } +} diff --git a/crates/lodedb-cloud-core/tests/sync_state.rs b/crates/lodedb-cloud-core/tests/sync_state.rs new file mode 100644 index 00000000..fcb303ba --- /dev/null +++ b/crates/lodedb-cloud-core/tests/sync_state.rs @@ -0,0 +1,125 @@ +//! Tests for the sync sidecar: round-trip, fail-closed on corruption, atomic +//! rewrite, and the two placement guarantees (never mistaken for an index, +//! never eaten by the engine's epoch GC). + +mod common; + +use common::commit_engine_generation; +use lodedb_cloud_core::generation_inventory::list_index_keys; +use lodedb_cloud_core::sync_state::sync_state_path; +use lodedb_cloud_core::{read_sync_state, write_sync_state, SnapRef, SyncState}; +use std::fs; + +fn state(key: &str, generation: u64) -> SyncState { + SyncState { + index_key: key.to_string(), + remote: "s3://bucket/prefix".to_string(), + base: SnapRef { + snapshot_id: format!("snap-{generation}"), + logical_id: format!("logical-{generation}"), + generation, + text_id: Some(format!("text-{generation}")), + lexical_id: None, + }, + updated_unix: 1_750_000_000, + } +} + +#[test] +fn round_trips_what_was_written() { + let dir = tempfile::tempdir().unwrap(); + let written = state("idx", 7); + write_sync_state(dir.path(), &written).unwrap(); + + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert!(!read.corrupt); + assert_eq!(read.state, Some(written)); +} + +#[test] +fn an_absent_sidecar_reads_as_absent_not_corrupt() { + let dir = tempfile::tempdir().unwrap(); + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert_eq!(read.state, None); + assert!(!read.corrupt); +} + +#[test] +fn a_rewrite_replaces_the_previous_base() { + let dir = tempfile::tempdir().unwrap(); + write_sync_state(dir.path(), &state("idx", 1)).unwrap(); + write_sync_state(dir.path(), &state("idx", 2)).unwrap(); + + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert_eq!(read.state.unwrap().base.generation, 2); +} + +#[test] +fn corruption_reads_as_absent_with_a_warning() { + let dir = tempfile::tempdir().unwrap(); + let path = sync_state_path(dir.path(), "idx"); + + // Torn write: valid JSON prefix, truncated. + write_sync_state(dir.path(), &state("idx", 1)).unwrap(); + let intact = fs::read(&path).unwrap(); + fs::write(&path, &intact[..intact.len() / 2]).unwrap(); + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert_eq!(read.state, None); + assert!(read.corrupt); + + // Tampering: parseable JSON whose fields disagree with the checksum. + let tampered = String::from_utf8(intact.clone()) + .unwrap() + .replace("\"generation\": 1", "\"generation\": 9"); + assert_ne!( + tampered.as_bytes(), + intact.as_slice(), + "tamper must change bytes" + ); + fs::write(&path, tampered).unwrap(); + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert_eq!(read.state, None); + assert!(read.corrupt); + + // A sidecar copied from another index: index_key mismatch. + write_sync_state(dir.path(), &state("other", 1)).unwrap(); + fs::copy(sync_state_path(dir.path(), "other"), &path).unwrap(); + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert_eq!(read.state, None); + assert!(read.corrupt); +} + +/// The sidecar sits beside `.commit.json` but must never be discovered as +/// an index: the engine (and `keys`) scan for the `.commit.json` suffix only. +#[test] +fn the_sidecar_is_not_mistaken_for_an_index() { + let dir = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), "idx", 1, 1, "v1", None); + write_sync_state(dir.path(), &state("idx", 1)).unwrap(); + + assert_eq!( + list_index_keys(dir.path()).unwrap(), + vec!["idx".to_string()] + ); +} + +/// The engine's epoch GC (default: retain 4) deletes artifacts only under +/// `.gen/`; the sidecar lives next to the pointer and must survive any +/// number of commits. +#[test] +fn the_sidecar_survives_engine_epoch_gc() { + let dir = tempfile::tempdir().unwrap(); + commit_engine_generation(dir.path(), "idx", 1, 1, "v1", None); + write_sync_state(dir.path(), &state("idx", 1)).unwrap(); + + // Enough further commits to cycle the retained-epoch window several times. + for epoch in 2..=8u64 { + commit_engine_generation(dir.path(), "idx", epoch, epoch, &format!("v{epoch}"), None); + } + // GC actually ran: the first epoch's base artifact is gone. + assert!(!dir.path().join("idx.gen/g1.json").exists()); + + let read = read_sync_state(dir.path(), "idx").unwrap(); + assert!(!read.corrupt); + assert_eq!(read.state, Some(state("idx", 1))); +} diff --git a/crates/lodedb-cloud-core/tests/verify_status.rs b/crates/lodedb-cloud-core/tests/verify_status.rs new file mode 100644 index 00000000..f49c3ea9 --- /dev/null +++ b/crates/lodedb-cloud-core/tests/verify_status.rs @@ -0,0 +1,183 @@ +//! Tests for `verify_generation` (checksum re-hash), `compare_generations` +//! (push status from two inventories), and `status_for_push` (store-level, +//! policy-aware status). + +mod common; + +use common::*; +use lodedb_cloud_core::{ + compare_generations, export_generation, status_for_push, verify_generation, ArtifactStoreError, + GenerationInventory, LocalArtifactStore, TransferPolicy, +}; +use serde_json::json; +use std::fs; + +#[test] +fn verify_accepts_an_intact_generation() { + let dir = tempfile::tempdir().unwrap(); + let json = store_sub( + dir.path(), + "idx", + "g0.json", + b"base-state", + ".json-delta", + &[("delta-00000000.jsd", b"delta-a")], + ); + write_json_commit(dir.path(), "idx", 1, 0, json); + + let store = LocalArtifactStore::new(dir.path(), false); + let report = verify_generation(&store, "idx").unwrap(); + assert_eq!(report.generation, 1); + assert_eq!(report.artifacts_verified, 2); + assert_eq!( + report.bytes_verified, + (b"base-state".len() + b"delta-a".len()) as u64 + ); +} + +#[test] +fn verify_detects_a_tampered_artifact() { + let dir = tempfile::tempdir().unwrap(); + let json = store_sub( + dir.path(), + "idx", + "g0.json", + b"original", + ".json-delta", + &[], + ); + write_json_commit(dir.path(), "idx", 1, 0, json); + // Corrupt the base file on disk after it was committed: its bytes no longer + // match the checksum the manifest records. + fs::write(dir.path().join("idx.gen/g0.json"), b"tampered").unwrap(); + + let store = LocalArtifactStore::new(dir.path(), false); + let err = verify_generation(&store, "idx").unwrap_err(); + assert!(matches!(err, ArtifactStoreError::Integrity(_))); +} + +#[test] +fn verify_missing_generation_is_not_found() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalArtifactStore::new(dir.path(), false); + let err = verify_generation(&store, "idx").unwrap_err(); + assert!(matches!(err, ArtifactStoreError::NotFound(_))); +} + +#[test] +fn status_reports_a_full_upload_against_an_empty_remote() { + let local = inventory(vec![ + artifact("idx.gen/g0.json", "sha-json", true), + artifact("idx.gen/g0.json.json-delta/d0", "sha-d0", false), + ]); + let report = compare_generations("idx", Some(&local), None); + assert_eq!(report.artifacts_to_upload, 2); + assert!(report.ships_base); + assert!(!report.in_sync); + assert_eq!(report.local_generation, Some(1)); + assert_eq!(report.remote_generation, None); +} + +#[test] +fn status_reports_in_sync_when_remote_holds_everything() { + let local = inventory(vec![artifact("idx.gen/g0.json", "sha-json", true)]); + let report = compare_generations("idx", Some(&local), Some(&local)); + assert_eq!(report.artifacts_to_upload, 0); + assert_eq!(report.bytes_to_upload, 0); + assert!(report.in_sync); +} + +#[test] +fn status_with_no_local_generation_is_in_sync() { + let remote = inventory(vec![artifact("idx.gen/g0.json", "sha-json", true)]); + let report = compare_generations("idx", None, Some(&remote)); + assert!(report.in_sync); + assert_eq!(report.local_generation, None); + assert_eq!(report.remote_generation, Some(1)); + assert_eq!(report.artifacts_to_upload, 0); +} + +#[test] +fn status_for_push_reflects_the_redacted_push_it_describes() { + // Source commits a json store AND a payload-bearing text store. + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let json_sub = store_sub(src.path(), "idx", "g0.json", b"state", ".json-delta", &[]); + let text_sub = store_sub( + src.path(), + "idx", + "g0.tvtext", + b"secret text", + ".tvtext-delta", + &[], + ); + write_commit( + src.path(), + "idx", + 1, + 0, + Some(json_sub), + None, + Some(text_sub), + None, + None, + ); + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + + // Redacted status counts only the json base — the text artifact would not + // ship, so it must not be reported as pending upload. + let redacted = status_for_push(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); + assert_eq!(redacted.artifacts_to_upload, 1); + assert!(!redacted.in_sync); + // A full-policy status against the same empty remote counts both. + let full = status_for_push(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + assert_eq!(full.artifacts_to_upload, 2); + + // After the redacted push actually runs, redacted status is in sync (the + // remote holds exactly the redacted body) while full status is not (a full + // push would still ship the text artifact and republish the pointer). + export_generation(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); + let redacted = status_for_push(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); + assert!(redacted.in_sync); + assert_eq!(redacted.artifacts_to_upload, 0); + let full = status_for_push(&source, &dest, "idx", TransferPolicy::full()).unwrap(); + assert!(!full.in_sync); + assert_eq!(full.artifacts_to_upload, 1); +} + +#[test] +fn status_for_push_with_nothing_committed_anywhere_is_in_sync() { + let src = tempfile::tempdir().unwrap(); + let dst = tempfile::tempdir().unwrap(); + let source = LocalArtifactStore::new(src.path(), false); + let dest = LocalArtifactStore::new(dst.path(), false); + let report = status_for_push(&source, &dest, "idx", TransferPolicy::redacted()).unwrap(); + assert!(report.in_sync); + assert_eq!(report.local_generation, None); + assert_eq!(report.remote_generation, None); +} + +#[test] +fn status_is_not_in_sync_when_bodies_differ_despite_matching_artifacts() { + // A redacted local against a previously-full remote: the shared json artifact + // matches (nothing to upload), but the bodies differ (remote still references + // text). A push would republish the pointer, so this is NOT in sync. + let shared = vec![artifact("idx.gen/g0.json", "sha-json", true)]; + let local = GenerationInventory { + index_key: "idx".into(), + generation: 1, + base_epoch: 0, + document_count: 0, + chunk_count: 0, + root_body: json!({ "generation": 1, "json": {"present": true}, "tvtext": null }), + artifacts: shared.clone(), + }; + let remote = GenerationInventory { + root_body: json!({ "generation": 1, "json": {"present": true}, "tvtext": {"present": true} }), + ..local.clone() + }; + let report = compare_generations("idx", Some(&local), Some(&remote)); + assert_eq!(report.artifacts_to_upload, 0); + assert!(!report.in_sync); +} diff --git a/crates/lodedb-core/src/engine/mod.rs b/crates/lodedb-core/src/engine/mod.rs index 971ca635..eb51b7ed 100644 --- a/crates/lodedb-core/src/engine/mod.rs +++ b/crates/lodedb-core/src/engine/mod.rs @@ -2698,6 +2698,23 @@ struct PersistentLock { _file: File, } +/// RAII exclusive hold on a store directory's single-writer lock +/// (`/.lodedb.lock`), for out-of-band tools that mutate a database +/// directory without opening an engine — the cloud transfer plane's +/// pull/restore path. Contends with every engine writer (native, Python, +/// Swift) across processes; dropping the guard releases the hold. +pub struct DirWriterLock { + _lock: PersistentLock, +} + +/// Takes the exclusive single-writer lock on `dir` for an out-of-band mutation +/// of the database directory (see [`DirWriterLock`]). Fails with the standard +/// writer-contention error when an engine writer (or another guard) holds it; +/// honours `LODEDB_PERSIST_LOCK_TIMEOUT` like the engine's own acquisition. +pub fn acquire_dir_writer_lock(dir: &Path) -> Result { + PersistentLock::acquire(dir).map(|lock| DirWriterLock { _lock: lock }) +} + /// Outcome of one non-blocking attempt to take the writer lock. enum TryLock { /// Another holder blocks this mode; retry until the timeout elapses. diff --git a/pyproject.toml b/pyproject.toml index 5dbafe47..3eaa8557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,13 +86,20 @@ mem0 = ["mem0ai>=2.0.0"] # cognee supports python <3.15, so the marker keeps this extra out of the resolution above # that (LodeDB's own requires-python has no upper bound). cognee = ["cognee>=1.1.0,<2; python_version < '3.15'"] +# The managed cloud (OreCloud): sync/serve LodeDB stores in the managed cloud and use the +# hosted per-user data plane. The client is first-party (`lodedb.cloud` + the `lodedb cloud` +# CLI, over the bundled native transfer core); this extra adds only its third-party deps — +# httpx (the HTTP client) and pynacl (the sealed-box browser-approval login). Both are +# imported lazily, never on a plain `import lodedb` (tests/test_import_boundary.py). +cloud = ["httpx>=0.27", "pynacl>=1.5"] # Optional image + text (CLIP) embedding via the "clip" preset / add_image. CLIP runs on the # sentence-transformers stack (the [torch] tier) and adds Pillow for decoding image files. Both # are imported lazily, so a plain `import lodedb` pulls neither (tests/test_import_boundary.py). image = ["pillow>=10.0.0", "lodedb[torch]"] # Everything a user would reach for: built-in embedding (ONNX + PyTorch), image search, the MCP # server, and the framework adapters. Excludes the niche [onnx-export]/[gpu] and the [dev] set, -# and [cognee] (its large transitive stack — litellm/lancedb/sqlalchemy/fastapi — is opt-in). +# [cognee] (its large transitive stack — litellm/lancedb/sqlalchemy/fastapi — is opt-in), and +# [cloud] (the proprietary managed-cloud client is an explicit choice, never a side effect). all = ["lodedb[embeddings,torch,image,mcp,langchain,llama-index,mem0]"] # ruff is pinned (not floored) so `ruff format`/`ruff check` are reproducible across # contributors and CI — the formatter's output can change between ruff releases. @@ -135,5 +142,11 @@ extend-exclude = ["third_party"] [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] +[tool.ruff.lint.per-file-ignores] +# Generated-content templates (SKILL.md / AGENTS.md scaffolds): YAML frontmatter +# and prose must stay on one physical line inside the template strings, so the +# line-length rule cannot apply without corrupting the generated output. +"src/lodedb/cloud/_agents_scaffold.py" = ["E501"] + [tool.ruff.lint.isort] known-first-party = ["lodedb"] diff --git a/src/lodedb/cloud/__init__.py b/src/lodedb/cloud/__init__.py new file mode 100644 index 00000000..a64aec08 --- /dev/null +++ b/src/lodedb/cloud/__init__.py @@ -0,0 +1,106 @@ +"""``lodedb.cloud`` — the OreCloud managed-cloud client (the ``[cloud]`` extra). + +First-party code, shipped in the lodedb wheel: :class:`Client` (one +credential, one bound org/environment, per-user store handles), +:func:`connect` / :class:`CloudStore` (one end user's store over HTTPS, the +handle ``LodeDB.cloud()`` returns), the transfer verbs (``push`` / ``pull`` / +``sync`` / ``status`` / ``verify`` / ``keys``) over the native transfer core +bundled as ``lodedb._turbovec.cloud``, and the ``lodedb cloud`` CLI. The +``[cloud]`` extra installs only the client's third-party dependencies +(``httpx``, ``pynacl``) — there is no separate cloud distribution. + +Every name resolves lazily (PEP 562): importing this module (or ``lodedb`` +itself) touches neither httpx nor the network, guarded by +``tests/test_import_boundary.py``. Fetching an HTTP-backed name without the +extra's dependencies installed raises an ``ImportError`` carrying the +install hint; the transfer verbs ride the always-present native extension. +""" + +from __future__ import annotations + +import importlib +import inspect +from typing import Any + +# Managed-cloud target scheme recognized by the LodeDB(...) config-string +# dispatch. LodeDB.cloud() additionally accepts scheme-less short forms +# ("user-42", "org/environment/store"); the plain constructor must not, +# because a bare store id is indistinguishable from a relative local path. +CLOUD_TARGET_SCHEME = "orecloud://" + +_INSTALL_HINT = ( + 'the cloud client dependencies are not installed — run: pip install "lodedb[cloud]"' +) + +# The six local↔remote transfer verbs live on the bundled native extension +# (no third-party dependency), so they resolve even without the extra. +_NATIVE_EXPORTS = frozenset({"keys", "pull", "push", "status", "sync", "verify"}) + +# HTTP-backed exports and the submodule each lives in. These modules import +# httpx (and, for login, pynacl) at module level, so they are reached only +# through this lazy table — never on a plain import. +_HTTP_EXPORTS = { + "Client": "client", + "connect": "serving", + "CloudStore": "serving", + "CloudIndex": "serving", # back-compat alias for CloudStore + "CloudSearchHit": "serving", + "CloudClient": "transfer", + "CloudError": "transfer", + "SyncConflictError": "transfer", +} + +__all__ = sorted({"CLOUD_TARGET_SCHEME", "open_cloud_target", *_NATIVE_EXPORTS, *_HTTP_EXPORTS}) + + +def open_cloud_target(target: str, options: dict[str, Any]) -> Any: + """Opens a managed-cloud target via :func:`connect` and returns its store + handle — the one funnel behind both constructor front doors, + ``LodeDB.cloud("user-42")`` and the ``LodeDB("orecloud://…")`` + config-string dispatch. + + The client is imported lazily, only here, so a plain ``import lodedb`` + stays network-free; without the ``[cloud]`` extra's dependencies this + raises an ``ImportError`` carrying the install hint instead of a bare + traceback. ``options`` must be keywords of :func:`connect` (``token``, + ``host``, ...) — local-only construction options (``model=``, + ``read_only=``, ...) are rejected up front, because embedding and + storage for a cloud store are configured server-side, not per handle. + """ + try: + from lodedb.cloud.serving import connect + except ImportError: + raise ImportError( + f"{target!r} is a managed-cloud target, but {_INSTALL_HINT}" + ) from None + # Derive the accepted option set from connect's own signature so the two + # never drift; anything else is a local-only option that has no cloud + # meaning and deserves a targeted error, not a confusing TypeError from + # deep inside the client. + allowed = set(inspect.signature(connect).parameters) - {"target"} + unknown = sorted(set(options) - allowed) + if unknown: + raise TypeError( + f"cloud targets do not accept {', '.join(unknown)}; available " + f"options: {', '.join(sorted(allowed))} (embedding and storage " + "are configured server-side, per store)" + ) + return connect(target, **options) + + +def __getattr__(name: str) -> Any: + """PEP 562: resolves the public surface lazily — transfer verbs from the + bundled native extension, HTTP-backed names from their submodule (with + the install hint when the extra's dependencies are absent).""" + if name in _NATIVE_EXPORTS: + from lodedb import _turbovec + + return getattr(_turbovec.cloud, name) + submodule = _HTTP_EXPORTS.get(name) + if submodule is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + try: + module = importlib.import_module(f"lodedb.cloud.{submodule}") + except ImportError: + raise ImportError(f"lodedb.cloud.{name} is unavailable — {_INSTALL_HINT}") from None + return getattr(module, name) diff --git a/src/lodedb/cloud/_agents_scaffold.py b/src/lodedb/cloud/_agents_scaffold.py new file mode 100644 index 00000000..dff32134 --- /dev/null +++ b/src/lodedb/cloud/_agents_scaffold.py @@ -0,0 +1,203 @@ +"""Generators for `lodedb cloud init --agents`: the artifacts that make a consumer +repo one-shot for a coding agent, generated from the repo's REAL link (host, +org, environment, store) rather than placeholders. + +Three artifacts, all safe to commit (no secrets — credentials stay in env +vars or ~/.orecloud): + +- `.claude/skills/orecloud/SKILL.md` — Agent Skills spec (name + description + frontmatter, short body); read by Claude Code, Cursor, Codex, and Copilot. + Ours to regenerate: rewritten on every run so it tracks the link. +- An `## OreCloud` section appended to `AGENTS.md` (created if absent); + never touched again once the heading exists — the section is theirs to + edit after the first drop. +- An `orecloud-control` entry in `.mcp.json` pointing at the control-plane + MCP endpoint with `${ORECLOUD_TOKEN}` env expansion; merged into an + existing file, skipped if that file is unparseable or already has the + entry. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +AGENTS_SECTION_HEADING = "## OreCloud" +MCP_SERVER_NAME = "orecloud-control" + + +def skill_markdown(host: str, org: str, environment: str, store: str) -> str: + """The .claude/skills/orecloud/SKILL.md body for one linked repo.""" + target = f"{org}/{environment}/{store}" + return f"""--- +name: orecloud +description: Work with this repo's OreCloud-managed LodeDB search store ({target} on {host}). Covers connecting via the Python SDK, asynchronous write semantics (seq / min_seq), semantic search, the lodedb cloud CLI's JSON output and exit codes, and MCP wiring. Use when adding, searching, or syncing documents against OreCloud, or when provisioning stores and API keys. +--- + +# OreCloud + +This repository is linked to a managed OreCloud remote (recorded in +`orecloud.toml` — committable, no secrets): + +- host: `{host}` +- target: `{target}` + +## Connect (Python SDK) + +```python +from lodedb.cloud import Client # pip install "lodedb[cloud]" + +client = Client(token=..., host="{host}") # an ore_sk_ key carries +store = client.store("{store}") # its org/environment +store.add_many([{{"text": "hello", "metadata": {{"kind": "note"}}}}]) +hits = store.search("greeting", k=3) +``` + +Tokens come from the `ORECLOUD_TOKEN` env var (usually the repo's `.env`, +written by `lodedb cloud init --agents`) or `lodedb cloud tokens mint --env-file .env` +— the secret goes straight to the file, never to stdout (scopes: `write`, +`read:search`, `read:text`; use the narrowest set). Mint keys scoped to this +environment (`{environment}`) so the client binds itself; only unscoped +personal tokens pass `org=`/`environment=`. Never hard-code or print a token. + +## Write semantics (important) + +Writes are asynchronous: `add`/`add_many` return once the write is durably +ACCEPTED (returning the document ids), BEFORE it is searchable — visibility +follows within seconds. The SDK handles read-your-writes automatically: a +search on the same handle waits briefly for that handle's own writes, and +`store.wait_for(store.last_write_id)` blocks until a write is fully applied. +Only when calling the raw HTTP API directly: pass the acceptance's `seq` to +search as `min_seq`, and treat HTTP 425 as "not folded yet — retry after a +moment", never as failure. + +## CLI + +The `lodedb cloud` CLI prints JSON whenever stdout is not a terminal (force +with `--json`/`--no-json`), so its output is directly parseable: + +```bash +lodedb cloud store list # registered stores (JSON when piped) +lodedb cloud store create # register + print a connect snippet +lodedb cloud sync cloud # converge a local LodeDB with the cloud +lodedb cloud status cloud # read-only preview of a sync +``` + +Errors land on stderr as `error: …` plus a `hint: …` line naming the exact +next command, with exit codes by class: 2 usage, 3 auth, 4 not found, +5 refused (quota/conflict — do not retry blindly), 6 transient (retry with +backoff). Confirmation prompts never hang a pipe: destructive verbs need +`--yes` when non-interactive. + +## MCP + +The store itself is an MCP server (search/add tools): +`{host}/mcp/{target}` with `Authorization: Bearer `. +Provisioning (environments, stores, keys, rollback) is the separate +control-plane MCP server at `{host}/mcp/control` — see `.mcp.json`. + +## Boundaries + +- Never commit tokens or `~/.orecloud/credentials.json`; `orecloud.toml` + and this skill are safe to commit. +- A 402 answer means a plan limit was reached; reads keep serving — + surface the message instead of retrying. +""" + + +def agents_md_section(host: str, org: str, environment: str, store: str) -> str: + """The `## OreCloud` section appended to the consumer repo's AGENTS.md.""" + target = f"{org}/{environment}/{store}" + return f"""{AGENTS_SECTION_HEADING} + +This repo uses an OreCloud-managed LodeDB store: `{target}` on `{host}` +(link recorded in `orecloud.toml`; no secrets in either file). + +```bash +lodedb cloud status cloud # what a sync would do (read-only) +lodedb cloud sync cloud # converge local and cloud +lodedb cloud store list --environment {environment} # registered stores +``` + +- Credentials: set both `ORECLOUD_HOST` and `ORECLOUD_TOKEN`, or run + `lodedb cloud login --host {host}` once. Never commit tokens. +- The CLI prints JSON when piped; errors carry a `hint:` line and per-class + exit codes (3 auth, 4 not found, 5 refused, 6 transient). +- SDK writes are asynchronous: `add_many` returns the document ids once the + write is accepted, before it is searchable. The SDK handle does + read-your-writes automatically; `store.wait_for(store.last_write_id)` + blocks until the write applies. (Raw HTTP callers pass the acceptance's + `seq` as `min_seq` and treat HTTP 425 as "retry shortly".) +- Full guide: `.claude/skills/orecloud/SKILL.md` (or {host}/llms.txt). +""" + + +def mcp_server_entry(host: str) -> dict: + """The .mcp.json server entry for the control-plane MCP endpoint.""" + return { + "type": "http", + "url": f"{host}/mcp/control", + "headers": {"Authorization": "Bearer ${ORECLOUD_TOKEN}"}, + } + + +def scaffold_agent_artifacts( + local_dir: str | Path, *, host: str, org: str, environment: str, store: str +) -> tuple[list[str], list[str]]: + """Writes the three artifacts into `local_dir`. Returns (written, notes): + repo-relative paths that were created/updated, and human-readable notes + for anything deliberately left alone. Paths use forward slashes on every + platform — they land in the CLI's JSON output, so the shape is a contract, + not a display choice.""" + root = Path(local_dir) + written: list[str] = [] + notes: list[str] = [] + + skill_path = root / ".claude" / "skills" / "orecloud" / "SKILL.md" + skill_path.parent.mkdir(parents=True, exist_ok=True) + skill_path.write_text(skill_markdown(host, org, environment, store), encoding="utf-8") + written.append(skill_path.relative_to(root).as_posix()) + + agents_path = root / "AGENTS.md" + if agents_path.exists(): + existing = agents_path.read_text(encoding="utf-8") + if AGENTS_SECTION_HEADING in existing: + # The section is the repo's to edit after the first drop — + # regenerating over their changes would be data loss. + notes.append(f"AGENTS.md already has an '{AGENTS_SECTION_HEADING}' section; left as is") + else: + joiner = ( + "" if existing.endswith("\n\n") else ("\n" if existing.endswith("\n") else "\n\n") + ) + agents_path.write_text( + existing + joiner + agents_md_section(host, org, environment, store), + encoding="utf-8", + ) + written.append("AGENTS.md") + else: + agents_path.write_text( + agents_md_section(host, org, environment, store), encoding="utf-8" + ) + written.append("AGENTS.md") + + mcp_path = root / ".mcp.json" + entry = mcp_server_entry(host) + if mcp_path.exists(): + try: + config = json.loads(mcp_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + notes.append(".mcp.json exists but is not valid JSON; left as is") + return written, notes + if not isinstance(config, dict): + notes.append(".mcp.json exists but is not a JSON object; left as is") + return written, notes + servers = config.setdefault("mcpServers", {}) + if MCP_SERVER_NAME in servers: + notes.append(f".mcp.json already has an '{MCP_SERVER_NAME}' server; left as is") + return written, notes + servers[MCP_SERVER_NAME] = entry + else: + config = {"mcpServers": {MCP_SERVER_NAME: entry}} + mcp_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + written.append(".mcp.json") + return written, notes diff --git a/src/lodedb/cloud/_config.py b/src/lodedb/cloud/_config.py new file mode 100644 index 00000000..f76b6a9c --- /dev/null +++ b/src/lodedb/cloud/_config.py @@ -0,0 +1,147 @@ +"""Where the CLI keeps its cloud credentials. + +One JSON file, `~/.orecloud/credentials.json`, chmod 0600, holding the +control-plane host and the personal access token from `lodedb cloud login`. +A plain file (not the OS keychain) is deliberate for now: it works headless +— CI boxes, containers, SSH sessions — with the same permission story as +`~/.aws/credentials`. ORECLOUD_TOKEN / ORECLOUD_HOST env vars override the +file entirely, so ephemeral environments never need to write it. +""" + +from __future__ import annotations + +import json +import os +import stat +from dataclasses import dataclass +from pathlib import Path + + +def config_dir() -> Path: + # Computed per call, not at import: ORECLOUD_CONFIG_DIR must work when + # set after import (tests, wrappers). + return Path(os.environ.get("ORECLOUD_CONFIG_DIR", "~/.orecloud")).expanduser() + + +def credentials_file() -> Path: + return config_dir() / "credentials.json" + + +@dataclass(frozen=True) +class Credentials: + host: str + token: str + source: str # "env" or "file" — surfaced by whoami for debuggability + + +class CredentialsError(RuntimeError): + """Credential configuration is wrong in a way that must not be guessed + around (e.g. half-set environment overrides).""" + + +def load_credentials() -> Credentials | None: + env_token = os.environ.get("ORECLOUD_TOKEN") + env_host = os.environ.get("ORECLOUD_HOST") + if env_token and env_host: + return Credentials(host=env_host.rstrip("/"), token=env_token, source="env") + if env_token or env_host: + # Half an override must fail closed: silently mixing an env token + # with a file host (or vice versa) would aim mutating commands at the + # wrong account or control plane. + missing = "ORECLOUD_HOST" if env_token else "ORECLOUD_TOKEN" + present = "ORECLOUD_TOKEN" if env_token else "ORECLOUD_HOST" + raise CredentialsError( + f"{present} is set but {missing} is not — set both to use environment " + "credentials, or unset both to use the stored credentials file" + ) + try: + raw = json.loads(credentials_file().read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return None + host, token = raw.get("host"), raw.get("token") + if not host or not token: + return None + return Credentials(host=host.rstrip("/"), token=token, source="file") + + +def save_credentials(host: str, token: str) -> Path: + target = credentials_file() + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + # Write-then-rename with the final mode already set: the secret never + # exists on disk world-readable, even briefly. + tmp = target.with_suffix(".json.tmp") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as handle: + json.dump({"host": host.rstrip("/"), "token": token}, handle, indent=2) + handle.write("\n") + os.replace(tmp, target) + os.chmod(target, stat.S_IRUSR | stat.S_IWUSR) + return target + + +def delete_credentials() -> bool: + try: + credentials_file().unlink() + return True + except FileNotFoundError: + return False + + +# ---------------------------------------------------------------- orecloud.toml + +# The committable per-directory remote record: which control plane and +# org/environment/store a data directory syncs with. No secrets — credentials +# stay in ~/.orecloud — so the file is safe to commit next to the data. +REMOTE_FILE_NAME = "orecloud.toml" + + +@dataclass(frozen=True) +class RemoteConfig: + host: str + org: str + environment: str + store: str + + +def remote_config_file(local_dir: str | Path) -> Path: + return Path(local_dir) / REMOTE_FILE_NAME + + +def load_remote(local_dir: str | Path) -> RemoteConfig | None: + """The directory's recorded remote, or None. A present-but-malformed file + raises — guessing around a broken record could aim a push at the wrong + environment.""" + import tomllib + + path = remote_config_file(local_dir) + try: + raw = path.read_bytes() + except FileNotFoundError: + return None + try: + table = tomllib.loads(raw.decode("utf-8")).get("remote", {}) + host, org = table["host"], table["org"] + environment = table["environment"] + store = table.get("store", "default") + except (tomllib.TOMLDecodeError, UnicodeDecodeError, KeyError, TypeError) as error: + raise CredentialsError(f"{path} is malformed: {error}") from error + if not all(isinstance(v, str) and v for v in (host, org, environment, store)): + raise CredentialsError(f"{path} is malformed: empty or non-string fields") + return RemoteConfig(host=host.rstrip("/"), org=org, environment=environment, store=store) + + +def save_remote(local_dir: str | Path, config: RemoteConfig) -> Path: + def quote(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + path = remote_config_file(local_dir) + path.write_text( + "# OreCloud managed remote for this data directory (no secrets; safe to commit).\n" + "[remote]\n" + f"host = {quote(config.host.rstrip('/'))}\n" + f"org = {quote(config.org)}\n" + f"environment = {quote(config.environment)}\n" + f"store = {quote(config.store)}\n" + ) + return path diff --git a/src/lodedb/cloud/_env_file.py b/src/lodedb/cloud/_env_file.py new file mode 100644 index 00000000..0d67ecfa --- /dev/null +++ b/src/lodedb/cloud/_env_file.py @@ -0,0 +1,120 @@ +"""Writing credentials into a dotenv file without ever printing them. + +`lodedb cloud tokens mint --env-file` and `lodedb cloud init --agents` land the minted +secret here instead of on stdout: anything printed by an agent-driven CLI run +becomes a permanent part of the agent's transcript, while a file write keeps +the secret on disk only (the same philosophy as the sealed-box login, one hop +further). The writer is deliberately conservative: existing lines it does not +manage are preserved byte-for-byte, managed keys are replaced in place, and a +missing file is created 0600. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +# Patterns in any .gitignore between the env file and the repo root that +# already cover a file named `.env` (or `.env`). Deliberately a simple +# whitelist rather than full gitignore semantics: a miss just means we append +# one more (harmless, idempotent-by-note) line. +_IGNORE_PATTERNS = ("{name}", "/{name}", "*.env", ".env*", "**/{name}") + + +def write_env_values(path: str | Path, values: dict[str, str]) -> Path: + """Set `KEY=value` lines in the dotenv file at `path`, creating it if + missing. Managed keys replace their existing line in place; every other + line — comments, other keys, blanks — is preserved. The result is always + owner-only (0600): secrets land here, and a pre-existing 0644 `.env` must + not keep the minted token group/world-readable. Returns the path.""" + target = Path(path) + try: + lines = target.read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + lines = [] + remaining = dict(values) + updated: list[str] = [] + for line in lines: + key = line.split("=", 1)[0].strip() if "=" in line else None + if key in remaining: + updated.append(f"{key}={remaining.pop(key)}") + else: + updated.append(line) + updated.extend(f"{key}={value}" for key, value in remaining.items()) + content = "\n".join(updated) + "\n" + # Atomic replace with the final mode already set: the secret never exists + # on disk readable to anyone but the owner, even briefly, and an existing + # permissive file's mode is not inherited. mkstemp (unique name, O_EXCL, + # 0600) rather than a fixed sibling name: a predictable scratch path in a + # shared directory could be pre-planted as a symlink, aiming the secret at + # an attacker-chosen file. + import tempfile + + fd, scratch = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + # The rename carries mkstemp's owner-only mode onto the target; no + # post-publication chmod (a path-based chmod on the published name + # would reintroduce the symlink race this scratch file exists to + # avoid, and has no exact-mode meaning on Windows anyway). + os.replace(scratch, target) + except BaseException: + try: + os.unlink(scratch) + except OSError: + pass + raise + return target + + +def read_env_value(path: str | Path, key: str) -> str | None: + """The value of `key` in the dotenv file, or None (missing file or key). + Last assignment wins, matching how dotenv loaders behave.""" + try: + lines = Path(path).read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return None + value = None + for line in lines: + if "=" in line and line.split("=", 1)[0].strip() == key: + value = line.split("=", 1)[1].strip() + return value + + +def _git_root(start: Path) -> Path | None: + """The enclosing git working-tree root, or None when `start` is untracked + territory. A `.git` file (worktrees, submodules) counts.""" + for candidate in (start, *start.parents): + if (candidate / ".git").exists(): + return candidate + return None + + +def ensure_gitignored(env_path: str | Path) -> str | None: + """Make sure the env file cannot be committed: if it lives in a git repo + and no .gitignore between it and the repo root covers it, append its name + to the repo-root .gitignore (creating the file if needed). Returns a + human-readable note describing what happened, or None when the file was + already covered.""" + target = Path(env_path).resolve() + root = _git_root(target.parent) + if root is None: + return f"{target.name} is not inside a git repository — keep it out of version control" + covered = {pattern.format(name=target.name) for pattern in _IGNORE_PATTERNS} + directories = [target.parent] + while directories[-1] != root: + directories.append(directories[-1].parent) + for directory in directories: + gitignore = directory / ".gitignore" + try: + lines = {line.strip() for line in gitignore.read_text(encoding="utf-8").splitlines()} + except FileNotFoundError: + continue + if lines & covered: + return None + root_ignore = root / ".gitignore" + existing = root_ignore.read_text(encoding="utf-8") if root_ignore.exists() else "" + separator = "" if existing.endswith("\n") or not existing else "\n" + root_ignore.write_text(f"{existing}{separator}{target.name}\n", encoding="utf-8") + return f"added {target.name} to {root_ignore}" diff --git a/src/lodedb/cloud/cli.py b/src/lodedb/cloud/cli.py new file mode 100644 index 00000000..2caf44bf --- /dev/null +++ b/src/lodedb/cloud/cli.py @@ -0,0 +1,1677 @@ +"""The `lodedb cloud` command line: the six client verbs over the native +transfer core (`lodedb._turbovec.cloud`), +plus the cloud verbs (login, tenancy, managed transfer). + +Thin by design — argument parsing and report formatting only. Every operation +(and all of its validation) lives in the Rust core or `lodedb.cloud.transfer`, so +the CLI cannot offer a code path the library does not have. + +Agent-native output contract: +- Results print as JSON when stdout is not a terminal (or with `--json`); + the human render is the TTY default. Progress/context lines move to stderr + in JSON mode so stdout stays parseable. +- Errors print to stderr as `error: …` (+ a `hint: …` line naming the exact + next command where one exists), with an exit code per failure class: + 1 unexpected, 2 usage/local config, 3 auth (401/403), 4 not found (404), + 5 refused (402/409/422 — understood and denied), 6 transient (425/429/503 + — retry with backoff). +- Confirmation prompts never hang a pipe: without a TTY they fail + immediately, naming `--yes`. + +Transfer verbs accept three REMOTE spellings: a directory path or `s3://` +URL (the dumb targets, handled entirely by the Rust core), an explicit +`orecloud://org/environment[/store]` managed target, or the literal word +`cloud` — the managed remote recorded in LOCAL_DIR/orecloud.toml by +`lodedb cloud init`/`lodedb cloud link`. +""" + +import json +import sys +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Annotated, TypeVar + +import typer + +from lodedb._turbovec import cloud as _core + +app = typer.Typer( + name="cloud", + help="Durable backup/restore for LodeDB indexes, to a directory or object storage.", + no_args_is_help=True, + add_completion=False, +) + +T = TypeVar("T") + +# Exit-code classes (see the module docstring). +EXIT_UNEXPECTED = 1 +EXIT_USAGE = 2 +EXIT_AUTH = 3 +EXIT_NOT_FOUND = 4 +EXIT_REFUSED = 5 +EXIT_RETRY = 6 + +# Tri-state output mode set by the app callback: True/False from +# --json/--no-json, None = auto (JSON exactly when stdout is not a TTY). +_json_output: bool | None = None + + +@app.callback() +def _main( + json_output: Annotated[ + bool | None, + typer.Option( + "--json/--no-json", + help="Force JSON (or human) output; default: JSON when stdout is not a terminal.", + ), + ] = None, +) -> None: + # Records the output-mode override for this invocation (global option: + # `lodedb cloud --json store list`). + global _json_output + _json_output = json_output + + +def _machine_output() -> bool: + """True when results should print as JSON (forced, or piped stdout).""" + if _json_output is not None: + return _json_output + return not sys.stdout.isatty() + + +def _emit(data: object, human: Callable[[], None]) -> None: + """Prints one result twice over: `data` as JSON for programs, or the + `human` renderer for a terminal.""" + if _machine_output(): + typer.echo(json.dumps(data, indent=2, default=str)) + else: + human() + + +def _emit_report(report: Mapping[str, object]) -> None: + """A report dict as JSON or aligned `field value` lines.""" + _emit(dict(report), lambda: _echo_report(report)) + + +def _note(message: str) -> None: + """Progress/context lines: stdout for people, stderr in JSON mode so + stdout stays pure JSON.""" + typer.echo(message, err=_machine_output()) + +# Shared option declarations for the payload opt-ins (redacted-by-default +# posture: raw text and lexical terms leave the machine only on explicit ask). +IncludeText = Annotated[ + bool, + typer.Option("--include-text", help="Also ship the raw-text store (payload-bearing)."), +] +IncludeLexical = Annotated[ + bool, + typer.Option( + "--include-lexical", + help="Also ship the lexical-index store (tokenised text; payload-bearing).", + ), +] + + +def _run(operation: Callable[[], T]) -> T: + """Runs one binding operation, mapping its exceptions onto stderr + exit 1 + (sync refusals onto the documented "refused" class, like their managed + twins — the message itself carries the force-flag/checkpoint hint).""" + try: + return operation() + except _core.SyncConflictError as error: + typer.secho(f"error: {error}", fg=typer.colors.RED, err=True) + raise typer.Exit(code=EXIT_REFUSED) from error + except (FileNotFoundError, OSError, RuntimeError) as error: + typer.secho(f"error: {error}", fg=typer.colors.RED, err=True) + raise typer.Exit(code=1) from error + + +def _echo_report(report: Mapping[str, object]) -> None: + """Prints a report dict as aligned `field value` lines.""" + width = max(len(field) for field in report) + for field, value in report.items(): + typer.echo(f"{field:<{width}} {value}") + + +@app.command() +def keys(local_dir: str) -> None: + """List the index keys with a committed generation in LOCAL_DIR.""" + found = list(_run(lambda: _core.keys(local_dir))) + + def human() -> None: + for key in found: + typer.echo(key) + + _emit(found, human) + + +@app.command() +def status( + local_dir: str, + remote: str, + key: str, + include_text: IncludeText = False, + include_lexical: IncludeLexical = False, +) -> None: + """Compare LOCAL_DIR against REMOTE for a push of KEY (read-only on both ends).""" + managed = _managed(local_dir, remote) + if managed is not None: + client, target, host = managed + with client: + report = _cloud( + lambda: managed_status( + client, + local_dir, + target, + key, + host=host, + include_text=include_text, + include_lexical=include_lexical, + ) + ) + _emit_report(report) + return + report = _run( + lambda: _core.status( + local_dir, remote, key, include_text=include_text, include_lexical=include_lexical + ) + ) + _emit_report(report) + + +@app.command() +def push( + local_dir: str, + remote: str, + key: str, + include_text: IncludeText = False, + include_lexical: IncludeLexical = False, +) -> None: + """Push KEY's committed generation from LOCAL_DIR to REMOTE (redacted by default).""" + managed = _managed(local_dir, remote) + if managed is not None: + client, target, host = managed + with client: + result = _cloud( + lambda: managed_push( + client, + local_dir, + target, + key, + host=host, + include_text=include_text, + include_lexical=include_lexical, + ) + ) + _emit_report(result) + return + result = _run( + lambda: _core.push( + local_dir, remote, key, include_text=include_text, include_lexical=include_lexical + ) + ) + _emit_report(result) + + +@app.command() +def sync( + local_dir: str, + remote: str, + key: str, + include_text: IncludeText = False, + include_lexical: IncludeLexical = False, + force_push: Annotated[ + bool, + typer.Option( + "--force-push", help="Keep the local copy: push it even over a diverged remote." + ), + ] = False, + force_pull: Annotated[ + bool, + typer.Option( + "--force-pull", help="Keep the remote copy: pull it even over a diverged local." + ), + ] = False, +) -> None: + """Sync KEY between LOCAL_DIR and REMOTE: fast-forward in whichever direction moved. + + Refuses (with a nonzero exit) when the two ends diverged past the last + recorded sync, or when there is no sync record and the ends differ — + resolve with exactly one of --force-push / --force-pull. + """ + if force_push and force_pull: + typer.secho( + "error: --force-push and --force-pull are mutually exclusive", + fg=typer.colors.RED, + err=True, + ) + raise typer.Exit(code=EXIT_USAGE) + managed = _managed(local_dir, remote) + if managed is not None: + client, target, host = managed + with client: + outcome = _cloud( + lambda: managed_sync( + client, + local_dir, + target, + key, + host=host, + include_text=include_text, + include_lexical=include_lexical, + force_push=force_push, + force_pull=force_pull, + ) + ) + else: + outcome = _run( + lambda: _core.sync( + local_dir, + remote, + key, + include_text=include_text, + include_lexical=include_lexical, + force_push=force_push, + force_pull=force_pull, + ) + ) + if outcome.get("sidecar_corrupt"): + typer.secho( + "warning: the sync sidecar was present but corrupt and has been ignored", + fg=typer.colors.YELLOW, + err=True, + ) + _emit_report(outcome) + + +@app.command() +def pull(remote: str, local_dir: str, key: str) -> None: + """Restore KEY's committed generation from REMOTE into LOCAL_DIR and verify it opens.""" + managed = _managed(local_dir, remote) + if managed is not None: + client, target, host = managed + with client: + outcome = _cloud( + lambda: managed_pull(client, target, local_dir, key, host=host) + ) + _emit_report(outcome) + return + outcome = _run(lambda: _core.pull(remote, local_dir, key)) + _emit_report(outcome) + + +@app.command() +def verify(target: str, key: str) -> None: + """Re-hash every artifact KEY's committed generation pins in TARGET.""" + report = _run(lambda: _core.verify(target, key)) + _emit_report(report) + + +# --------------------------------------------------------------------------- +# Cloud verbs: the managed control plane (login, identity, tokens, tenancy). +# These speak HTTP via lodedb.cloud.transfer; everything above stays pure-local. +# --------------------------------------------------------------------------- + +import platform # noqa: E402 +import webbrowser # noqa: E402 +from datetime import UTC # noqa: E402 + +from lodedb.cloud import _config # noqa: E402 +from lodedb.cloud.transfer import ( # noqa: E402 + CloudClient, + CloudError, + LoginHandoff, + ManagedRemote, + managed_pull, + managed_push, + managed_status, + managed_sync, +) + +auth_app = typer.Typer(help="Credential helpers for external tools.", no_args_is_help=True) +tokens_app = typer.Typer(help="Mint, list, and revoke API tokens.", no_args_is_help=True) +environments_app = typer.Typer( + help="List the fixed environments (production and testing).", no_args_is_help=True +) +store_app = typer.Typer(help="Register and list managed stores.", no_args_is_help=True) +mcp_app = typer.Typer( + help="Connect AI agents to a hosted store over MCP.", no_args_is_help=True +) +org_app = typer.Typer( + help="Org lifecycle: delete, restore, trash, and export (offboarding).", + no_args_is_help=True, +) +app.add_typer(auth_app, name="auth") +app.add_typer(tokens_app, name="tokens") +app.add_typer(environments_app, name="environments") +app.add_typer(store_app, name="store") +app.add_typer(mcp_app, name="mcp") +app.add_typer(org_app, name="org") + + +def _fail(message: str, code: int = EXIT_UNEXPECTED, hint: str | None = None) -> typer.Exit: + """Prints `error: message` (+ optional `hint: …` naming the next command) + to stderr and returns the Exit to raise.""" + typer.secho(f"error: {message}", fg=typer.colors.RED, err=True) + if hint: + typer.secho(f"hint: {hint}", err=True) + return typer.Exit(code=code) + + +def _classify(error: CloudError) -> tuple[int, str | None]: + """Maps a control-plane refusal to (exit code, fix hint). The hint names + the exact next command — errors double as prompts for agents.""" + status = error.status_code + if status == 401: + return EXIT_AUTH, ( + "the credential was rejected — run `lodedb cloud login --host `, " + "or set both ORECLOUD_HOST and ORECLOUD_TOKEN" + ) + if status == 403: + return EXIT_AUTH, ( + "this credential lacks a needed scope — mint one with " + "`lodedb cloud tokens mint --scope ` (scopes: admin, write, " + "read:search, read:text)" + ) + if status == 404: + return EXIT_NOT_FOUND, ( + "check the target spelling with `lodedb cloud environments list` / " + "`lodedb cloud store list`" + ) + if status == 402: + return EXIT_REFUSED, ( + "a plan limit was reached — reads keep serving; upgrade or free the " + "resource (see `lodedb cloud org export` and the console's Usage page)" + ) + if status in (409, 422): + return EXIT_REFUSED, None + if status in (425, 429): + return EXIT_RETRY, "transient — retry after a moment" + if status == 503: + return EXIT_RETRY, ( + "retry with backoff; if the message names a missing runtime, that is " + "a deployment fix, not a retry" + ) + return EXIT_UNEXPECTED, None + + +def _confirm(prompt: str) -> None: + """`typer.confirm` with an agent guard: when stdin is not a TTY there is + nobody to answer — fail immediately, naming `--yes`, instead of hanging + the calling program.""" + if not sys.stdin.isatty(): + raise _fail( + "confirmation needed but stdin is not interactive — re-run with --yes", + code=EXIT_USAGE, + ) + typer.confirm(prompt, abort=True) + + +def _load_credentials() -> "_config.Credentials | None": + try: + return _config.load_credentials() + except _config.CredentialsError as error: + raise _fail(str(error), code=EXIT_USAGE) from error + + +def _client() -> CloudClient: + creds = _load_credentials() + if creds is None: + raise _fail("not logged in — run `lodedb cloud login --host `") + return CloudClient(creds.host, creds.token) + + +def _cloud(operation: Callable[[], T]) -> T: + # The same exception surface as _run(), plus CloudError (itself a + # RuntimeError): managed operations interleave HTTP calls with local + # Rust-core work (planning, materialisation), and both halves must + # print as `error: …` rather than a traceback. Control-plane refusals + # additionally classify into per-class exit codes with fix hints. + try: + return operation() + except CloudError as error: + code, hint = _classify(error) + raise _fail(str(error), code=code, hint=hint) from error + except _core.SyncConflictError as error: + # The native core's refusals (diverged lineage, pending WAL) are the + # same "resolve with force or checkpoint" class as the managed 409s. + raise _fail(str(error), code=EXIT_REFUSED) from error + except (FileNotFoundError, OSError, RuntimeError) as error: + raise _fail(str(error)) from error + + +# The tenancy flags share one story: the credential knows where it belongs. +_ORG_HELP = "Org slug; defaults to the token's binding, else your only org." +_ENVIRONMENT_HELP = ( + "Environment slug; defaults to the token's binding, else the org's only environment." +) +_TENANCY_HINT = ( + "name the target with --org/--environment (see `lodedb cloud environments list`)" +) + + +def _org_scope(client: CloudClient, org: str | None) -> str: + """The org a command addresses: the explicit --org, else the token's + binding (environment tokens), else the account's only org — failing + with the actual choices when several orgs qualify.""" + if org: + return org + info = _cloud(client.token_self) + if info["org"]: + return info["org"] + slugs = [row["slug"] for row in _cloud(client.me)["orgs"]] + if not slugs: + raise _fail("your account belongs to no org") + if len(slugs) > 1: + raise _fail( + f"this account belongs to several orgs — pass --org (one of: {', '.join(slugs)})", + code=EXIT_USAGE, + ) + return slugs[0] + + +def _tenancy( + client: CloudClient, org: str | None, environment: str | None +) -> tuple[str, str]: + """The (org, environment) pair a command addresses — the SDK's + resolution (explicit flags win; environment tokens supply their binding + and refuse contradictions; personal tokens fall back to the only + org/environment), re-raised with the CLI flags in the hint.""" + from lodedb.cloud.client import resolve_tenancy + + try: + return resolve_tenancy(client, org, environment) + except CloudError as error: + code, _hint = _classify(error) + raise _fail(str(error), code=code, hint=_TENANCY_HINT) from error + + +def _managed(local_dir: str, remote: str) -> tuple[CloudClient, ManagedRemote, str] | None: + """Resolves REMOTE as a managed target: an explicit `orecloud://` URL or + the literal `cloud` (the remote recorded in LOCAL_DIR/orecloud.toml). + Returns None for dumb targets (paths, `s3://`).""" + if remote == "cloud": + try: + config = _config.load_remote(local_dir) + except _config.CredentialsError as error: + raise _fail(str(error), code=EXIT_USAGE) from error + if config is None: + raise _fail( + f"no {_config.REMOTE_FILE_NAME} in {local_dir!r} — run " + "`lodedb cloud link` (or `lodedb cloud init`) first, or name the remote " + "explicitly as orecloud://org/environment" + ) + creds = _load_credentials() + if creds is None: + raise _fail("not logged in — run `lodedb cloud login --host `") + if creds.host.rstrip("/") != config.host: + raise _fail( + f"this directory is linked to {config.host} but you are logged in to " + f"{creds.host} — log in to the linked control plane or re-link" + ) + target = ManagedRemote(config.org, config.environment, config.store) + return CloudClient(creds.host, creds.token), target, creds.host + target = _cloud(lambda: ManagedRemote.parse(remote)) + if target is None: + return None + creds = _load_credentials() + if creds is None: + raise _fail("not logged in — run `lodedb cloud login --host `") + return CloudClient(creds.host, creds.token), target, creds.host + + +@app.command() +def login( + host: Annotated[ + str | None, + typer.Option("--host", help="Control-plane URL, e.g. https://console.example.com."), + ] = None, + token: Annotated[ + str | None, + typer.Option( + "--token", + help="Skip the browser handoff and store this token directly (CI).", + ), + ] = None, + no_browser: Annotated[ + bool, + typer.Option( + "--no-browser", + help="Don't open a browser; print the approval URL and code instead.", + ), + ] = False, +) -> None: + """Log in to an OreCloud control plane and store the credential locally. + + Default flow: this machine generates a keypair, your browser approves the + login, and the server returns the new token sealed to that keypair — the + secret never crosses in the clear and is never stored server-side. + """ + creds = _load_credentials() + if host is None: + host = creds.host if creds else None + if host is None: + raise _fail("--host is required for the first login") + + if token is None: + token = _browser_login(host, open_browser=not no_browser) + + # Prove the credential works before persisting it — a typo'd --token must + # not leave broken state behind. + with CloudClient(host, token) as client: + me = _cloud(client.me) + path = _config.save_credentials(host, token) + _emit( + {"host": host, "email": me["user"]["email"], "credentials": str(path)}, + lambda: typer.echo( + f"Logged in to {host} as {me['user']['email']} (credentials in {path})." + ), + ) + + +def _browser_login(host: str, open_browser: bool = True) -> str: + """The sealed-box browser handoff against `host`: prints the confirmation + code and approval URL to stderr, blocks until a signed-in browser approves, + and returns the minted token secret (sealed in transit to this process's + keypair — it never crosses in the clear).""" + with CloudClient(host) as anonymous: + handoff = _cloud( + lambda: LoginHandoff( + anonymous, client_label=f"lodedb cloud CLI on {platform.node() or 'unknown host'}" + ) + ) + _note(f"Confirmation code: {handoff.start.user_code}") + _note(f"Approve this login at: {handoff.start.verification_url}") + if open_browser: + webbrowser.open(handoff.start.verification_url) + _note("Waiting for approval…") + return _cloud(handoff.wait) + + +def _ensure_logged_in(host: str | None) -> "_config.Credentials": + """Stored credentials, or an inline sealed-box login when there are none — + the single human touchpoint of `lodedb cloud init`. A `host` that contradicts + the stored credential fails closed rather than silently re-aiming the + command at another control plane.""" + creds = _load_credentials() + if creds is not None: + if host is not None and host.rstrip("/") != creds.host: + raise _fail( + f"already logged in to {creds.host} — run `lodedb cloud login --host {host}` " + "to switch control planes first", + code=EXIT_USAGE, + ) + return creds + if host is None: + raise _fail( + "not logged in — pass --host to log in as part of this command, " + "or run `lodedb cloud login --host ` first", + code=EXIT_USAGE, + ) + token = _browser_login(host) + with CloudClient(host, token) as client: + me = _cloud(client.me) + _config.save_credentials(host, token) + _note(f"logged in to {host} as {me['user']['email']}") + return _config.Credentials(host=host.rstrip("/"), token=token, source="file") + + +@app.command() +def link( + local_dir: str, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + store: Annotated[str, typer.Option(help="Store name.")] = "memory", +) -> None: + """Record LOCAL_DIR's managed remote in orecloud.toml (committable, no + secrets). Transfer verbs then accept the literal remote `cloud`.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + creds = _load_credentials() + assert creds is not None # _client() above required it + path = _config.save_remote( + local_dir, + _config.RemoteConfig(host=creds.host, org=org, environment=environment, store=store), + ) + _emit( + {"org": org, "environment": environment, "store": store, "path": str(path)}, + lambda: typer.echo( + f"linked {local_dir} -> orecloud://{org}/{environment}/{store} ({path})" + ), + ) + + +def _agents_repo_token( + client: CloudClient, + *, + host: str, + org: str, + environment: str, + local_dir: str, + env_file: str | Path, +) -> tuple[dict[str, object], list[str]]: + """The repo's deterministic `init-` scoped key, written into the + dotenv file. Reused when the file still holds a live token of that name; + otherwise every stale same-name token is revoked and one fresh key is + minted — re-running init never accumulates keys. Returns (report data, + notes).""" + from datetime import datetime + + from lodedb.cloud import _env_file as env + + name = f"init-{Path(local_dir).resolve().name}"[:200] + + def alive(row: Mapping[str, object]) -> bool: + if row["revoked_at"] is not None: + return False + expires_at = row["expires_at"] + if expires_at is None: + return True + return datetime.fromisoformat(str(expires_at)) > datetime.now(UTC) + + live = [ + row + for row in _cloud(client.list_tokens) + if row["name"] == name and row["org"] == org and row["environment"] == environment + and alive(row) + ] + existing_secret = env.read_env_value(env_file, "ORECLOUD_TOKEN") + if existing_secret: + held = [row for row in live if existing_secret.startswith(str(row["prefix"]))] + if held: + # Reuse the key the file already holds; only heal the host line + # (the token line stays untouched — we could not rewrite it anyway, + # secrets are shown once). + env.write_env_values(env_file, {"ORECLOUD_HOST": host}) + return ( + {"token": held[0], "env_file": str(env_file)}, + [f"reusing token {name} already in {env_file}"], + ) + notes: list[str] = [] + for row in live: + _cloud(lambda row=row: client.revoke_token(str(row["id"]))) + notes.append(f"revoked stale token {row['prefix']}… ({name})") + # Same scope set `store create` mints: text access stays double-gated by + # the store's own expose_text switch, so read:text here grants nothing on + # stores that keep text closed. + minted = _cloud( + lambda: client.mint_token( + "secret", + ["write", "read:search", "read:text"], + name=name, + org=org, + environment=environment, + ) + ) + data: dict[str, object] = {"token": minted["token"]} + data.update(_write_credential_env(env_file, host, minted["secret"])) + return data, notes + + +@app.command() +def init( + local_dir: str, + environment: Annotated[ + str | None, typer.Option(help=_ENVIRONMENT_HELP) + ] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + store: Annotated[str, typer.Option(help="Store name.")] = "memory", + host: Annotated[ + str | None, + typer.Option( + "--host", + help="Control-plane URL — when no credential is stored, init logs " + "in first (browser approval, the one human step).", + ), + ] = None, + agents: Annotated[ + bool, + typer.Option( + "--agents", + help="Also mint a scoped key into --env-file and drop agent " + "artifacts generated from this link: .claude/skills/orecloud/" + "SKILL.md (regenerated each run), an ## OreCloud section in " + "AGENTS.md (first run only), and an orecloud-control entry in " + ".mcp.json (${ORECLOUD_TOKEN} auth).", + ), + ] = False, + env_file: Annotated[ + str | None, + typer.Option( + "--env-file", + help="Where --agents writes the minted credential " + "(default: LOCAL_DIR/.env; the secret is never printed).", + ), + ] = None, +) -> None: + """Link LOCAL_DIR to one of the org's environments — the one-command path + from a local data directory to a managed remote. With --agents this is + the whole setup for a coding agent: log in if needed (one browser + approval), mint a scoped key into .env, and scaffold the repo's agent + artifacts. Environments are fixed (production and testing, seeded at + signup); init resolves the slug from the credential — or validates an + explicit one — rather than creating anything.""" + creds = _ensure_logged_in(host) + data: dict[str, object] = {} + notes: list[str] = [] + with CloudClient(creds.host, creds.token) as client: + if environment is None: + org, environment = _tenancy(client, org, environment) + else: + org = _org_scope(client, org) + available = sorted( + p["slug"] for p in _cloud(lambda: client.list_environments(org)) + ) + if environment not in available: + raise _fail( + f"no environment {environment!r} in org {org!r}", + code=EXIT_NOT_FOUND, + hint=f"pick one of: {', '.join(available)} (via --environment)", + ) + if agents: + token_data, notes = _run( + lambda: _agents_repo_token( + client, + host=creds.host, + org=org, + environment=environment, + local_dir=local_dir, + env_file=env_file or Path(local_dir) / ".env", + ) + ) + data.update(token_data) + path = _config.save_remote( + local_dir, + _config.RemoteConfig(host=creds.host, org=org, environment=environment, store=store), + ) + data.update(org=org, environment=environment, store=store, path=str(path)) + written: list[str] = [] + if agents: + from lodedb.cloud._agents_scaffold import scaffold_agent_artifacts + + written, scaffold_notes = scaffold_agent_artifacts( + local_dir, host=creds.host, org=org, environment=environment, store=store + ) + notes.extend(scaffold_notes) + data["agents"] = {"written": written, "notes": notes} + + def human() -> None: + typer.echo(f"linked {local_dir} -> orecloud://{org}/{environment}/{store} ({path})") + if "env_file" in data: + typer.echo(f"wrote credential to {data['env_file']}") + for name in written: + typer.echo(f"wrote {name}") + for note in notes: + typer.echo(f"note: {note}") + + _emit(data, human) + + +@auth_app.command("print-headers") +def auth_print_headers() -> None: + """Print the stored credential as an MCP headers object: + {"Authorization": "Bearer …"}. + + The headersHelper contract (Claude Code and friends run a command to + obtain MCP auth headers), so the browser-approved login feeds MCP + servers without anyone pasting a key into a config file. Always JSON — + that IS the output contract.""" + creds = _load_credentials() + if creds is None: + raise _fail( + "not logged in — run `lodedb cloud login --host `", + code=EXIT_AUTH, + ) + typer.echo(json.dumps({"Authorization": f"Bearer {creds.token}"})) + + +@app.command() +def logout() -> None: + """Forget the locally stored credential (the token itself stays valid — + revoke it with `lodedb cloud tokens revoke` to kill it server-side).""" + deleted = _config.delete_credentials() + _emit( + {"logged_out": deleted}, + lambda: typer.echo("Logged out." if deleted else "No stored credentials."), + ) + + +@app.command() +def whoami() -> None: + """Show who (a personal token: the account) or what (an environment + token: its kind, scopes, and binding) the stored credential is.""" + with _client() as client: + try: + info = client.token_self() + except CloudError as error: + # A control plane predating /v1/tokens/self answers 404; the + # personal-token rendering below still works there via /v1/auth/me. + if error.status_code != 404: + code, hint = _classify(error) + raise _fail(str(error), code=code, hint=hint) from error + info = None + me = _cloud(client.me) if info is None or info["kind"] == "personal" else None + + if me is not None: + data = { + "kind": info["kind"] if info else "personal", + "email": me["user"]["email"], + "auth": me["auth"], + "orgs": [{"slug": org["slug"], "role": org["role"]} for org in me["orgs"]], + } + + def human() -> None: + typer.echo(f"{me['user']['email']} ({me['auth']})") + for org in me["orgs"]: + typer.echo(f" org {org['slug']} ({org['role']})") + else: + data = { + "kind": info["kind"], + "org": info["org"], + "environment": info["environment"], + "scopes": info["scopes"], + "name": info["name"], + } + + def human() -> None: + typer.echo( + f"{info['kind']} token {info['prefix']}… bound to " + f"{info['org']}/{info['environment']} (scopes: {', '.join(info['scopes'])})" + ) + + _emit(data, human) + + +@tokens_app.command("list") +def tokens_list() -> None: + """List your tokens (personal, plus your orgs' environment tokens).""" + with _client() as client: + rows = _cloud(client.list_tokens) + + def human() -> None: + if not rows: + typer.echo("No tokens.") + return + for row in rows: + state = "revoked" if row["revoked_at"] else "active" + typer.echo( + f"{row['id']} {row['prefix']}… {row['kind']:<11} {state:<7} " + f"scopes={','.join(row['scopes'])} {row['name']}" + ) + + _emit(rows, human) + + +def _write_credential_env(env_file: str | Path, host: str, secret: str) -> dict[str, object]: + """Write ORECLOUD_HOST + ORECLOUD_TOKEN into the dotenv file (the secret + never touches stdout — printed output becomes part of an agent's + transcript) and make sure the file is git-ignored. Returns the env_file + path and any gitignore note, for merging into the command's output.""" + from lodedb.cloud import _env_file as env + + path = env.write_env_values(env_file, {"ORECLOUD_HOST": host, "ORECLOUD_TOKEN": secret}) + data: dict[str, object] = {"env_file": str(path)} + note = env.ensure_gitignored(path) + if note: + data["env_note"] = note + _note(note) + return data + + +@tokens_app.command("mint") +def tokens_mint( + kind: Annotated[str, typer.Option(help="personal | secret | publishable.")] = "personal", + scope: Annotated[ + list[str], typer.Option("--scope", help="Repeatable: admin, write, read:search, read:text.") + ] = ["admin"], # noqa: B006 — typer requires a literal default + name: Annotated[str, typer.Option(help="Label shown in listings.")] = "", + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + expires_in_days: Annotated[int | None, typer.Option(min=1)] = None, + env_file: Annotated[ + str | None, + typer.Option( + "--env-file", + help="Write the secret into this dotenv file (ORECLOUD_HOST + " + "ORECLOUD_TOKEN) instead of printing it — output then carries " + "only the token metadata.", + ), + ] = None, +) -> None: + """Mint a token. The secret is printed once and never retrievable again — + or, with --env-file, written straight to the file and never printed. + Environment tokens (secret/publishable) default to the resolved + org/environment; personal tokens take neither flag.""" + creds = _load_credentials() + if creds is None: + raise _fail("not logged in — run `lodedb cloud login --host `") + with CloudClient(creds.host, creds.token) as client: + if kind != "personal" and (org is None or environment is None): + org, environment = _tenancy(client, org, environment) + minted = _cloud( + lambda: client.mint_token( + kind, + scope, + name=name, + org=org, + environment=environment, + expires_in_days=expires_in_days, + ) + ) + if env_file is not None: + data: dict[str, object] = {"token": minted["token"]} + data.update(_run(lambda: _write_credential_env(env_file, creds.host, minted["secret"]))) + _emit(data, lambda: typer.echo(f"wrote token to {data['env_file']}")) + else: + _emit(minted, lambda: typer.echo(minted["secret"])) + typer.secho( + f"(token {minted['token']['id']}, prefix {minted['token']['prefix']}… — " + f"{'in ' + str(env_file) if env_file else 'store it now'}; " + "it cannot be shown again)", + err=True, + ) + + +@tokens_app.command("revoke") +def tokens_revoke(token_id: str) -> None: + """Revoke a token by id (see `lodedb cloud tokens list`).""" + with _client() as client: + row = _cloud(lambda: client.revoke_token(token_id)) + _emit(row, lambda: typer.echo(f"revoked {row['prefix']}… at {row['revoked_at']}")) + + +@mcp_app.command("install") +def mcp_install( + store: Annotated[str, typer.Argument(help="Store name — the end user's id.")], + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + token: Annotated[ + str | None, + typer.Option( + help="Embed this token in the printed configs (otherwise a " + "placeholder is printed — mint one with `lodedb cloud tokens mint`)." + ), + ] = None, +) -> None: + """Print ready-to-paste MCP configs (Claude Code, Cursor, VS Code) for + one end user's store. The endpoint speaks Streamable HTTP with Bearer + auth; the token's scopes decide which tools the agent sees. The store + needs no create step — it provisions on the agent's first write.""" + import json as json_module + from urllib.parse import quote + + with _client() as client: + org, environment = _tenancy(client, org, environment) + creds = _load_credentials() + assert creds is not None # _client() above required it + # One URL per end user's store — always the 3-segment form. Slugs are + # [a-z0-9-] but store names (end-user ids) are free-form: quote them. + url = ( + f"{creds.host}/mcp/{quote(org, safe='')}/" + f"{quote(environment, safe='')}/{quote(store, safe='')}" + ) + bearer = token or "" + + claude_command = ( + f'claude mcp add --transport http orecloud "{url}" ' + f'--header "Authorization: Bearer {bearer}"' + ) + cursor_config = { + "mcpServers": {"orecloud": {"url": url, "headers": {"Authorization": f"Bearer {bearer}"}}} + } + vscode_config = { + "servers": { + "orecloud": { + "type": "http", + "url": url, + "headers": {"Authorization": "Bearer ${input:orecloud-token}"}, + } + }, + "inputs": [ + { + "id": "orecloud-token", + "type": "promptString", + "password": True, + "description": "OreCloud API token", + } + ], + } + + def human() -> None: + typer.secho("MCP endpoint:", bold=True) + typer.echo(f" {url}\n") + typer.secho("Claude Code:", bold=True) + typer.echo(f" {claude_command}\n") + typer.secho("Cursor (~/.cursor/mcp.json):", bold=True) + typer.echo(json_module.dumps(cursor_config, indent=2) + "\n") + typer.secho("VS Code (.vscode/mcp.json):", bold=True) + typer.echo(json_module.dumps(vscode_config, indent=2)) + + _emit( + { + "url": url, + "claude_code": claude_command, + "cursor": cursor_config, + "vscode": vscode_config, + }, + human, + ) + if token is None: + typer.secho( + "\n(mint a key first, e.g. `lodedb cloud tokens mint --kind secret " + f"--scope read:search --scope read:text --org {org} --environment {environment}`)", + err=True, + ) + + +@environments_app.command("list") +def environments_list( + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """List environments in an org.""" + with _client() as client: + org = _org_scope(client, org) + rows = _cloud(lambda: client.list_environments(org)) + + def human() -> None: + for row in rows: + typer.echo(f"{org}/{row['slug']} {row['name']}") + + _emit(rows, human) + + +@store_app.command("list") +def store_list( + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + after: Annotated[ + str | None, typer.Option(help="Keyset cursor: the last store name of the previous page.") + ] = None, + limit: Annotated[int, typer.Option(help="Page size (1-200).")] = 50, +) -> None: + """One page of an environment's stores (a store is one end user).""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + out = _cloud(lambda: client.list_stores(org, environment, after=after, limit=limit)) + + def human() -> None: + rows = out["stores"] + if not rows: + typer.echo("No stores registered.") + return + for row in rows: + text_flag = "text" if row["expose_text"] else "no-text" + typer.echo( + f"{row['store']}/{row['key']} mode={row['mode']} {text_flag} " + f"last write {row['last_write_at']}" + ) + typer.echo(f"({out['count']} stores in the environment)") + + _emit(out, human) + + +@store_app.command("create") +def store_create( + store: Annotated[str, typer.Argument(help="Store name.")] = "default", + key: Annotated[ + str | None, + typer.Option("--key", help="Index key; omit for the LodeDB default (the usual case)."), + ] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + mode: Annotated[ + str, + typer.Option( + help="cloud_writer (write via the API — the quickstart path) | " + "local_push (a LodeDB directory you push)." + ), + ] = "cloud_writer", + preset: Annotated[ + str | None, + typer.Option( + help="Embedding preset (minilm | bge | clip); cloud_writer " + "defaults to minilm unless --vector-dim is given." + ), + ] = None, + vector_dim: Annotated[ + int | None, + typer.Option( + "--vector-dim", + min=1, + max=4096, + help="Bring-your-own-vectors store at this dimensionality: you " + "supply vectors (add_vectors/search_by_vector); the server never " + "embeds. Mutually exclusive with --preset.", + ), + ] = None, + expose_text: Annotated[ + bool, typer.Option("--expose-text", help="Allow read:text-scoped reads to return text.") + ] = False, + connect_key: Annotated[ + bool, + typer.Option( + "--connect-key/--no-connect-key", + help="Mint a secret key and print a ready-to-run Python snippet.", + ), + ] = True, +) -> None: + """Register a store and print everything Python needs to use it. + + The default is the API-first path: a cloud_writer store with the minilm + preset plus a freshly minted secret key, so the printed snippet runs + with zero edits. Pushing an existing LodeDB directory instead? Use + `--mode local_push` (identity then comes from the pushed store). + """ + if preset is not None and vector_dim is not None: + raise _fail( + "pass --preset or --vector-dim, not both (a store embeds " + "server-side OR takes your vectors)", + code=EXIT_USAGE, + ) + if mode == "cloud_writer" and preset is None and vector_dim is None: + preset = "minilm" + minted = None + with _client() as client: + org, environment = _tenancy(client, org, environment) + row = _cloud( + lambda: client.create_store( + org, + environment, + store, + key, + mode=mode, + expose_text=expose_text, + preset=preset, + vector_dim=vector_dim, + ) + ) + if connect_key and mode == "cloud_writer": + scopes = ["write", "read:search"] + (["read:text"] if expose_text else []) + minted = _cloud( + lambda: client.mint_token( + "secret", + scopes, + name=f"cli connect ({environment})", + org=org, + environment=environment, + ) + ) + creds = _load_credentials() + host = creds.host if creds else "https://" + target = f"{org}/{environment}/{row['store']}" + + def human() -> None: + suffix = f", preset={preset}" if preset else "" + typer.echo( + f"registered {org}/{environment}/{row['store']} (key {row['key']}, " + f"mode={row['mode']}{suffix})" + ) + if minted is None: + return + # The key was minted bound to this environment, so the snippet never + # restates the tenancy — Client resolves it from the credential. + typer.echo() + typer.echo("Connect from Python (the key is shown once — this is your copy):") + typer.echo() + typer.echo(" from lodedb.cloud import Client") + typer.echo( + f' client = Client(token="{minted["secret"]}", host="{host}")' + ) + typer.echo(f' idx = client.store("{row["store"]}")') + typer.echo(' idx.add_many([{"text": "hello lodedb"}]); print(idx.search("hello", k=3))') + + _emit( + { + "store": row, + "target": target, + "host": host, + # The connect key, shown once, exactly like the human snippet. + "secret": minted["secret"] if minted else None, + "token": minted["token"] if minted else None, + }, + human, + ) + + +def _resolve_index_key( + client: CloudClient, org: str, environment: str, store: str, key: str | None +) -> str: + """The explicit key, or the store's only index key when it is unambiguous + (mirrors `store create`'s omit-the-key ergonomics). Exact-name lookup — + never a page walk (an environment holds one store per end user).""" + if key is not None: + return key + rows = _cloud(lambda: client.list_stores(org, environment, store=store))["stores"] + if len(rows) == 1: + return rows[0]["key"] + if not rows: + raise _fail(f"no such store {org}/{environment}/{store}") + keys = ", ".join(row["key"] for row in rows) + raise _fail(f"{org}/{environment}/{store} holds several index keys — pass --key ({keys})") + + +@store_app.command("delete") +def store_delete( + store: Annotated[str, typer.Argument(help="Store name.")], + key: Annotated[ + str | None, + typer.Option( + "--key", + help="Delete only this index key (advanced; without it the whole store goes).", + ), + ] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + yes: Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")] = False, +) -> None: + """Soft-delete a store (restorable for the grace period). With --key, + delete only that index key inside the store.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + if key is None: + if not yes: + _confirm(f"Delete store {org}/{environment}/{store}?") + out = _cloud(lambda: client.delete_store(org, environment, store)) + restore_hint = ( + f" lodedb cloud store restore {out['slug']} " + f"--environment {environment} --org {org}" + ) + else: + if not yes: + _confirm(f"Delete index key {key} in store {org}/{environment}/{store}?") + out = _cloud(lambda: client.delete_store_key(org, environment, store, key)) + restore_hint = ( + f" lodedb cloud store restore {store} --key {out['slug']} " + f"--environment {environment} --org {org}" + ) + _emit( + out, + lambda: typer.echo( + f"deleted (restorable until {out['purge_after']}):\n{restore_hint}" + ), + ) + + +@store_app.command("restore") +def store_restore( + parked: Annotated[ + str, + typer.Argument( + help="The parked store name `store delete` printed " + "(or, with --key, the live store name)." + ), + ], + key: Annotated[ + str | None, + typer.Option("--key", help="Restore only this parked index key inside store PARKED."), + ] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """Restore a soft-deleted store (or, with --key, one index key) inside + its grace period.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + if key is None: + out = _cloud(lambda: client.restore_store(org, environment, parked)) + _emit(out, lambda: typer.echo(f"restored {org}/{environment}/{out['slug']}")) + return + row = _cloud(lambda: client.restore_store_key(org, environment, parked, key)) + _emit( + row, lambda: typer.echo(f"restored {org}/{environment}/{row['store']} (key {row['key']})") + ) + + +@store_app.command("history") +def store_history( + store: Annotated[str, typer.Argument(help="Store name.")] = "default", + key: Annotated[ + str | None, + typer.Option("--key", help="Index key; omit when the store holds exactly one."), + ] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """The store's restore window: every snapshot still held, newest first.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + key = _resolve_index_key(client, org, environment, store, key) + rows = _cloud(lambda: client.store_history(org, environment, store, key)) + + def human() -> None: + if not rows: + typer.echo("No snapshots yet.") + return + for row in rows: + marker = " <- current" if row["current"] else "" + # Full ids: rollback takes the exact digest, so truncating here + # would print a command the user can't complete. + typer.echo( + f"{row['snapshot_id']} gen {row['generation']:<4} " + f"{row['created_at']}{marker}" + ) + typer.echo( + f"\nRoll back with: lodedb cloud store rollback {store}" + ) + + _emit(rows, human) + + +@store_app.command("rollback") +def store_rollback( + snapshot_id: Annotated[str, typer.Argument(help="Target snapshot id (from `store history`).")], + store: Annotated[str, typer.Argument(help="Store name.")] = "default", + key: Annotated[ + str | None, + typer.Option("--key", help="Index key; omit when the store holds exactly one."), + ] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + yes: Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")] = False, +) -> None: + """Move the store back to a retained snapshot. Reversible: the replaced + head stays in the restore window for the retention period.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + key = _resolve_index_key(client, org, environment, store, key) + if not yes: + _confirm(f"Roll {org}/{environment}/{store} back to {snapshot_id[:12]}…?") + out = _cloud( + lambda: client.rollback_store(org, environment, store, snapshot_id, key=key) + ) + previous = out["previous_snapshot_id"] + _emit( + out, + lambda: typer.echo( + f"rolled back to {out['snapshot_id'][:12]} (gen {out['generation']})" + + (f"; the replaced head {previous[:12]} stays restorable" if previous else "") + ), + ) + + +@environments_app.command("delete") +def environments_delete( + slug: str, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + yes: Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")] = False, +) -> None: + """Soft-delete an environment and everything in it (restorable for the grace period).""" + with _client() as client: + org = _org_scope(client, org) + if not yes: + _confirm(f"Delete environment {org}/{slug} and every store in it?") + out = _cloud(lambda: client.delete_environment(org, slug)) + _emit( + out, + lambda: typer.echo( + f"deleted (restorable until {out['purge_after']}):\n" + f" lodedb cloud environments restore {out['slug']} --org {org}" + ), + ) + + +@environments_app.command("restore") +def environments_restore( + parked_slug: Annotated[ + str, typer.Argument(help="The parked slug `environments delete` printed.") + ], + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """Restore a soft-deleted environment inside its grace period.""" + with _client() as client: + org = _org_scope(client, org) + out = _cloud(lambda: client.restore_environment(org, parked_slug)) + _emit(out, lambda: typer.echo(f"restored {org}/{out['slug']}")) + + +@org_app.command("delete") +def org_delete( + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + yes: Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")] = False, +) -> None: + """Soft-delete an org and everything in it (restorable for the grace period).""" + with _client() as client: + org = _org_scope(client, org) + if not yes: + _confirm(f"Delete org {org} with all environments, stores, and tokens?") + out = _cloud(lambda: client.delete_org(org)) + _emit( + out, + lambda: typer.echo( + f"deleted (restorable until {out['purge_after']}):\n" + f" lodedb cloud org restore {out['slug']}" + ), + ) + + +@org_app.command("restore") +def org_restore( + parked_slug: Annotated[str, typer.Argument(help="The parked slug `org delete` printed.")], +) -> None: + """Restore a soft-deleted org inside its grace period.""" + with _client() as client: + out = _cloud(lambda: client.restore_org(parked_slug)) + _emit(out, lambda: typer.echo(f"restored {out['slug']}")) + + +@org_app.command("trash") +def org_trash( + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """List soft-deleted resources still restorable under an org.""" + with _client() as client: + org = _org_scope(client, org) + items = _cloud(lambda: client.list_trash(org)) + + def human() -> None: + if not items: + typer.echo("Trash is empty.") + return + for item in items: + where = "/".join( + part for part in (item.get("environment"), item.get("store")) if part + ) + scope = f" in {where}" if where else "" + typer.echo( + f"{item['kind']} {item['original']}{scope} — restore with " + f"'{item['slug']}' until {item['purge_after']}" + ) + + _emit(items, human) + + +@org_app.command("export") +def org_export( + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, +) -> None: + """Print the org export manifest (JSON): every live environment and store + with its head snapshot. Pull each store's bytes with `lodedb cloud pull`.""" + import json as _json + + with _client() as client: + org = _org_scope(client, org) + manifest = _cloud(lambda: client.export_org(org)) + typer.echo(_json.dumps(manifest, indent=2)) + + +# ------------------------------------------------- memory verbs (Phase 8d) +# A store is one end user's LodeDB instance, so these take the store name +# (= the user's id) directly; `store list` is the user listing and +# `store delete` the forget-the-user verb. + +def _scope_fields(agent: str | None, run: str | None) -> dict: + """The optional narrowing axes as request fields.""" + fields: dict[str, str] = {} + if agent is not None: + fields["agent_id"] = agent + if run is not None: + fields["run_id"] = run + return fields + + +@store_app.command("browse") +def store_browse( + store: Annotated[str, typer.Argument(help="Store name (the end user's id).")], + key: Annotated[str | None, typer.Option(help="Index key.")] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + after: Annotated[ + str | None, + typer.Option(help="Keyset cursor: the last document id of the previous page."), + ] = None, + limit: Annotated[int, typer.Option(help="Page size (1-100).")] = 25, + include_text: Annotated[ + bool, + typer.Option("--include-text", help="Return stored text (needs read:text + expose_text)."), + ] = False, + agent: Annotated[str | None, typer.Option(help="Narrow to one agent's memories.")] = None, + run: Annotated[str | None, typer.Option(help="Narrow to one session's memories.")] = None, +) -> None: + """One page of a user's memories (ids + metadata, text on request).""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + payload = { + "store": store, + "key": key, + "after": after, + "limit": limit, + "include_text": include_text, + **_scope_fields(agent, run), + } + out = _cloud(lambda: client.browse_documents(org, environment, payload)) + + def human() -> None: + documents = out["documents"] + if not documents: + typer.echo("no memories") + return + for doc in documents: + line = doc["id"] + if include_text and doc.get("text"): + line += f" {' '.join(doc['text'].split())[:80]}" + typer.echo(line) + typer.echo(f"(page of {len(documents)}; next --after {documents[-1]['id']})") + + _emit(out, human) + + +@store_app.command("export") +def store_export( + store: Annotated[str, typer.Argument(help="Store name (the end user's id).")], + key: Annotated[str | None, typer.Option(help="Index key.")] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + include_text: Annotated[ + bool, + typer.Option( + "--include-text/--no-include-text", + help="Include stored text (needs read:text + expose_text).", + ), + ] = True, +) -> None: + """Export ALL of a user's memories as JSON (pages the browse endpoint + to exhaustion) — the per-end-user data-export verb.""" + import json as _json + + with _client() as client: + org, environment = _tenancy(client, org, environment) + documents: list[dict] = [] + after: str | None = None + while True: + payload = { + "store": store, + "key": key, + "after": after, + "limit": 100, + "include_text": include_text, + } + # bind=payload: the lambda runs inside this iteration, but binding + # explicitly keeps the closure honest (and ruff B023 quiet). + page = _cloud(lambda bind=payload: client.browse_documents(org, environment, bind)) + documents.extend(page["documents"]) + if len(page["documents"]) < 100: + break + after = page["documents"][-1]["id"] + typer.echo(_json.dumps({"store": store, "documents": documents}, indent=2)) + + +@store_app.command("recall") +def store_recall( + store: Annotated[str, typer.Argument(help="Store name (the end user's id).")], + text: Annotated[str, typer.Argument(help="Raw text — a whole user message, not a query.")], + key: Annotated[str | None, typer.Option(help="Index key.")] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + k: Annotated[int, typer.Option(help="How many memories to return.")] = 10, + include_text: Annotated[ + bool, typer.Option("--include-text", help="Return stored text inline.") + ] = False, + agent: Annotated[str | None, typer.Option(help="Narrow to one agent's memories.")] = None, + run: Annotated[str | None, typer.Option(help="Narrow to one session's memories.")] = None, +) -> None: + """Non-exact retrieval from raw text: the server derives sub-queries + (windows + key phrases) and fuses their rankings.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + payload = { + "store": store, + "key": key, + "text": text, + "k": k, + "include_text": include_text, + **_scope_fields(agent, run), + } + out = _cloud(lambda: client.recall(org, environment, payload)) + + def human() -> None: + if not out["hits"]: + typer.echo("no memories recalled") + return + for hit in out["hits"]: + line = f"{hit['score']:.4f} {hit['id']}" + if include_text and hit.get("text"): + line += f" {' '.join(hit['text'].split())[:80]}" + typer.echo(line) + typer.echo(f" matched: {', '.join(hit['matched'])}") + typer.echo(f"(sub-queries: {', '.join(out['queries'])})") + + _emit(out, human) + + +@store_app.command("delete-memories") +def store_delete_memories( + store: Annotated[str, typer.Argument(help="Store name (the end user's id).")], + key: Annotated[str | None, typer.Option(help="Index key.")] = None, + environment: Annotated[str | None, typer.Option(help=_ENVIRONMENT_HELP)] = None, + org: Annotated[str | None, typer.Option(help=_ORG_HELP)] = None, + agent: Annotated[ + str | None, typer.Option(help="Narrow the deletion to one agent's memories.") + ] = None, + run: Annotated[ + str | None, typer.Option(help="Narrow the deletion to one session's memories.") + ] = None, + yes: Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")] = False, +) -> None: + """Delete a user's memories in place (expired ones included), keeping + the store registered. Async like every cloud write. To forget the user + entirely — row, entitlement slot and all — use `store delete`.""" + with _client() as client: + org, environment = _tenancy(client, org, environment) + if not yes: + narrowed = "".join( + f" {label} {value}" for label, value in (("agent", agent), ("run", run)) if value + ) + _confirm( + f"Delete ALL memories{narrowed} in {org}/{environment}/{store}?" + ) + payload = {"store": store, "key": key, **_scope_fields(agent, run)} + out = _cloud(lambda: client.delete_memories(org, environment, payload)) + _emit( + out, + lambda: typer.echo( + f"accepted: {out['document_count']} memories queued for removal " + f"({len(out['write_ids'])} writes)" + ), + ) diff --git a/src/lodedb/cloud/client.py b/src/lodedb/cloud/client.py new file mode 100644 index 00000000..82f94282 --- /dev/null +++ b/src/lodedb/cloud/client.py @@ -0,0 +1,326 @@ +"""`lodedb.cloud.Client` — the developer-facing handle, bound to one tenancy. + + from lodedb.cloud import Client + + client = Client() # credentials + tenancy from the token + store = client.store("user-42") # one end user's LodeDB instance + store.add("prefers email over phone") + +The org and environment are properties of the *credential*, not of every +call site: `ore_sk_`/`ore_pk_` tokens are minted bound to one environment, +and the server rejects any other. So the client asks the control plane what +the token is (`GET /v1/tokens/self`) and binds itself accordingly — code +that connects with a production key and code that connects with a testing +key are the same code. Personal tokens (`ore_pat_`) span environments, so +they resolve to the account's only org and default to the ``testing`` +environment (production is the explicit go-live choice, never a guess); +passing ``org=``/``environment=`` always wins. Passing exactly one of the +pair is checked against a bound token rather than silently ignored; passing +both skips introspection entirely (zero HTTP calls) and relies on the +server rejecting a mismatched binding at the first request. + +Credentials resolve like the CLI's: explicit ``token=``/``host=`` arguments, +then the ``ORECLOUD_TOKEN``/``ORECLOUD_HOST`` environment pair, then the +``lodedb cloud login`` credentials file. + +Every control-plane verb the SDK speaks hangs off this object with the +tenancy already applied; per-store reads and writes live on the +:class:`~lodedb.cloud.serving.CloudStore` handles that :meth:`Client.store` +returns. The legacy ``lodedb.cloud.connect("org/environment/store")`` remains +as sugar over this class. +""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from lodedb.cloud import _config +from lodedb.cloud.transfer import CloudClient, CloudError, ManagedRemote + + +def resolve_credentials(token: str | None, host: str | None) -> tuple[str, str]: + """(host, token) from explicit arguments, the environment pair, or the + `lodedb cloud login` file — raising a CloudError naming the fix when either + half is missing.""" + if token is None or host is None: + stored = _config.load_credentials() + if token is None: + token = stored.token if stored else None + if host is None: + host = stored.host if stored else None + if not host: + raise CloudError( + 401, + "no control-plane host configured — pass host=, set ORECLOUD_HOST, " + "or run `lodedb cloud login`", + ) + if not token: + raise CloudError( + 401, + "no credential configured — pass token=, set ORECLOUD_TOKEN, or run " + "`lodedb cloud login`", + ) + return host, token + + +def resolve_tenancy( + client: CloudClient, org: str | None, environment: str | None +) -> tuple[str, str]: + """The (org, environment) this credential addresses. + + Explicit values win. An environment-scoped token supplies its own + binding and *refuses* a conflicting explicit value — silently ignoring + one would aim writes somewhere the caller didn't name. A personal token + falls back to the account's only org, then to the org's only live + environment or the seeded ``testing`` default; anything else raises + with the actual choices. + """ + if org and environment: + return org, environment + try: + info = client.token_self() + except CloudError as error: + if error.status_code == 404: + # A control plane too old to introspect tokens: don't guess what + # the credential is — name both escape hatches. + raise CloudError( + 404, + "this control plane does not support token introspection — " + "pass org= and environment= explicitly, or upgrade the server", + ) from error + raise + if info["environment"]: + bound_org, bound_environment = info["org"], info["environment"] + if org and org != bound_org: + raise CloudError( + 403, f"this token is bound to org {bound_org!r}, not {org!r}" + ) + if environment and environment != bound_environment: + raise CloudError( + 403, + f"this token is bound to environment {bound_environment!r}, " + f"not {environment!r}", + ) + return bound_org, bound_environment + if org is None: + slugs = [row["slug"] for row in client.me()["orgs"]] + if not slugs: + raise CloudError(404, "this account belongs to no org") + if len(slugs) > 1: + raise CloudError( + 422, + "this account belongs to several orgs — pass org= " + f"(one of: {', '.join(slugs)})", + ) + org = slugs[0] + if environment is None: + slugs = [row["slug"] for row in client.list_environments(org)] + if len(slugs) == 1: + environment = slugs[0] + elif "testing" in slugs: + # The fixed pair seeded at signup: testing is the default; + # production is the explicit go-live choice, never a guess. + environment = "testing" + else: + raise CloudError( + 422, + f"org {org!r} has {len(slugs)} environments — pass environment= " + + (f"(one of: {', '.join(slugs)})" if slugs else "(none live)"), + ) + return org, environment + + +class Client: + """One credential, one control plane, one bound (org, environment). + + Construction resolves credentials and tenancy (see the module + docstring) and performs at most two HTTP calls — zero when both + ``org=`` and ``environment=`` are passed. The client owns one HTTP + connection pool that every handle it creates shares; close it with + :meth:`close` or a ``with`` block when done. + """ + + def __init__( + self, + *, + token: str | None = None, + host: str | None = None, + org: str | None = None, + environment: str | None = None, + timeout: float = 30.0, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.host, _token = resolve_credentials(token, host) + self._client = CloudClient(self.host, _token, transport=transport, timeout=timeout) + try: + self.org, self.environment = resolve_tenancy(self._client, org, environment) + except BaseException: + # A half-constructed client must not leak its connection pool. + self._client.close() + raise + + # ------------------------------------------------------------- stores + + def store( + self, + store: str, + *, + key: str | None = None, + warm: bool = False, + read_your_writes: bool = True, + ): + """A read/write handle over one store — one end user's LodeDB + instance, auto-provisioned by its first write, so this makes no + HTTP call by default. `warm=True` additionally asks the serving + tier to hydrate it now (first query skips the cold start). `key` + names the index key when the store holds more than one (rare). + The handle shares this client's connection pool; closing the + handle does not close the client.""" + from lodedb.cloud.serving import CloudStore + + handle = CloudStore( + self._client, + self.org, + self.environment, + store, + key, + read_your_writes=read_your_writes, + owns_client=False, + ) + if warm: + handle.stats(warm=True) + return handle + + def list_stores(self, **params: Any) -> dict: + """One keyset page of this environment's stores (a store is one end + user): {stores, count}. Paging/filter params are `CloudClient + .list_stores`'s.""" + return self._client.list_stores(self.org, self.environment, **params) + + def store_stats(self) -> dict: + """Fleet-level counts over this environment's live stores: + {stores, active_24h, active_7d, new_7d}.""" + return self._client.store_stats(self.org, self.environment) + + def create_store(self, store: str, key: str | None = None, **options: Any) -> dict: + """Register a store explicitly (first writes auto-provision, so this + is for choosing `mode=`/`preset=`/`expose_text=` up front — or + `vector_dim=` for a bring-your-own-vectors store, which accepts + `add_vectors`/`search_by_vector` and never embeds server-side).""" + return self._client.create_store(self.org, self.environment, store, key, **options) + + def update_store(self, store: str, key: str, **changes: Any) -> dict: + """Flip a store's `expose_text`/`mode` flags.""" + return self._client.update_store(self.org, self.environment, store, key, **changes) + + def delete_store(self, store: str) -> dict: + """Soft-delete a whole store — forget this end user (restorable for + the grace period; the entitlement slot frees immediately).""" + return self._client.delete_store(self.org, self.environment, store) + + def restore_store(self, parked_store: str) -> dict: + return self._client.restore_store(self.org, self.environment, parked_store) + + def delete_store_key(self, store: str, key: str) -> dict: + """Soft-delete ONE index key inside a store (the advanced multi-key + path).""" + return self._client.delete_store_key(self.org, self.environment, store, key) + + def restore_store_key(self, store: str, parked_key: str) -> dict: + return self._client.restore_store_key(self.org, self.environment, store, parked_key) + + def store_history(self, store: str, key: str) -> list[dict]: + """The store's restore window: every retained snapshot, newest + first.""" + return self._client.store_history(self.org, self.environment, store, key) + + def rollback_store(self, store: str, snapshot_id: str, key: str | None = None) -> dict: + """Move the store's head back to a retained snapshot (reversible + within the retention window).""" + return self._client.rollback_store(self.org, self.environment, store, snapshot_id, key=key) + + # ------------------------------------------------------- environments + + def list_environments(self) -> list[dict]: + """The bound org's environments (an environment token sees only its + own). The pair is fixed — production and testing, seeded at signup; + there is no creation surface.""" + return self._client.list_environments(self.org) + + def delete_environment(self, slug: str | None = None) -> dict: + """Soft-delete an environment (the bound one when `slug` is + omitted) and everything in it.""" + return self._client.delete_environment(self.org, slug or self.environment) + + def restore_environment(self, parked_slug: str) -> dict: + return self._client.restore_environment(self.org, parked_slug) + + # ---------------------------------------------------------------- org + + def list_trash(self) -> list[dict]: + """Soft-deleted resources under the org, restorable until their + `purge_after`.""" + return self._client.list_trash(self.org) + + def export_org(self) -> dict: + """The offboarding manifest: every live environment and store with + its head snapshot identity (metadata only).""" + return self._client.export_org(self.org) + + def delete_org(self) -> dict: + """Soft-delete the whole org. Returns the parked slug restore takes + and the purge deadline.""" + return self._client.delete_org(self.org) + + def restore_org(self, parked_slug: str) -> dict: + return self._client.restore_org(parked_slug) + + # ------------------------------------------------------------- tokens + + def me(self) -> dict: + """The signed-in account (personal tokens only — an environment + token is not a person and gets a 403).""" + return self._client.me() + + def token_self(self) -> dict: + """What the presented credential is: kind, scopes, and its + org/environment binding (None/None for personal tokens).""" + return self._client.token_self() + + def list_tokens(self) -> list[dict]: + return self._client.list_tokens() + + def mint_token(self, kind: str, scopes: list[str], **options: Any) -> dict: + """Mint a token; environment tokens (`secret`/`publishable`) + default to this client's bound org/environment unless overridden.""" + if kind != "personal": + options.setdefault("org", self.org) + options.setdefault("environment", self.environment) + return self._client.mint_token(kind, scopes, **options) + + def revoke_token(self, token_id: str) -> dict: + return self._client.revoke_token(token_id) + + # ------------------------------------------------------------ transfer + + def remote(self, store: str = "memory") -> ManagedRemote: + """This tenancy as a `ManagedRemote`, for the module-level transfer + verbs (`managed_push`/`managed_pull`/`managed_sync`) that move a + local LodeDB directory to and from the cloud.""" + return ManagedRemote(self.org, self.environment, store) + + # ----------------------------------------------------------- lifecycle + + def close(self) -> None: + self._client.close() + + def __enter__(self) -> Client: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def __repr__(self) -> str: + return f"Client({self.org!r}, {self.environment!r}, host={self.host!r})" diff --git a/src/lodedb/cloud/serving.py b/src/lodedb/cloud/serving.py new file mode 100644 index 00000000..08b8692f --- /dev/null +++ b/src/lodedb/cloud/serving.py @@ -0,0 +1,773 @@ +"""`CloudStore` — the cloud handle that duck-types a local LodeDB. + + from lodedb.cloud import Client + + client = Client() # tenancy from the credential + memories = client.store("user-42") + memories.add("prefers email over phone") + hits = memories.recall("how should I contact them about the invoice?") + +A store is one end user's own LodeDB instance (the user's id in the +agentic-memory product), auto-provisioned by its first write — open a user +that doesn't exist yet and reads answer empty until the first `add`. +Isolation is physical: this handle cannot reach any other user's instance. + +`lodedb.cloud.connect("org/environment/store")` remains as path-string sugar +over the same handle for one-off scripts and console copy-paste. + +The returned :class:`CloudStore` implements the read subset of the local +`lodedb.LodeDB` handle — `search` / `search_many` / `get` / `get_texts` / +`stats` / `count`, with hits shaped exactly like the local `LodeSearchHit` +(`hit.score` / `hit.id` / `hit.metadata`, and tuple unpacking) — so RAG +adapters and MCP tool bodies written against a local handle work unmodified +against the cloud. On top of that come the memory verbs: `add` (with TTL), +`recall`, `context_block`, `browse`, and `delete_memories`. + +Credentials resolve like the CLI's: explicit ``token=``/``host=`` arguments +win, then the ``ORECLOUD_TOKEN``/``ORECLOUD_HOST`` environment pair, then the +``lodedb cloud login`` credentials file. Server-side, queries are embedded with +the same preset that indexed the data, so scores are the engine's own. + +Connecting pre-warms the store on the serving tier by default (the first +query then skips the hydration cold start); pass ``warm=False`` to skip. +""" + +from __future__ import annotations + +import time +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +import httpx + +from lodedb.cloud.transfer import CloudClient, CloudError, ManagedRemote + + +class CloudSearchHit: + """One scored hit — attribute access and tuple unpacking, mirroring the + local `LodeSearchHit`. `text` is set when the search asked for it; + `matched` (recall only) names the sub-queries that surfaced the hit.""" + + __slots__ = ("score", "id", "metadata", "text", "matched") + + def __init__( + self, + *, + score: float, + id: str, + metadata: dict[str, Any], + text: str | None = None, + ) -> None: + self.score = float(score) + self.id = str(id) + self.metadata = dict(metadata) + self.text = text + self.matched: list[str] = [] + + def __iter__(self): + yield self.score + yield self.id + yield self.metadata + + def __repr__(self) -> str: + return f"CloudSearchHit(score={self.score:.4f}, id={self.id!r}, metadata={self.metadata!r})" + + def __eq__(self, other: object) -> bool: + if isinstance(other, CloudSearchHit): + return (self.score, self.id, self.metadata) == ( + other.score, + other.id, + other.metadata, + ) + if isinstance(other, tuple): + return tuple(self) == other + return NotImplemented + + +def _hit(row: dict) -> CloudSearchHit: + return CloudSearchHit( + score=row["score"], id=row["id"], metadata=row["metadata"], text=row.get("text") + ) + + +class CloudStore: + """A read handle over one managed store, duck-typing the local LodeDB + read surface. Create via :func:`connect`.""" + + def __init__( + self, + client: CloudClient, + org: str, + environment: str, + store: str, + key: str | None = None, + *, + read_your_writes: bool = True, + write_visibility_timeout: float = 30.0, + owns_client: bool = True, + ) -> None: + # Handles from `Client.store()` share the Client's connection pool + # (owns_client=False), so closing one user's handle must not sever + # every other user's; a standalone `connect()` handle owns its pool. + self._client = client + self._owns_client = bool(owns_client) + self.org = org + self.environment = environment + self.store = store + self.key = key + # Session read-your-writes: the highest `seq` a write on THIS handle + # acked with. Searches pass it as `min_seq` and briefly retry on 425 + # until the fold covers it (opt out with read_your_writes=False). + self._read_your_writes = bool(read_your_writes) + self._write_visibility_timeout = float(write_visibility_timeout) + self._last_seq = 0 + # The most recent accepted write's id — the `wait_for` handle. + self.last_write_id: str | None = None + + # ------------------------------------------------------------- queries + + def _empty_if_unprovisioned(self, call, empty): + """Runs one read, answering `empty` when this user's store simply + doesn't exist yet — a store is one end user and materializes on its + first write, so reading a fresh user before their first memory is + the normal zero-setup flow (the hosted MCP tools behave the same). + Every other error stays loud.""" + try: + return call() + except CloudError as error: + if error.status_code == 404 and "no such store" in error.detail: + return empty + raise + + def _searched(self, call, payload: dict[str, Any]) -> dict: + """Runs one search call with session read-your-writes: `min_seq` is + this handle's last acked write, and a 425 (fold not caught up yet) is + retried briefly instead of surfacing — the write is durable, only its + visibility is trailing by a fold cycle.""" + if self._read_your_writes and self._last_seq > 0: + payload["min_seq"] = self._last_seq + deadline = time.monotonic() + self._write_visibility_timeout + while True: + try: + return call(self.org, self.environment, payload) + except CloudError as error: + if error.status_code != 425 or time.monotonic() >= deadline: + raise + time.sleep(0.25) + + def search( + self, + query: str, + *, + k: int = 10, + filter: dict[str, Any] | None = None, + mode: str | None = None, + include_text: bool = False, + ) -> list[CloudSearchHit]: + """Top-`k` hits, engine-scored. `include_text=True` returns each + hit's stored text inline (requires a `read:text`-scoped key and the + store's `expose_text` flag). After a write on this handle, the search + waits (briefly) for the write to become visible — session + read-your-writes; disable with `connect(..., read_your_writes=False)`.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "query": query, + "k": k, + "filter": filter, + "mode": mode, + "include_text": include_text, + } + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.search, payload), {"hits": []} + ) + return [_hit(row) for row in result["hits"]] + + def search_many( + self, + queries: list[str], + *, + k: int = 10, + filter: dict[str, Any] | None = None, + mode: str | None = None, + include_text: bool = False, + ) -> list[list[CloudSearchHit]]: + """Top-`k` hits per query, order-preserving — the batched search.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "queries": queries, + "k": k, + "filter": filter, + "mode": mode, + "include_text": include_text, + } + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.search_many, payload), + {"results": [[] for _ in queries]}, + ) + return [[_hit(row) for row in rows] for rows in result["results"]] + + # -------------------------------------------------------------- writes + + # Transport-level retries per write. Safe only because every write + # carries an idempotency key: the server either registers the request + # once or replays the original acceptance. + _WRITE_TRANSPORT_RETRIES = 2 + + def _accepted(self, result: dict) -> dict: + """Records an accepted write's `seq` as this session's + read-your-writes floor and remembers its `write_id`.""" + seq = int(result.get("seq", 0) or 0) + if seq > self._last_seq: + self._last_seq = seq + self.last_write_id = str(result["write_id"]) + return result + + def _written(self, call, payload: dict[str, Any]) -> dict: + """One accepted write under an auto-generated idempotency key. + + The failure this exists for: the server accepts the write but the + response is lost (timeout, dropped connection). A naive resend would + register a second segment — duplicate documents under fresh ids. The + key pins the request, so the retry (same key, byte-identical body) + gets the original acceptance replayed instead. Only transport-level + failures are retried; an HTTP error is a real answer. + """ + payload["idempotency_key"] = uuid.uuid4().hex + attempt = 0 + while True: + try: + return self._accepted(call(self.org, self.environment, payload)) + except httpx.TransportError: + attempt += 1 + if attempt > self._WRITE_TRANSPORT_RETRIES: + raise + time.sleep(0.25 * attempt) + + def add( + self, + text: str, + *, + id: str | None = None, + metadata: dict[str, Any] | None = None, + ttl_seconds: int | None = None, + agent_id: str | None = None, + run_id: str | None = None, + ) -> str: + """Add (or replace) one document — the cloud `db.add`. The text is + embedded server-side and the write is ACCEPTED (durable + ordered) + when this returns; visibility follows within seconds, and a search on + this handle waits for it (session read-your-writes). The first write + to a store that doesn't exist yet provisions it (a store is one end + user). `ttl_seconds` hides the memory from reads once it lapses + (hide-not-delete); `agent_id`/`run_id` stamp provenance that reads + can narrow on. Requires a `write`-scoped key. Returns the document + id.""" + (doc_id,) = self.add_many( + [{"text": text, "id": id, "metadata": metadata}], + ttl_seconds=ttl_seconds, + agent_id=agent_id, + run_id=run_id, + ) + return doc_id + + def add_many( + self, + documents: list[dict[str, Any]], + *, + ttl_seconds: int | None = None, + agent_id: str | None = None, + run_id: str | None = None, + ) -> list[str]: + """Add a batch of ``{"text", "id"?, "metadata"?}`` documents as one + accepted write (one segment; the fold batches concurrent writes into + one commit). Returns the ids, in order — assigned at acceptance.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "documents": [ + { + "text": doc["text"], + "id": doc.get("id"), + "metadata": doc.get("metadata"), + } + for doc in documents + ], + } + if ttl_seconds is not None: + payload["ttl_seconds"] = ttl_seconds + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._written(self._client.add_documents, payload) + return list(result["ids"]) + + def add_vectors( + self, + vector: Sequence[float], + *, + id: str | None = None, + text: str | None = None, + metadata: dict[str, Any] | None = None, + ttl_seconds: int | None = None, + agent_id: str | None = None, + run_id: str | None = None, + ) -> str: + """Add (or replace) one pre-embedded document — the cloud + `db.add_vectors`, for stores created with `vector_dim` (the server + never embeds; the vector must be exactly the store's dims and is + unit-normalized server-side, the local default). `text` is optional + retained payload for `get()`. Returns the document id.""" + (doc_id,) = self.add_vectors_many( + [{"vector": list(vector), "id": id, "text": text, "metadata": metadata}], + ttl_seconds=ttl_seconds, + agent_id=agent_id, + run_id=run_id, + ) + return doc_id + + def add_vectors_many( + self, + documents: list[dict[str, Any]], + *, + ttl_seconds: int | None = None, + agent_id: str | None = None, + run_id: str | None = None, + ) -> list[str]: + """Add a batch of ``{"vector", "id"?, "text"?, "metadata"?}`` + documents as one accepted write. Vector-store counterpart of + `add_many`; returns the ids in order.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "documents": [ + { + "vector": list(doc["vector"]), + "text": doc.get("text"), + "id": doc.get("id"), + "metadata": doc.get("metadata"), + } + for doc in documents + ], + } + if ttl_seconds is not None: + payload["ttl_seconds"] = ttl_seconds + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._written(self._client.add_documents, payload) + return list(result["ids"]) + + def search_by_vector( + self, + vector: Sequence[float], + *, + k: int = 10, + filter: dict[str, Any] | None = None, + include_text: bool = False, + ) -> list[CloudSearchHit]: + """Top-`k` hits for a pre-embedded query — the cloud + `db.search_by_vector`, for vector stores (which have no server-side + embedder; the vector must be exactly the store's dims).""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "query_vector": list(vector), + "k": k, + "filter": filter, + "include_text": include_text, + } + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.search, payload), {"hits": []} + ) + return [_hit(row) for row in result["hits"]] + + def search_many_by_vector( + self, + vectors: list[Sequence[float]], + *, + k: int = 10, + filter: dict[str, Any] | None = None, + include_text: bool = False, + ) -> list[list[CloudSearchHit]]: + """One engine batch of pre-embedded queries — the cloud + `db.search_many_by_vector`.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "query_vectors": [list(vector) for vector in vectors], + "k": k, + "filter": filter, + "include_text": include_text, + } + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.search_many, payload), + # One empty hit list PER query (mirroring search_many): callers + # zip queries to results, so the unprovisioned answer must keep + # the cardinality. + {"results": [[] for _ in vectors]}, + ) + return [[_hit(row) for row in hits] for hits in result["results"]] + + def remove(self, id: str) -> str: + """Remove one document by id — the cloud `db.remove`, async-first: + returns the accepted write's id once the removal is durably queued. + Whether the document existed is decided when the fold applies the + delete — ``wait_for(write_id)["result"]["removed"][0]`` answers it.""" + return self.remove_many([id]) + + def remove_many(self, ids: Sequence[str]) -> str: + """Remove a batch of documents by id as one accepted write — the + cloud `db.remove_many`, async-first like :meth:`remove`: returns the + write's id once the removals are durably queued (one segment, one + fold). Per-id outcomes are decided when the fold applies the deletes + — ``wait_for(write_id)["result"]["removed"]`` is the parallel bool + list. An empty batch raises: with no accepted write there is no id to + return (the local handle's ``remove_many([])`` no-op returns 0 + instead).""" + document_ids = list(ids) + if not document_ids: + raise ValueError("ids is empty — nothing to remove") + payload: dict[str, Any] = {"store": self.store, "key": self.key, "ids": document_ids} + result = self._written(self._client.remove_documents, payload) + return str(result["write_id"]) + + def wait_for(self, write_id: str, *, timeout: float = 30.0) -> dict[str, Any]: + """Blocks until an accepted write folds; returns its final status + (`state`, covering `snapshot_id`/`generation`, `result`). Raises + :class:`CloudError` (502) when the write was condemned, and + :class:`TimeoutError` when `timeout` elapses first (the write stays + queued and will still fold).""" + deadline = time.monotonic() + float(timeout) + while True: + status = self._client.write_status( + self.org, self.environment, self.store, write_id, key=self.key + ) + if status["state"] == "folded": + return status + if status["state"] == "condemned": + raise CloudError( + 502, + "this write could not be applied (segment condemned: " + f"{status.get('error')}) — resubmit the documents", + ) + if time.monotonic() >= deadline: + raise TimeoutError( + f"write {write_id} did not fold within {timeout}s (it stays " + "queued and will still be applied)" + ) + time.sleep(0.25) + + # ---------------------------------------------------------------- text + + def get(self, id: str) -> str | None: + """One document's stored raw text by id (None when absent) — the + cloud `db.get(id)`. Requires `read:text` and the store's + `expose_text` flag.""" + result = self._empty_if_unprovisioned( + lambda: self._client.store_text( + self.org, self.environment, self.store, id, key=self.key + ), + {"found": False, "text": None}, + ) + return result["text"] if result["found"] else None + + # The local handle's alias. + get_text = get + + def get_texts(self, ids: list[str]) -> dict[str, str]: + """Stored text for several ids (missing ids are omitted). One request + per id today — prefer `search(..., include_text=True)` for RAG.""" + texts: dict[str, str] = {} + for id in ids: + text = self.get(id) + if text is not None: + texts[id] = text + return texts + + # --------------------------------------------------------------- stats + + def stats(self, *, warm: bool = False) -> dict[str, Any]: + """Metrics-only serving stats (counts, snapshot identity, payload + flags) — the cloud `db.stats()` subset.""" + return self._client.serving_stats( + self.org, self.environment, self.store, self.key, warm=warm + ) + + def count(self) -> int: + stats = self._empty_if_unprovisioned(self.stats, {}) + return int(stats.get("document_count", 0) or 0) + + # -------------------------------------------------------- memory verbs + + def recall( + self, + text: str, + *, + k: int = 10, + filter: dict[str, Any] | None = None, + include_text: bool = False, + agent_id: str | None = None, + run_id: str | None = None, + ) -> list[CloudSearchHit]: + """Non-exact retrieval from RAW text — pass a whole user message; + the server derives sub-queries and fuses the rankings. Each hit's + `matched` attribute (set on the returned objects) names the + sub-queries that surfaced it. `agent_id`/`run_id` narrow to one + agent's or one session's memories.""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "text": text, + "k": k, + "filter": filter, + "include_text": include_text, + } + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.recall, payload), {"hits": []} + ) + hits = [] + for row in result["hits"]: + hit = _hit(row) + hit.matched = row.get("matched", []) # provenance, recall-only + hits.append(hit) + return hits + + def context_block( + self, + text: str | None = None, + *, + max_chars: int = 4_000, + agent_id: str | None = None, + run_id: str | None = None, + ) -> str: + """A prompt-ready context block of this user's memories: the most + recent ones plus (when `text` is given) the ones relevant to it. + Requires text access (`read:text` scope + the store's `expose_text` + flag).""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "max_chars": max_chars, + } + if text is not None: + payload["text"] = text + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._empty_if_unprovisioned( + lambda: self._searched(self._client.context_block, payload), + {"block": f"# Context: user {self.store}"}, + ) + return result["block"] + + def browse( + self, + *, + after: str | None = None, + limit: int = 25, + include_text: bool = False, + filter: dict[str, Any] | None = None, + agent_id: str | None = None, + run_id: str | None = None, + ) -> list[dict[str, Any]]: + """This store's memories, keyset-paged (ids + metadata, text when + asked and allowed).""" + payload: dict[str, Any] = { + "store": self.store, + "key": self.key, + "after": after, + "limit": limit, + "include_text": include_text, + "filter": filter, + } + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._empty_if_unprovisioned( + lambda: self._client.browse_documents(self.org, self.environment, payload), + {"documents": []}, + ) + return result["documents"] + + def delete_memories( + self, *, agent_id: str | None = None, run_id: str | None = None + ) -> dict[str, Any]: + """Delete this store's memories in place (expired ones included), + narrowable to one agent/run. The store stays registered — to forget + the user entirely (and free their entitlement slot), delete the + store itself (`CloudClient.delete_store` / `lodedb cloud store delete`). + Returns the acceptance (`write_ids`, `document_count`, `max_seq`); + pass a write id to ``wait_for`` to block until the removal folds. A + search on this handle waits for the deletion (session + read-your-writes), same as any other write.""" + payload: dict[str, Any] = {"store": self.store, "key": self.key} + if agent_id is not None: + payload["agent_id"] = agent_id + if run_id is not None: + payload["run_id"] = run_id + result = self._client.delete_memories(self.org, self.environment, payload) + for write_id in result.get("write_ids", []): + self.last_write_id = str(write_id) + # The deletion is a write like any other: raise the session's + # read-your-writes floor so a search on this handle waits for it — + # otherwise deleted memories can resurface until the fold lands. + max_seq = result.get("max_seq") + if max_seq is not None and int(max_seq) > self._last_seq: + self._last_seq = int(max_seq) + return result + + # ----------------------------------------------------------- lifecycle + + def close(self) -> None: + """Closes the underlying HTTP pool when this handle owns it; a + handle borrowed from a `Client` leaves the shared pool alone.""" + if self._owns_client: + self._client.close() + + def __enter__(self) -> CloudStore: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def __repr__(self) -> str: + where = f"{self.org}/{self.environment}/{self.store}" + return f"CloudStore({where!r})" + + +@dataclass(frozen=True) +class _BareStore: + """A single-segment target: just the store id. The org/environment half + comes from the credential (`resolve_tenancy`), the same way + `Client().store(...)` resolves it — so a user never retypes what their + environment-scoped token already pins down.""" + + store: str + + +def _parse_target(target: str) -> ManagedRemote | _BareStore: + """Accepts a bare store id (`user-42` — org/environment resolve from the + credential), `org/environment/store`, and the explicit `orecloud://` + spellings of all of these (the URL form also allows `org/environment`, + defaulting the store). The store segment is the end-user id in the + agentic-memory product — a store auto-provisions on its first write, so + nothing needs creating first.""" + body = target + is_url = target.startswith(ManagedRemote.SCHEME) + if is_url: + body = target[len(ManagedRemote.SCHEME) :] + # Empty segments are malformed, never silently collapsed: filtering them + # would reinterpret `orecloud://org//store` as the two-segment + # org/environment form and aim the handle at the wrong tenancy. + segments = body.strip("/").split("/") + if len(segments) == 1 and segments[0]: + return _BareStore(segments[0]) + if (len(segments) == 3 or (len(segments) == 2 and is_url)) and all(segments): + return ManagedRemote(*segments) + raise CloudError( + 422, + f"malformed target {target!r}; expected a bare store id ('user-42' — " + "org/environment come from the credential), 'org/environment/store', " + "or an orecloud:// URL", + ) + + +def connect( + target: str, + *, + token: str | None = None, + host: str | None = None, + key: str | None = None, + warm: bool = True, + timeout: float = 30.0, + read_your_writes: bool = True, + transport: httpx.BaseTransport | None = None, +) -> CloudStore: + """Open a read handle over a managed store, addressed by path string. + + Prefer :class:`lodedb.cloud.Client`: the org/environment half of `target` + repeats what an environment-scoped token already pins down, and + ``Client().store("user-42")`` resolves it from the credential instead. + This stays as sugar for one-off scripts and console copy-paste. It is + also the seam behind ``lodedb``'s constructor front doors — + ``LodeDB.cloud("user-42")`` and the ``LodeDB("orecloud://…")`` + config-string dispatch both land here (lodedb releases that ship the + `[cloud]` extra), so the two must keep accepting the same keywords. + + `target` is a bare store id (`"user-42"` — the org/environment half + resolves from the credential via `resolve_tenancy`, exactly like + `Client().store()`), `"org/environment/store"`, or an `orecloud://` URL + of either (the URL form also allows `org/environment`, defaulting the + store). Credentials: explicit arguments, else the + `ORECLOUD_TOKEN`/`ORECLOUD_HOST` environment pair, else the credentials + file `lodedb cloud login` wrote. `warm=True` (default) asks the serving tier + to hydrate and open the store now, so the first query is warm; it also + verifies the target exists and the credential can read it. `key` names + the index key when the store holds more than one (rare — a pushed LodeDB + directory can carry several). `read_your_writes=True` (default) makes a + search after a write on this handle wait briefly for that write's fold, + so the session always sees its own writes. + """ + from lodedb.cloud.client import resolve_credentials, resolve_tenancy + + remote = _parse_target(target) + host, token = resolve_credentials(token, host) + client = CloudClient(host, token, transport=transport, timeout=timeout) + if isinstance(remote, _BareStore): + # A bare store id carries no org/environment: resolve them from the + # credential (token binding, else the account's only choices), and + # never leak the pool when that resolution refuses. + try: + org, environment = resolve_tenancy(client, None, None) + except BaseException: + client.close() + raise + remote = ManagedRemote(org, environment, remote.store) + store = CloudStore( + client, + remote.org, + remote.environment, + remote.store, + key, + read_your_writes=read_your_writes, + ) + if warm: + try: + store.stats(warm=True) + except CloudError as error: + # Two fine-to-connect 404s: a store that exists but holds + # nothing yet (first `add()` creates its first snapshot), and a + # store that doesn't exist at all — a store is one end user, + # and users materialize on their first write, so connecting to + # a new user before their first memory is the normal flow. A + # bad org/environment still fails loudly (different detail). + connectable = error.status_code == 404 and ( + "nothing has been pushed" in error.detail + or "no such store" in error.detail + ) + if not connectable: + client.close() + raise + except BaseException: + # Transport failures (DNS, refused connection, timeout) must not + # leak the freshly opened pool either. + client.close() + raise + return store + + +# Back-compat alias from the index->store rename: existing imports of +# CloudIndex keep working; new code should use CloudStore. +CloudIndex = CloudStore diff --git a/src/lodedb/cloud/transfer.py b/src/lodedb/cloud/transfer.py new file mode 100644 index 00000000..49173f3b --- /dev/null +++ b/src/lodedb/cloud/transfer.py @@ -0,0 +1,964 @@ +"""The HTTP-speaking half of the client: a thin typed wrapper over the /v1 +control-plane API, the sealed-box login handoff, and the managed +(`orecloud://`) transfer verbs. + +Synchronous httpx by design — the CLI is a sequential tool, and the SDK's +cloud methods (Phase 3) will wrap this client in executors where needed. +`transport` is injectable so tests drive the real client against an +in-process ASGI app without a socket. + +The managed transfer functions compose two layers with a deliberate seam: +this module moves bytes over HTTP (begin/commit sessions, presigned or +proxied blob transfers), while everything that touches the commit format — +identities, inventories, classification, the pointer document, sidecar +trust, the verified restore — happens in the Rust core via +``lodedb._turbovec.cloud.managed_*``. Python never interprets a manifest: a +head body is parsed only as opaque JSON and re-serialised for the Rust core, +which recomputes every identity through the engine's own canonical writer +(key-order- and formatting-insensitive), so no digest ever depends on +Python's serialisation. +""" + +from __future__ import annotations + +import base64 +import json +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, BinaryIO +from urllib.parse import quote + +import httpx +from nacl.public import PrivateKey, SealedBox + +from lodedb._turbovec import cloud as _core + + +class CloudError(RuntimeError): + """A control-plane refusal, carrying the HTTP status and its detail.""" + + def __init__(self, status_code: int, detail: str): + super().__init__(f"{detail} (HTTP {status_code})") + self.status_code = status_code + self.detail = detail + + +def _raise_for(response: httpx.Response) -> None: + if response.status_code < 400: + return + try: + detail = response.json().get("detail", response.text) + except ValueError: + detail = response.text + raise CloudError(response.status_code, str(detail)) + + +class CloudClient: + def __init__( + self, + host: str, + token: str | None = None, + *, + transport: httpx.BaseTransport | None = None, + timeout: float = 30.0, + ): + headers = {} + if token: + headers["authorization"] = f"Bearer {token}" + self._http = httpx.Client( + base_url=host.rstrip("/"), headers=headers, transport=transport, timeout=timeout + ) + + def close(self) -> None: + self._http.close() + + def __enter__(self) -> CloudClient: + return self + + def __exit__(self, *exc) -> None: + self.close() + + def _request(self, method: str, path: str, **kwargs) -> Any: + response = self._http.request(method, path, **kwargs) + _raise_for(response) + if response.status_code == 204 or not response.content: + return None + return response.json() + + # ------------------------------------------------------------ identity + + def me(self) -> dict: + return self._request("GET", "/v1/auth/me") + + def token_self(self) -> dict: + """The presented token's own identity: kind, scopes, and — for + environment tokens — the org/environment slugs it is bound to + (both None for personal tokens).""" + return self._request("GET", "/v1/tokens/self") + + # ------------------------------------------------------------ tokens + + def list_tokens(self) -> list[dict]: + return self._request("GET", "/v1/tokens")["tokens"] + + def mint_token( + self, + kind: str, + scopes: list[str], + *, + name: str = "", + org: str | None = None, + environment: str | None = None, + expires_in_days: int | None = None, + ) -> dict: + body: dict[str, Any] = {"kind": kind, "scopes": scopes, "name": name} + if org: + body["org"] = org + if environment: + body["environment"] = environment + if expires_in_days is not None: + body["expires_in_days"] = expires_in_days + return self._request("POST", "/v1/tokens", json=body) + + def revoke_token(self, token_id: str) -> dict: + return self._request("POST", f"/v1/tokens/{token_id}/revoke") + + # ------------------------------------------------------------ tenancy + + def list_environments(self, org: str) -> list[dict]: + return self._request("GET", f"/v1/orgs/{org}/environments")["environments"] + + def list_stores( + self, + org: str, + environment: str, + *, + after: str | None = None, + limit: int = 50, + store: str | None = None, + prefix: str | None = None, + order: str = "name", + after_last_write: str | None = None, + ) -> dict: + """One keyset page of an environment's stores (a store is one end + user): {stores, count}. `store` narrows to one exact name (key + resolution, not paging); `prefix` to a name prefix. `order="name"` + pages with `after` (the previous page's last name); `order="recent"` + lists most-recently-written-first and pages with `after` AND + `after_last_write` (the previous page's last row's name and + ISO `last_write_at`) together.""" + params: dict[str, str] = {"limit": str(limit)} + if after is not None: + params["after"] = after + if store is not None: + params["store"] = store + if prefix is not None: + params["prefix"] = prefix + if order != "name": + params["order"] = order + if after_last_write is not None: + params["after_last_write"] = after_last_write + return self._request( + "GET", f"/v1/orgs/{org}/environments/{environment}/stores", params=params + ) + + def store_stats(self, org: str, environment: str) -> dict: + """Fleet-level counts over an environment's live stores (store = + one end user): {stores, active_24h, active_7d, new_7d}.""" + return self._request( + "GET", f"/v1/orgs/{org}/environments/{environment}/stores/stats" + ) + + def create_store( + self, + org: str, + environment: str, + store: str, + key: str | None = None, + *, + mode: str = "local_push", + expose_text: bool = False, + preset: str | None = None, + vector_dim: int | None = None, + ) -> dict: + body: dict[str, Any] = {"store": store, "mode": mode, "expose_text": expose_text} + if key is not None: + body["key"] = key + if preset is not None: + body["preset"] = preset + if vector_dim is not None: + body["vector_dim"] = vector_dim + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/stores", json=body + ) + + # ------------------------------------------------------------ lifecycle + + def delete_org(self, org: str) -> dict: + """Soft-delete an org. Returns the parked slug (restore takes it) + and the purge deadline.""" + return self._request("DELETE", f"/v1/orgs/{org}") + + def restore_org(self, parked_slug: str) -> dict: + return self._request("POST", f"/v1/orgs/{parked_slug}/restore") + + def delete_environment(self, org: str, environment: str) -> dict: + return self._request("DELETE", f"/v1/orgs/{org}/environments/{environment}") + + def restore_environment(self, org: str, parked_slug: str) -> dict: + return self._request("POST", f"/v1/orgs/{org}/environments/{parked_slug}/restore") + + def delete_store(self, org: str, environment: str, store: str) -> dict: + """Soft-delete a whole store (every index key in it hides with it). + + Store names are end-user ids — free-form up to '/' — so the path + segment is percent-encoded: a raw `?` or `#` would truncate the URL + and address a DIFFERENT store.""" + return self._request( + "DELETE", + f"/v1/orgs/{org}/environments/{environment}/stores/{quote(store, safe='')}", + ) + + def restore_store(self, org: str, environment: str, parked_store: str) -> dict: + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/" + f"{quote(parked_store, safe='')}/restore", + ) + + def delete_store_key(self, org: str, environment: str, store: str, key: str) -> dict: + """Soft-delete ONE index key inside a store (the advanced multi-key + path).""" + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/delete", + json={"store": store, "key": key}, + ) + + def restore_store_key(self, org: str, environment: str, store: str, parked_key: str) -> dict: + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/restore", + json={"store": store, "key": parked_key}, + ) + + def list_trash(self, org: str) -> list[dict]: + """Soft-deleted resources under a live org, restorable until their + `purge_after`.""" + return self._request("GET", f"/v1/orgs/{org}/trash")["items"] + + def export_org(self, org: str) -> dict: + """The offboarding manifest: every live environment and store with its + head snapshot identity (metadata only — bytes move via pull).""" + return self._request("GET", f"/v1/orgs/{org}/export") + + # ------------------------------------------------------------ transfer + + def store_head(self, org: str, environment: str, store: str, key: str) -> dict: + return self._request( + "GET", + f"/v1/orgs/{org}/environments/{environment}/stores/head", + params={"store": store, "key": key}, + ) + + def pull_plan(self, org: str, environment: str, store: str, key: str) -> dict: + return self._request( + "GET", + f"/v1/orgs/{org}/environments/{environment}/stores/pull-plan", + params={"store": store, "key": key}, + ) + + def store_history(self, org: str, environment: str, store: str, key: str) -> list[dict]: + """The store's restore window: every retained snapshot, newest first.""" + return self._request( + "GET", + f"/v1/orgs/{org}/environments/{environment}/stores/head-history", + params={"store": store, "key": key}, + )["snapshots"] + + def rollback_store( + self, org: str, environment: str, store: str, snapshot_id: str, key: str | None = None + ) -> dict: + """Moves the branch head back to a retained snapshot (reversible — + the displaced head stays in the window for the retention period).""" + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/rollback", + json={"store": store, "key": key, "snapshot_id": snapshot_id}, + ) + + def begin_push(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/push/begin", json=payload + ) + + def commit_push(self, org: str, environment: str, session_id: str, payload: dict) -> dict: + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/push/{session_id}/commit", + json=payload, + ) + + def heartbeat_push(self, org: str, environment: str, session_id: str) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/push/{session_id}/heartbeat" + ) + + def abort_push(self, org: str, environment: str, session_id: str) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/push/{session_id}/abort" + ) + + def update_store( + self, + org: str, + environment: str, + store: str, + key: str, + *, + expose_text: bool | None = None, + mode: str | None = None, + ) -> dict: + body: dict[str, Any] = {"store": store, "key": key} + if expose_text is not None: + body["expose_text"] = expose_text + if mode is not None: + body["mode"] = mode + return self._request( + "PATCH", f"/v1/orgs/{org}/environments/{environment}/stores", json=body + ) + + # ------------------------------------------------------------ serving + + def search(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/stores/search", json=payload + ) + + def search_many(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/stores/search-many", json=payload + ) + + def store_text( + self, org: str, environment: str, store: str, id: str, key: str | None = None + ) -> dict: + params: dict[str, str] = {"store": store, "id": id} + if key: + params["key"] = key + return self._request( + "GET", f"/v1/orgs/{org}/environments/{environment}/stores/text", params=params + ) + + def add_documents(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/stores/documents", json=payload + ) + + def remove_documents(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/documents/remove", + json=payload, + ) + + def browse_documents(self, org: str, environment: str, payload: dict) -> dict: + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/documents/browse", + json=payload, + ) + + # ---------------------------------------------- memory verbs (Phase 8d) + + def recall(self, org: str, environment: str, payload: dict) -> dict: + """Non-exact retrieval from raw text (server-side sub-queries + RRF).""" + return self._request( + "POST", f"/v1/orgs/{org}/environments/{environment}/stores/recall", json=payload + ) + + def context_block(self, org: str, environment: str, payload: dict) -> dict: + """A prompt-ready context block from one user's store.""" + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/context-block", + json=payload, + ) + + def delete_memories(self, org: str, environment: str, payload: dict) -> dict: + """Delete a store's memories in place (async, remove segments) — + the store row stays; `delete_store` forgets the user entirely.""" + return self._request( + "POST", + f"/v1/orgs/{org}/environments/{environment}/stores/memories/delete", + json=payload, + ) + + def write_status( + self, org: str, environment: str, store: str, write_id: str, key: str | None = None + ) -> dict: + """One accepted write's lifecycle: registered → folded/condemned.""" + params: dict[str, str] = {"store": store} + if key: + params["key"] = key + return self._request( + "GET", + f"/v1/orgs/{org}/environments/{environment}/stores/writes/{write_id}", + params=params, + ) + + def serving_stats( + self, + org: str, + environment: str, + store: str, + key: str | None = None, + *, + warm: bool = False, + ) -> dict: + params: dict[str, str] = {"store": store} + if key: + params["key"] = key + if warm: + params["warm"] = "true" + return self._request( + "GET", f"/v1/orgs/{org}/environments/{environment}/stores/serving-stats", params=params + ) + + def upload_blob_proxy(self, proxy_path: str, handle: BinaryIO) -> None: + """Streams one blob through the control plane's authenticated proxy.""" + response = self._http.put(proxy_path, content=handle, timeout=None) + _raise_for(response) + + def download_blob_proxy(self, proxy_path: str, dest: Path) -> None: + with self._http.stream("GET", proxy_path, timeout=None) as response: + if response.status_code >= 400: + response.read() # buffer the error body so _raise_for can parse it + _raise_for(response) + with open(dest, "wb") as out: + for chunk in response.iter_bytes(): + out.write(chunk) + + +# ---------------------------------------------------------------- login + +@dataclass(frozen=True) +class LoginStart: + session_id: str + user_code: str + verification_url: str + poll_interval_seconds: int + + +class LoginHandoff: + """One CLI login attempt: keypair, session, poll loop, unseal. + + The private key lives only in this object (memory, this process); the + server ever sees the public half, and the minted token comes back as a + sealed box only this keypair opens. + """ + + def __init__(self, client: CloudClient, client_label: str): + self._client = client + self._key = PrivateKey.generate() + info = client._request( + "POST", + "/v1/cli/sessions", + json={ + "public_key": bytes(self._key.public_key).hex(), + "client_label": client_label, + }, + ) + self.start = LoginStart( + session_id=info["id"], + user_code=info["user_code"], + verification_url=info["verification_url"], + poll_interval_seconds=info["poll_interval_seconds"], + ) + + def poll_once(self) -> tuple[str, str | None]: + """One poll: (state, token). The token appears on the claiming poll.""" + result = self._client._request("GET", f"/v1/cli/sessions/{self.start.session_id}") + sealed = result.get("sealed_token") + if sealed is None: + return result["state"], None + token = SealedBox(self._key).decrypt(base64.b64decode(sealed)).decode("utf-8") + return result["state"], token + + def wait(self, *, timeout_seconds: float = 900.0, sleep=time.sleep) -> str: + """Polls until approved and returns the token; raises CloudError on + denial/expiry/timeout.""" + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + state, token = self.poll_once() + if token is not None: + return token + if state in ("denied", "expired", "claimed"): + raise CloudError(409, f"login {state}") + sleep(self.start.poll_interval_seconds) + raise CloudError(408, "login timed out waiting for approval") + + +# ---------------------------------------------------------------- managed + +# Engine store kinds → the wire contract's blob kinds. `tvann` (persisted ANN +# clusters) and `tvvf` (the rescore original-vector sidecar) are vector-derived, +# payload-free like `tvim`. Deliberately no default: an engine kind this table +# does not know must fail loudly at `_wire_kind` rather than ship mislabelled — +# the Rust inventory fails closed on unknown sub-manifests for the same reason. +_ENGINE_KIND_TO_WIRE = { + "json": "state", + "tvim": "vector", + "tvmv": "vector", + "tvtext": "text", + "tvlex": "lexical", + "tvann": "vector", + "tvvf": "vector", +} + + +def _wire_kind(engine_kind: str) -> str: + try: + return _ENGINE_KIND_TO_WIRE[engine_kind] + except KeyError: + raise CloudError( + 422, + f"this client cannot classify engine store kind {engine_kind!r} for the " + "wire contract — upgrade lodedb before pushing this generation", + ) from None + + +class SyncConflictError(CloudError): + """A managed sync refused to transfer, or lost a race with a concurrent + writer (diverged/unknown lineage needing an explicit force, a commit-time + pointer conflict, or a head that moved mid-sync). The single conflict + surface: callers retry the sync or resolve with --force-push/--force-pull; + every other CloudError is a genuine refusal.""" + + +@dataclass(frozen=True) +class ManagedRemote: + """A parsed `orecloud://org/environment[/store]` target.""" + + org: str + environment: str + store: str = "memory" + + SCHEME = "orecloud://" + + @classmethod + def parse(cls, target: str) -> ManagedRemote | None: + """The parsed remote, or None when `target` is not an orecloud URL. + A malformed orecloud URL raises (never silently falls through to the + dumb-target path).""" + if not target.startswith(cls.SCHEME): + return None + segments = target[len(cls.SCHEME) :].strip("/").split("/") + if len(segments) not in (2, 3) or not all(segments): + raise CloudError( + 422, + f"malformed managed target {target!r}; expected " + "orecloud://org/environment[/store]", + ) + return cls(*segments) + + def identity(self, host: str) -> str: + """The sidecar remote-identity string. Includes the control-plane + host: the same org/environment on two deployments is two remotes.""" + return ( + f"{self.SCHEME}{self.org}/{self.environment}/{self.store}" + f"#host={host.rstrip('/')}" + ) + + +def _plan(client: CloudClient, dir: str, key: str, remote: ManagedRemote, + host: str, *, include_text: bool, include_lexical: bool) -> tuple[dict, dict]: + """One head fetch + one Rust plan: (head response, plan dict).""" + head = client.store_head(remote.org, remote.environment, remote.store, key) + body = head.get("body") + plan = _core.managed_plan( + dir, + key, + remote.identity(host), + json.dumps(body) if body is not None else None, + include_text=include_text, + include_lexical=include_lexical, + ) + return head, plan + + +def _upload_blobs(client: CloudClient, dir: str, plan_local: dict, need_upload: list[dict]) -> int: + """Moves the need-upload set: presigned PUT when offered (falling back to + the proxy on transport failure), else the proxy. Returns bytes sent.""" + name_by_sha: dict[str, str] = {} + for artifact in plan_local["artifacts"]: + name_by_sha.setdefault(artifact["sha256"], artifact["name"]) + sent = 0 + for item in need_upload: + name = name_by_sha.get(item["sha256"]) + if name is None: + raise CloudError( + 409, + f"server asked for blob {item['sha256']} this push never declared", + ) + path = Path(dir) / name + uploaded = False + if item.get("put_url"): + try: + with open(path, "rb") as handle: + response = httpx.put( + item["put_url"], + content=handle, + headers=item.get("put_headers") or {}, + timeout=None, + ) + response.raise_for_status() + uploaded = True + except httpx.HTTPError: + uploaded = False # unreachable endpoint or refusal: use the proxy + if not uploaded: + with open(path, "rb") as handle: + client.upload_blob_proxy(item["proxy_path"], handle) + sent += item["size_bytes"] + return sent + + +def _push_with_plan( + client: CloudClient, + dir: str, + key: str, + remote: ManagedRemote, + host: str, + head: dict, + plan: dict, +) -> dict: + """The push protocol against an already-classified head: begin (CAS-armed + with that head), upload, commit, record the sidecar base.""" + local = plan["local"] + if local is None: + raise CloudError(404, f"no committed generation to push for index key {key!r}") + identity = remote.identity(host) + expected_head = head["head"]["snapshot_id"] if head.get("head") else None + + if expected_head == local["snapshot_id"]: + # The remote already holds exactly this snapshot: record the agreed + # base and publish nothing. + _core.managed_record_base(dir, key, identity, local["body_json"]) + return { + "index_key": key, + "generation": local["generation"], + "artifacts_written": 0, + "artifacts_skipped": len(local["artifacts"]), + "bytes_written": 0, + "pointer_published": False, + } + + begin = client.begin_push( + remote.org, + remote.environment, + { + "store": remote.store, + "key": key, + "snapshot_id": local["snapshot_id"], + "logical_id": local["logical_id"], + "generation": local["generation"], + "expected_head": expected_head, + "artifacts": [ + { + "sha256": artifact["sha256"], + "size_bytes": artifact["size_bytes"], + "kind": _wire_kind(artifact["kind"]), + } + for artifact in local["artifacts"] + ], + }, + ) + try: + bytes_written = _upload_blobs(client, dir, local, begin["need_upload"]) + base = plan.get("base") + client.commit_push( + remote.org, + remote.environment, + begin["session_id"], + { + "body": json.loads(local["body_json"]), + "pointer_document": local["pointer_document"], + "parent_snapshot_id": base["snapshot_id"] if base else None, + }, + ) + except BaseException: + # Best-effort: release the server-side session instead of leaving it + # to expire on its own. The original failure is what matters — an + # abort that itself fails (network already gone) must not mask it. + try: + client.abort_push(remote.org, remote.environment, begin["session_id"]) + except Exception: + pass + raise + _core.managed_record_base(dir, key, identity, local["body_json"]) + return { + "index_key": key, + "generation": local["generation"], + "artifacts_written": len(begin["need_upload"]), + "artifacts_skipped": len(local["artifacts"]) - len(begin["need_upload"]), + "bytes_written": bytes_written, + "pointer_published": True, + } + + +def managed_push( + client: CloudClient, + dir: str, + remote: ManagedRemote, + key: str, + *, + host: str, + include_text: bool = False, + include_lexical: bool = False, +) -> dict: + """Publish the local committed generation, raced through the head CAS — + the managed analogue of the dumb `push` verb (last writer wins; a + concurrent advance surfaces as a 409, and divergence *protection* is + `managed_sync`'s job).""" + head, plan = _plan( + client, dir, key, remote, host, + include_text=include_text, include_lexical=include_lexical, + ) + return _push_with_plan(client, dir, key, remote, host, head, plan) + + +def _pull_with_body( + client: CloudClient, + dir: str, + key: str, + remote: ManagedRemote, + host: str, + expected_snapshot_id: str | None, + discard_pending_wal: bool = False, + expected_local_snapshot_id: str | None = None, +) -> dict: + """The pull protocol: plan, download the missing blob set into staging, + then let the Rust core materialise + verify-open + record the sidecar. + + `expected_snapshot_id` pins the pull to a classified head: if the remote + advanced between classification and the plan fetch, refuse (retry the + sync) rather than restore a snapshot nothing classified.""" + plan_response = client.pull_plan(remote.org, remote.environment, remote.store, key) + snapshot = plan_response["snapshot"] + if expected_snapshot_id is not None and snapshot["snapshot_id"] != expected_snapshot_id: + # Only sync pins the expected head, so this is always a sync-level + # conflict (SyncConflictError subclasses CloudError; plain-pull + # callers that catch CloudError are unaffected). + raise SyncConflictError( + 409, + "the remote head moved between classification and pull; re-run the sync", + ) + body_json = json.dumps(plan_response["body"]) + needed = { + artifact["sha256"] + for artifact in _core.managed_pull_requirements(dir, key, body_json) + } + downloads = { + blob["sha256"]: blob for blob in plan_response["blobs"] if blob["sha256"] in needed + } + missing = needed - set(downloads) + if missing: + raise CloudError( + 502, f"pull plan omits {len(missing)} blob(s) the body references" + ) + + with tempfile.TemporaryDirectory(prefix="orecloud-pull-") as staging: + for sha, blob in sorted(downloads.items()): + dest = Path(staging) / sha + fetched = False + if blob.get("get_url"): + try: + with httpx.stream("GET", blob["get_url"], timeout=None) as response: + response.raise_for_status() + with open(dest, "wb") as out: + for chunk in response.iter_bytes(): + out.write(chunk) + fetched = True + except httpx.HTTPError: + fetched = False # unreachable endpoint: use the proxy + if not fetched: + client.download_blob_proxy(blob["proxy_path"], dest) + return _core.managed_materialize( + dir, + key, + remote.identity(host), + body_json, + staging, + discard_pending_wal=discard_pending_wal, + expected_local_snapshot_id=expected_local_snapshot_id, + ) + + +def managed_pull( + client: CloudClient, + remote: ManagedRemote, + dir: str, + key: str, + *, + host: str, +) -> dict: + """Restore the branch head into `dir` and verify it opens — the managed + analogue of the dumb `pull` verb.""" + return _pull_with_body(client, dir, key, remote, host, expected_snapshot_id=None) + + +def managed_status( + client: CloudClient, + dir: str, + remote: ManagedRemote, + key: str, + *, + host: str, + include_text: bool = False, + include_lexical: bool = False, +) -> dict: + """The status report for a managed remote — same fields as the dumb + `status` verb, lineage included.""" + _head, plan = _plan( + client, dir, key, remote, host, + include_text=include_text, include_lexical=include_lexical, + ) + return { + field: value + for field, value in plan.items() + if field + not in ("local", "remote", "base", "base_is_current", "local_raw_snapshot_id") + } + + +def managed_sync( + client: CloudClient, + dir: str, + remote: ManagedRemote, + key: str, + *, + host: str, + include_text: bool = False, + include_lexical: bool = False, + force_push: bool = False, + force_pull: bool = False, +) -> dict: + """Three-pointer sync against a managed remote: classify (local, sidecar + base, branch head), then run at most one fast-forward — the same decision + table as the Rust `sync` verb, with the head CAS closing the race window. + """ + if force_push and force_pull: + raise ValueError("force_push and force_pull are mutually exclusive") + head, plan = _plan( + client, dir, key, remote, host, + include_text=include_text, include_lexical=include_lexical, + ) + classification = plan["classification"] + identity = remote.identity(host) + + def outcome(action: str, forced: bool, transfer: dict | None) -> dict: + report = dict(transfer) if transfer else {"index_key": key} + report["classification"] = classification + report["action"] = action + report["forced"] = forced + report["sidecar_corrupt"] = plan["sidecar_corrupt"] + return report + + def _push_translating_conflict() -> dict: + """The commit-time CAS 409 (head moved since the push began) is a + sync-level conflict, not a generic refusal: re-running the sync + re-classifies against the new head and usually resolves it.""" + try: + return _push_with_plan(client, dir, key, remote, host, head, plan) + except CloudError as error: + if error.status_code == 409 and "pointer conflict" in error.detail: + raise SyncConflictError( + 409, f"{error.detail}; re-run the sync to reconcile" + ) from error + raise + + # The local state this classification saw, as the materialization pin: + # a local commit landing after this point refuses instead of being + # overwritten ("" pins to classified-as-absent). + classified_local = plan.get("local_raw_snapshot_id") or "" + + def _refuse_pending_wal() -> None: + """A pull-direction transfer must not run over a local WAL still + holding acknowledged writes (replaying them onto the pulled lineage + would corrupt it; dropping them silently loses acked data). Refusing + HERE — before a single blob downloads — mirrors the Rust verbs; the + materialize step re-checks authoritatively under the writer lock. The + scan runs only on this pull branch, so push/status planning never + pays for it.""" + ops = _core.local_wal_ops(dir, key) + if ops: + raise SyncConflictError( + 409, + f"the local database holds {ops} uncheckpointed WAL operation(s); " + "checkpoint them by opening the store once, or re-run with " + "--force-pull to discard them along with the local lineage", + ) + + if force_push: + return outcome("push", True, _push_translating_conflict()) + if force_pull: + head_sha = head["head"]["snapshot_id"] if head.get("head") else None + if head_sha is None: + raise CloudError(404, f"no committed generation to pull for index key {key!r}") + return outcome( + "pull", + True, + _pull_with_body( + client, + dir, + key, + remote, + host, + head_sha, + discard_pending_wal=True, + expected_local_snapshot_id=classified_local, + ), + ) + + if classification == "in_sync": + # Mirror the Rust sync's stale-base repair: agreeing ends with a + # missing/stale recorded base record the agreed state. + if not plan["base_is_current"] and head.get("body") is not None: + _core.managed_record_base(dir, key, identity, json.dumps(head["body"])) + return outcome("none", False, None) + if classification in ("local_ahead", "republish"): + return outcome("push", False, _push_translating_conflict()) + if classification == "remote_ahead": + head_sha = head["head"]["snapshot_id"] if head.get("head") else None + if head_sha is None: + raise CloudError(404, f"no committed generation to pull for index key {key!r}") + _refuse_pending_wal() + return outcome( + "pull", + False, + _pull_with_body( + client, + dir, + key, + remote, + host, + head_sha, + expected_local_snapshot_id=classified_local, + ), + ) + + # diverged/unknown: refuse with the same wording as the Rust verb. + hint = ( + "re-run with --force-push to keep the local copy or --force-pull to keep " + "the remote copy" + ) + if plan["sidecar_corrupt"]: + hint += ( + " (note: the sync sidecar was present but corrupt and was ignored, so the " + "recorded base could not be trusted)" + ) + raise SyncConflictError( + 409, f"sync refused: local and remote are {classification}; {hint}" + ) diff --git a/src/lodedb/local/cli.py b/src/lodedb/local/cli.py index 6a1ee27a..ae85a638 100644 --- a/src/lodedb/local/cli.py +++ b/src/lodedb/local/cli.py @@ -347,6 +347,40 @@ def serve( app.add_typer(migrate_app, name="migrate") +@app.command( + "cloud", + context_settings={ + # Everything after `lodedb cloud` belongs to the cloud CLI, including + # --help and unknown options — forward it all untouched. + "allow_extra_args": True, + "ignore_unknown_options": True, + "help_option_names": [], + }, +) +def cloud(ctx: typer.Context) -> None: + """Sync, serve, and manage LodeDB stores in the managed cloud (OreCloud). + + A trampoline into the first-party cloud CLI (`lodedb.cloud.cli`). The + import is deliberately lazy: the client modules pull the [cloud] extra's + dependencies (httpx), and a plain `import lodedb` must never load those + (tests/test_import_boundary.py). + """ + try: + from lodedb.cloud.cli import app as cloud_app + except ImportError: + typer.echo( + 'the cloud client dependencies are not installed — run: pip install "lodedb[cloud]"', + err=True, + ) + # from None: the hint IS the diagnosis; the ImportError adds noise. + raise typer.Exit(code=1) from None + from typer.main import get_command + + get_command(cloud_app).main( + args=list(ctx.args), prog_name="lodedb cloud", standalone_mode=True + ) + + @mcp_app.callback(invoke_without_command=True) def mcp( ctx: typer.Context, diff --git a/src/lodedb/local/db.py b/src/lodedb/local/db.py index 2e52058b..209c4004 100644 --- a/src/lodedb/local/db.py +++ b/src/lodedb/local/db.py @@ -36,6 +36,7 @@ import numpy as np +from lodedb.cloud import CLOUD_TARGET_SCHEME, open_cloud_target from lodedb.engine._atomic_io import durability_from_env, normalize_durability from lodedb.engine._commit_manifest import ( COMMIT_MANIFEST_SUFFIX, @@ -221,8 +222,63 @@ class LodeDB: fox = db.add("the quick brown fox") db.get(fox) # -> "the quick brown fox" db.get_texts([fox]) # -> {fox: "the quick brown fox"} + + A managed `OreCloud `_ store opens through + the same verbs via :meth:`cloud` (requires the ``lodedb[cloud]`` extra; + credentials come from ``token=``, the ``ORECLOUD_TOKEN``/``ORECLOUD_HOST`` + environment pair, or ``lodedb cloud login``):: + + db = LodeDB.cloud("user-42") # org/environment from the credential + db.add("the quick brown fox") # embedded server-side + db.search("fox", k=5) + + For config-driven code, where one string field (an env var, a YAML value) + must express either a local path or a cloud store, the constructor itself + accepts the explicit URL form — ``LodeDB("orecloud://org/environment/store")`` + — and returns the same handle. Either way the handle duck-types this + class's read/write surface (``add``, ``search``, ``get``, ``remove``, ...) + but runs over HTTPS, so local-only construction options (``model=``, + ``store_text=``, ...) don't apply and local-only verbs (``persist``, + ``open_readonly``) don't exist on it. """ + def __new__(cls, path: str | Path | None = None, **kwargs: Any) -> Any: + """Routes ``orecloud://`` config strings to the managed-cloud companion. + + This dispatch exists for config-driven cases where one string field + must express either a local path or a cloud store; humans should + reach for :meth:`cloud` instead. Everything else falls through to + normal construction, keeping ``LodeDB(...)`` the single front door. + The returned cloud handle is not a ``LodeDB`` instance (so + ``__init__`` never runs on it); it duck-types the same verb surface. + """ + target = path if path is not None else kwargs.get("path") + if isinstance(target, str) and target.startswith(CLOUD_TARGET_SCHEME): + options = {name: value for name, value in kwargs.items() if name != "path"} + return open_cloud_target(target, options) + return super().__new__(cls) + + @classmethod + def cloud(cls, target: str, **options: Any) -> Any: + """Opens a managed OreCloud store — the cloud counterpart of + ``LodeDB(path)``, joining :meth:`open_readonly` / + :meth:`open_vector_store` in the alternate-constructor family. + + ``target`` is a bare store id (``"user-42"`` — org/environment + resolve from the environment-scoped credential, exactly like + ``orecloud.Client().store()``), an ``"org/environment/store"`` + triple for cross-environment scripts, or a full ``orecloud://`` URL. + Keyword options are ``orecloud.connect``'s (``token=``, ``host=``, + ``key=``, ``warm=``, ``timeout=``, ``read_your_writes=``, + ``transport=``). Returns the companion's store handle — duck-typed + to this class's verb surface, not a ``LodeDB`` instance — through + the same funnel as the ``LodeDB("orecloud://…")`` config-string + dispatch. Without the ``lodedb[cloud]`` extra installed this raises + an ``ImportError`` carrying the install hint; the companion is + imported only inside this call, never on a plain ``import lodedb``. + """ + return open_cloud_target(target, options) + def __init__( self, path: str | Path, diff --git a/tests/conftest.py b/tests/conftest.py index 1f70f3e2..559a736e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import types import numpy as np +import pytest try: import lodedb._turbovec # noqa: F401 @@ -164,3 +165,49 @@ def reconstruct_rows(self, stable_ids: np.ndarray) -> np.ndarray: return np.array(rows, dtype=np.float32) mock_turbovec.IdMapIndex = MockIdMapIndex + + +# -------------------------------------------------------------------------- +# Cloud-transfer fixtures: real committed generations in throwaway +# directories, authored through the actual engine (the same commit path a +# user's database goes through) with the deterministic hash embedding +# backend, so the `lodedb.cloud` transfer suites stay fast and network-free. + +# Small but non-trivial corpus: enough documents for real base + text artifacts. +DOCUMENTS = [{"text": f"the quick brown document number {i}", "id": f"doc-{i}"} for i in range(6)] + + +def write_committed_store(path) -> str: + """Writes one committed generation (with raw text) into `path`; returns its index key.""" + from lodedb import cloud + from lodedb.engine.embedding_backends import HashEmbeddingBackend + from lodedb.local.db import LodeDB + + db = LodeDB( + path=path, + model="minilm", + commit_mode="generation", + _embedding_backend=HashEmbeddingBackend(native_dim=384), + ) + try: + db.add_many(DOCUMENTS) + finally: + db.close() + (key,) = cloud.keys(str(path)) + return key + + +@pytest.fixture() +def committed_store(tmp_path): + """A LodeDB directory holding one committed generation, as `(dir_path, index_key)`.""" + source = tmp_path / "source" + source.mkdir() + return source, write_committed_store(source) + + +def read_pointer_body(path, key: str) -> dict: + """Reads the committed root body from `/.commit.json` (raw JSON, no engine).""" + import json + + with open(path / f"{key}.commit.json", encoding="utf-8") as pointer: + return json.load(pointer)["body"] diff --git a/tests/test_cloud_agents_scaffold.py b/tests/test_cloud_agents_scaffold.py new file mode 100644 index 00000000..7b281fe6 --- /dev/null +++ b/tests/test_cloud_agents_scaffold.py @@ -0,0 +1,91 @@ +"""`lodedb cloud init --agents` artifact generators: content correctness and the +leave-user-files-alone rules (no server needed — pure filesystem).""" + +import json + +from lodedb.cloud._agents_scaffold import ( + AGENTS_SECTION_HEADING, + MCP_SERVER_NAME, + scaffold_agent_artifacts, +) + +LINK = dict(host="https://cloud.example.com", org="acme", environment="support", store="docs") + + +def test_fresh_repo_gets_all_three_artifacts(tmp_path): + written, notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert written == [".claude/skills/orecloud/SKILL.md", "AGENTS.md", ".mcp.json"] + assert notes == [] + + skill = (tmp_path / ".claude/skills/orecloud/SKILL.md").read_text() + # Agent Skills spec: frontmatter with name + description (<=1024 chars). + assert skill.startswith("---\nname: orecloud\ndescription: ") + description = skill.split("description: ", 1)[1].split("\n", 1)[0] + assert len(description) <= 1024 + assert skill.count("\n") <= 500 + # Generated from the REAL link, not placeholders. + assert "acme/support/docs" in skill + assert "https://cloud.example.com" in skill + assert "min_seq" in skill # the async-write rule agents trip on + + agents = (tmp_path / "AGENTS.md").read_text() + assert agents.startswith(AGENTS_SECTION_HEADING) + assert "lodedb cloud sync" in agents + + config = json.loads((tmp_path / ".mcp.json").read_text()) + server = config["mcpServers"][MCP_SERVER_NAME] + assert server["url"] == "https://cloud.example.com/mcp/control" + # Env expansion, never a literal secret. + assert server["headers"]["Authorization"] == "Bearer ${ORECLOUD_TOKEN}" + + +def test_existing_agents_md_gains_section_once(tmp_path): + (tmp_path / "AGENTS.md").write_text("# My repo\n\nRules.\n") + written, notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert "AGENTS.md" in written + content = (tmp_path / "AGENTS.md").read_text() + assert content.startswith("# My repo") + assert AGENTS_SECTION_HEADING in content + + # Second run: the section is the repo's to edit now — never regenerated. + (tmp_path / "AGENTS.md").write_text(content.replace("lodedb cloud sync", "MY EDIT")) + written, notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert "AGENTS.md" not in written + assert any("AGENTS.md" in note for note in notes) + assert "MY EDIT" in (tmp_path / "AGENTS.md").read_text() + + +def test_existing_mcp_json_is_merged_not_replaced(tmp_path): + (tmp_path / ".mcp.json").write_text( + json.dumps({"mcpServers": {"other": {"type": "stdio", "command": "x"}}}) + ) + written, _notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert ".mcp.json" in written + config = json.loads((tmp_path / ".mcp.json").read_text()) + assert set(config["mcpServers"]) == {"other", MCP_SERVER_NAME} + + # An existing orecloud-control entry is theirs (maybe a different host). + config["mcpServers"][MCP_SERVER_NAME]["url"] = "https://elsewhere/mcp/control" + (tmp_path / ".mcp.json").write_text(json.dumps(config)) + written, notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert ".mcp.json" not in written + assert any(MCP_SERVER_NAME in note for note in notes) + reread = json.loads((tmp_path / ".mcp.json").read_text()) + assert reread["mcpServers"][MCP_SERVER_NAME]["url"] == "https://elsewhere/mcp/control" + + +def test_unparseable_mcp_json_is_left_alone(tmp_path): + (tmp_path / ".mcp.json").write_text("{not json") + written, notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert ".mcp.json" not in written + assert any("not valid JSON" in note for note in notes) + assert (tmp_path / ".mcp.json").read_text() == "{not json" + + +def test_skill_is_regenerated_each_run(tmp_path): + scaffold_agent_artifacts(tmp_path, **LINK) + skill_path = tmp_path / ".claude/skills/orecloud/SKILL.md" + skill_path.write_text("stale") + written, _notes = scaffold_agent_artifacts(tmp_path, **LINK) + assert ".claude/skills/orecloud/SKILL.md" in written + assert "acme/support/docs" in skill_path.read_text() diff --git a/tests/test_cloud_cli.py b/tests/test_cloud_cli.py new file mode 100644 index 00000000..29beefbd --- /dev/null +++ b/tests/test_cloud_cli.py @@ -0,0 +1,65 @@ +"""CLI smoke tests (`lodedb cloud ...` via typer's CliRunner): each verb runs the +same round trip the API tests cover, plus clean nonzero-exit error handling. + +CliRunner pipes stdout (not a TTY), so these tests exercise the agent-facing +default: JSON on stdout. `--no-json` pins the human render.""" + +import json + +import pytest +from typer.testing import CliRunner + +# Collection must skip, not error, without the [cloud] extra installed +# (the modules below import httpx / pynacl at module level). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb.cloud.cli import app # noqa: E402 + +runner = CliRunner() + + +def test_cli_round_trip(committed_store, tmp_path): + source, key = committed_store + remote = tmp_path / "remote" + restored = tmp_path / "restored" + + result = runner.invoke(app, ["keys", str(source)]) + assert result.exit_code == 0 + assert json.loads(result.output) == [key] + + result = runner.invoke(app, ["push", str(source), str(remote), key]) + assert result.exit_code == 0 + assert json.loads(result.output)["pointer_published"] is True + + result = runner.invoke(app, ["status", str(source), str(remote), key]) + assert result.exit_code == 0 + assert json.loads(result.output)["in_sync"] is True + + result = runner.invoke(app, ["verify", str(remote), key]) + assert result.exit_code == 0 + + result = runner.invoke(app, ["pull", str(remote), str(restored), key]) + assert result.exit_code == 0 + assert "document_count" in json.loads(result.output) + + +def test_cli_human_output_on_request(committed_store, tmp_path): + """`--no-json` forces the aligned field/value render even when piped.""" + source, key = committed_store + remote = tmp_path / "remote" + result = runner.invoke(app, ["--no-json", "push", str(source), str(remote), key]) + assert result.exit_code == 0 + assert "pointer_published True" in result.output + + +def test_cli_missing_generation_exits_nonzero(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + result = runner.invoke(app, ["verify", str(empty), "no-such-key"]) + assert result.exit_code == 1 + + +def test_cli_bad_scheme_exits_nonzero(tmp_path): + result = runner.invoke(app, ["verify", "ftp://nope/x", "idx"]) + assert result.exit_code == 1 diff --git a/tests/test_cloud_client_api.py b/tests/test_cloud_client_api.py new file mode 100644 index 00000000..2ee4a2b7 --- /dev/null +++ b/tests/test_cloud_client_api.py @@ -0,0 +1,84 @@ +"""End-to-end tests of the Python API (`lodedb.cloud.*` over the native core): +push/status/verify/pull round trips against real engine-written generations, +the redacted-by-default privacy posture, and error mapping onto stdlib +exceptions. +""" + +import pytest +from conftest import DOCUMENTS, read_pointer_body + +from lodedb import cloud + + +def test_push_status_pull_round_trip(committed_store, tmp_path): + """The full backup/restore cycle through local directories.""" + source, key = committed_store + remote = tmp_path / "remote" + restored = tmp_path / "restored" + + before = cloud.status(str(source), str(remote), key) + assert not before["in_sync"] + assert before["remote_generation"] is None + + pushed = cloud.push(str(source), str(remote), key) + assert pushed["pointer_published"] + assert pushed["artifacts_written"] > 0 + + after = cloud.status(str(source), str(remote), key) + assert after["in_sync"] + assert after["artifacts_to_upload"] == 0 + + report = cloud.verify(str(remote), key) + assert report["artifacts_verified"] > 0 + + # Pull restores AND proves the copy opens through the engine. + outcome = cloud.pull(str(remote), str(restored), key) + assert outcome["pointer_published"] + assert outcome["document_count"] == len(DOCUMENTS) + + # The restored copy verifies clean too. + cloud.verify(str(restored), key) + + +def test_repeated_push_is_idempotent(committed_store, tmp_path): + source, key = committed_store + remote = tmp_path / "remote" + cloud.push(str(source), str(remote), key) + again = cloud.push(str(source), str(remote), key) + assert again["artifacts_written"] == 0 + assert again["bytes_written"] == 0 + assert not again["pointer_published"] + + +def test_push_is_redacted_by_default(committed_store, tmp_path): + """Without the opt-in flags, the published remote body carries no text store.""" + source, key = committed_store + remote = tmp_path / "remote" + cloud.push(str(source), str(remote), key) + + body = read_pointer_body(remote, key) + assert body["tvtext"] is None + assert body["tvlex"] is None + assert body["json"] is not None + # The source itself still has its text store — redaction is per-transfer. + assert read_pointer_body(source, key)["tvtext"] is not None + + +def test_opt_in_flags_ship_the_text_store(committed_store, tmp_path): + source, key = committed_store + remote = tmp_path / "remote" + cloud.push(str(source), str(remote), key, include_text=True) + assert read_pointer_body(remote, key)["tvtext"] is not None + + +def test_missing_generation_raises_file_not_found(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(FileNotFoundError): + cloud.push(str(empty), str(tmp_path / "remote"), "no-such-key") + + +def test_bad_target_scheme_raises_runtime_error(committed_store, tmp_path): + source, key = committed_store + with pytest.raises(RuntimeError, match="scheme"): + cloud.push(str(source), "ftp://nope/x", key) diff --git a/tests/test_cloud_connect_targets.py b/tests/test_cloud_connect_targets.py new file mode 100644 index 00000000..4e19d1ba --- /dev/null +++ b/tests/test_cloud_connect_targets.py @@ -0,0 +1,124 @@ +"""`connect()` target forms: the full `org/environment/store` triple (and its +`orecloud://` spelling) parses locally with zero HTTP, while a bare store id +(`user-42`, the form behind `LodeDB.cloud("user-42")`) resolves its +org/environment from the credential via token introspection — exactly like +`Client().store()`. HTTP is a `httpx.MockTransport`, so these run serverless; +the introspection contract itself is covered in `server/tests`.""" + +from __future__ import annotations + +import pytest + +# Collection must skip, not error, without the [cloud] extra installed +# (httpx and the modules below are the extra's dependencies). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +import httpx # noqa: E402 + +from lodedb.cloud.serving import _BareStore, _parse_target, connect # noqa: E402 +from lodedb.cloud.transfer import CloudError # noqa: E402 + + +def _transport(handled: dict[str, dict]) -> httpx.MockTransport: + """Answers each path in `handled` with its JSON; anything else fails the + test loudly (a target that should parse locally must make no HTTP call).""" + + def handler(request: httpx.Request) -> httpx.Response: + body = handled.get(request.url.path) + assert body is not None, f"unexpected HTTP call: {request.url.path}" + return httpx.Response(200, json=body) + + return httpx.MockTransport(handler) + + +# ------------------------------------------------------------ parsing only + + +def test_triple_and_url_forms_parse_locally(): + for target in ("acme/prod/user-42", "orecloud://acme/prod/user-42"): + remote = _parse_target(target) + assert (remote.org, remote.environment, remote.store) == ("acme", "prod", "user-42") + + +def test_two_segment_url_defaults_the_store(): + remote = _parse_target("orecloud://acme/prod") + assert (remote.org, remote.environment, remote.store) == ("acme", "prod", "memory") + + +def test_single_segment_forms_are_bare_stores(): + for target in ("user-42", "orecloud://user-42"): + assert _parse_target(target) == _BareStore("user-42") + + +def test_two_segment_bare_form_stays_malformed(): + """Without the scheme, two segments are ambiguous (org/environment vs + environment/store) and refuse with the accepted forms named.""" + with pytest.raises(CloudError, match="bare store id"): + _parse_target("prod/user-42") + + +# --------------------------------------------------------------- connect() + + +def test_bare_store_resolves_tenancy_from_the_credential(): + """`connect("user-42")` introspects the token once and lands on the bound + org/environment — the seam `LodeDB.cloud("user-42")` rides.""" + transport = _transport( + { + "/v1/tokens/self": { + "kind": "secret", + "scopes": ["read:search", "write"], + "org": "acme", + "environment": "prod", + } + } + ) + store = connect( + "user-42", token="sk-test", host="https://api.test", warm=False, transport=transport + ) + try: + assert (store.org, store.environment, store.store) == ("acme", "prod", "user-42") + finally: + store.close() + + +def test_full_triple_makes_no_http_call(): + """A fully qualified target must not introspect — the transport rejects + every request, so connecting cold (warm=False) proves zero HTTP.""" + store = connect( + "acme/prod/user-42", + token="sk-test", + host="https://api.test", + warm=False, + transport=_transport({}), + ) + try: + assert (store.org, store.environment, store.store) == ("acme", "prod", "user-42") + finally: + store.close() + + +def test_bare_store_with_an_unbound_multi_org_token_refuses_with_choices(): + """A personal token spanning several orgs cannot pin a bare store id: + connect surfaces resolve_tenancy's 422 naming the choices (and must not + leak its freshly opened pool — closing after is a no-op either way).""" + transport = _transport( + { + "/v1/tokens/self": {"kind": "personal", "scopes": [], "org": None, "environment": None}, + "/v1/auth/me": {"orgs": [{"slug": "acme"}, {"slug": "umbrella"}]}, + } + ) + with pytest.raises(CloudError, match="several orgs"): + connect( + "user-42", token="pat", host="https://api.test", warm=False, transport=transport + ) + + +def test_empty_segments_are_malformed_not_reinterpreted(): + """`orecloud://org//store` must refuse: silently dropping the empty + segment would reparse it as the two-segment org/environment form and aim + the handle at the wrong tenancy.""" + for target in ("orecloud://acme//user-42", "acme//user-42"): + with pytest.raises(CloudError): + _parse_target(target) diff --git a/tests/test_cloud_env_file.py b/tests/test_cloud_env_file.py new file mode 100644 index 00000000..9e588644 --- /dev/null +++ b/tests/test_cloud_env_file.py @@ -0,0 +1,103 @@ +"""`lodedb.cloud._env_file`: dotenv writing that never clobbers, plus the +gitignore guard — the pieces that keep `tokens mint --env-file` and +`init --agents` from leaking or destroying anything.""" + +import stat +import sys + +from lodedb.cloud import _env_file + + +def _assert_owner_only(path): + """POSIX-only exact-mode check: Windows has no 0600 to assert (owner-only + there is an ACL property, which mkstemp's descriptor-level creation + already provides).""" + if sys.platform != "win32": + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + +def test_creates_missing_file_owner_only(tmp_path): + target = tmp_path / ".env" + _env_file.write_env_values(target, {"ORECLOUD_TOKEN": "ore_sk_abc"}) + assert target.read_text() == "ORECLOUD_TOKEN=ore_sk_abc\n" + _assert_owner_only(target) + + +def test_replaces_managed_keys_and_preserves_everything_else(tmp_path): + target = tmp_path / ".env" + target.write_text("# app config\nDATABASE_URL=postgres://x\nORECLOUD_TOKEN=old\n\nDEBUG=1\n") + _env_file.write_env_values(target, {"ORECLOUD_TOKEN": "new", "ORECLOUD_HOST": "https://c.example"}) + assert target.read_text() == ( + "# app config\nDATABASE_URL=postgres://x\nORECLOUD_TOKEN=new\n\nDEBUG=1\n" + "ORECLOUD_HOST=https://c.example\n" + ) + + +def test_existing_permissive_file_ends_up_owner_only(tmp_path): + """A pre-existing 0644 `.env` (the common default) must not keep the + minted token group/world-readable: the write always lands 0600.""" + import os + + target = tmp_path / ".env" + target.write_text("DEBUG=1\n") + os.chmod(target, 0o644) + _env_file.write_env_values(target, {"ORECLOUD_TOKEN": "ore_sk_abc"}) + _assert_owner_only(target) + assert target.read_text() == "DEBUG=1\nORECLOUD_TOKEN=ore_sk_abc\n" + + +def test_read_env_value_last_assignment_wins(tmp_path): + target = tmp_path / ".env" + target.write_text("ORECLOUD_TOKEN=first\nORECLOUD_TOKEN=second\n") + assert _env_file.read_env_value(target, "ORECLOUD_TOKEN") == "second" + assert _env_file.read_env_value(target, "MISSING") is None + assert _env_file.read_env_value(tmp_path / "absent", "ORECLOUD_TOKEN") is None + + +def test_gitignore_added_at_repo_root(tmp_path): + (tmp_path / ".git").mkdir() + note = _env_file.ensure_gitignored(tmp_path / ".env") + assert "added .env" in note + assert ".env\n" in (tmp_path / ".gitignore").read_text() + # Second call sees the fresh entry and does nothing. + assert _env_file.ensure_gitignored(tmp_path / ".env") is None + + +def test_gitignore_respects_existing_coverage_up_the_tree(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".gitignore").write_text("node_modules/\n.env\n") + sub = tmp_path / "service" + sub.mkdir() + assert _env_file.ensure_gitignored(sub / ".env") is None + assert not (sub / ".gitignore").exists() + + +def test_gitignore_appends_without_eating_the_last_line(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".gitignore").write_text("dist/") # no trailing newline + _env_file.ensure_gitignored(tmp_path / ".env") + assert (tmp_path / ".gitignore").read_text() == "dist/\n.env\n" + + +def test_outside_a_git_repo_notes_instead_of_writing(tmp_path): + note = _env_file.ensure_gitignored(tmp_path / ".env") + assert "not inside a git repository" in note + assert not (tmp_path / ".gitignore").exists() + + +def test_scratch_write_ignores_a_planted_symlink(tmp_path): + """The scratch file is unique and O_EXCL: a pre-planted predictable + `..tmp` symlink must never receive the secret (its target stays + untouched) and the final file is still owner-only.""" + import os + + target = tmp_path / ".env" + sentinel = tmp_path / "sentinel" + sentinel.write_text("untouched\n") + os.symlink(sentinel, tmp_path / "..env.tmp") + + _env_file.write_env_values(target, {"ORECLOUD_TOKEN": "ore_sk_abc"}) + + assert sentinel.read_text() == "untouched\n" + assert target.read_text() == "ORECLOUD_TOKEN=ore_sk_abc\n" + _assert_owner_only(target) diff --git a/tests/test_cloud_import_root.py b/tests/test_cloud_import_root.py new file mode 100644 index 00000000..e8516b5f --- /dev/null +++ b/tests/test_cloud_import_root.py @@ -0,0 +1,59 @@ +"""The `lodedb.cloud` import root: PEP 562 lazy resolution of the first-party +cloud client. HTTP-backed names (`Client`, `connect`, …) come from their +submodule with an install hint when the [cloud] extra's dependencies are +absent; the transfer verbs (`push`, `pull`, …) ride the bundled native +extension and need no extra at all. The laziness itself (importing +`lodedb.cloud` must not load httpx) is guarded by the fresh-subprocess probe +in `tests/test_import_boundary.py`.""" + +from __future__ import annotations + +import sys + +import pytest + +# The tests below import (or monkeypatch into) the client modules, which +# pull httpx/pynacl — skip cleanly without the [cloud] extra installed. +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb import _turbovec, cloud + + +def test_http_names_resolve_to_their_submodules(): + from lodedb.cloud.client import Client + from lodedb.cloud.serving import CloudStore, connect + + assert cloud.Client is Client + assert cloud.connect is connect + assert cloud.CloudStore is CloudStore + assert cloud.CloudIndex is CloudStore # back-compat alias + + +def test_transfer_verbs_resolve_to_the_native_extension(): + for name in ("keys", "pull", "push", "status", "sync", "verify"): + assert getattr(cloud, name) is getattr(_turbovec.cloud, name) + + +def test_missing_cloud_deps_raise_install_hint(monkeypatch): + # A None entry makes the submodule import raise ImportError even though + # the extra's dependencies are installed in the dev venv. + monkeypatch.setitem(sys.modules, "lodedb.cloud.client", None) + with pytest.raises(ImportError, match=r'pip install "lodedb\[cloud\]"'): + cloud.Client # noqa: B018 — the attribute fetch is the behavior under test + + +def test_unknown_names_stay_attribute_errors(): + """A name outside the public surface is a plain AttributeError, not a + misleading install hint.""" + with pytest.raises(AttributeError): + cloud.NoSuchName # noqa: B018 + + +def test_underscore_probes_never_touch_the_lazy_table(monkeypatch): + """Attribute machinery probes (`__path__`-style dunders) must resolve + locally — blocked submodules stay untouched.""" + monkeypatch.setitem(sys.modules, "lodedb.cloud.client", None) + monkeypatch.setitem(sys.modules, "lodedb.cloud.serving", None) + with pytest.raises(AttributeError): + cloud._private_probe # noqa: B018 diff --git a/tests/test_cloud_store_verbs.py b/tests/test_cloud_store_verbs.py new file mode 100644 index 00000000..e581f680 --- /dev/null +++ b/tests/test_cloud_store_verbs.py @@ -0,0 +1,80 @@ +"""CloudStore verb wiring against a stub transport: what payload each verb +puts on the wire and how it folds the acceptance back into session state +(read-your-writes floor, `last_write_id`). No server involved — the accepted +write contract itself is covered end-to-end in `server/tests`.""" + +from __future__ import annotations + +import pytest + +# Collection must skip, not error, without the [cloud] extra installed +# (the modules below import httpx / pynacl at module level). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb.cloud.serving import CloudStore # noqa: E402 + + +class _StubClient: + """Duck-types the one CloudClient call `remove_many` uses, recording the + payload and answering a canned acceptance.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, str, dict]] = [] + + def remove_documents(self, org: str, environment: str, payload: dict) -> dict: + self.calls.append((org, environment, payload)) + return {"write_id": "w-1", "seq": 7} + + +def _store(client: _StubClient) -> CloudStore: + """A CloudStore over the stub, addressed like a real handle.""" + return CloudStore(client, "acme", "prod", "user-42", owns_client=False) + + +def test_remove_many_sends_one_batch_and_records_the_acceptance(): + client = _StubClient() + store = _store(client) + + write_id = store.remove_many(["a", "b"]) + + assert write_id == "w-1" + assert store.last_write_id == "w-1" + (org, environment, payload) = client.calls[0] + assert (org, environment) == ("acme", "prod") + assert payload["ids"] == ["a", "b"] + assert payload["store"] == "user-42" + # Every write carries an idempotency key so a transport retry replays the + # original acceptance instead of registering a duplicate segment. + assert payload["idempotency_key"] + + +def test_remove_delegates_to_the_batch_verb(): + client = _StubClient() + store = _store(client) + + assert store.remove("a") == "w-1" + assert client.calls[0][2]["ids"] == ["a"] + + +def test_remove_many_rejects_an_empty_batch(): + store = _store(_StubClient()) + with pytest.raises(ValueError, match="nothing to remove"): + store.remove_many([]) + + +class _UnprovisionedClient: + """Answers every search with the 404 a not-yet-provisioned store gives.""" + + def search_many(self, org: str, environment: str, payload: dict) -> dict: + from lodedb.cloud.transfer import CloudError + + raise CloudError(404, "no such store") + + +def test_unprovisioned_batch_verbs_keep_query_cardinality(): + """Both batched search verbs answer an unprovisioned store with one empty + hit list PER query — callers zip queries to results.""" + store = CloudStore(_UnprovisionedClient(), "acme", "prod", "user-42", owns_client=False) + assert store.search_many(["a", "b", "c"]) == [[], [], []] + assert store.search_many_by_vector([[0.1, 0.2], [0.3, 0.4]]) == [[], []] diff --git a/tests/test_cloud_surface_parity.py b/tests/test_cloud_surface_parity.py new file mode 100644 index 00000000..c2718ac2 --- /dev/null +++ b/tests/test_cloud_surface_parity.py @@ -0,0 +1,107 @@ +"""Executable audit of the LodeDB-surface contract: `CloudStore` duck-types +the local `lodedb.LodeDB` verb surface (lodedb's `LodeDB("orecloud://…")` +front door returns one), so every shared verb must keep a matching call +shape. Two kinds of drift are allowed and named here: cloud-only keywords +are additive (multi-tenant provenance, TTL, text exposure, serving warm-up), +and a few local keywords have no cloud meaning (the server always +unit-normalizes vectors; there is no server-side filtered count yet). +Anything outside those allowances fails this suite before it ships.""" + +from __future__ import annotations + +import inspect + +import pytest + +# Collection must skip, not error, without the [cloud] extra installed +# (the modules below import httpx / pynacl at module level). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb.cloud.serving import CloudStore # noqa: E402 + +LodeDB = pytest.importorskip("lodedb").LodeDB + +# Every verb the two handles share. `get_text` is the documented synonym of +# `get` on both sides; `remove`/`remove_many` are async-first on the cloud +# (they return a write id, not a bool/count) but take the same arguments. +SHARED_VERBS = [ + "add", + "add_many", + "add_vectors", + "add_vectors_many", + "search", + "search_many", + "search_by_vector", + "search_many_by_vector", + "get", + "get_text", + "get_texts", + "remove", + "remove_many", + "count", + "stats", + "close", +] + +# Keyword-only params only the cloud has: write provenance and retention +# (`ttl_seconds`/`agent_id`/`run_id`), inline text on hits (`include_text`), +# and serving warm-up (`warm`). All additive — code written against the +# shared shape runs unchanged. +CLOUD_ONLY_KEYWORDS = {"ttl_seconds", "agent_id", "run_id", "include_text", "warm"} + +# Keyword-only params only the local handle has, per verb. `normalize` does +# not exist on the cloud because the server unit-normalizes every vector +# (the local default); `count(filter=)` needs a server-side filtered count +# the API does not offer yet. +LOCAL_ONLY_KEYWORDS = { + "add_vectors": {"normalize"}, + "add_vectors_many": {"normalize"}, + "search_by_vector": {"normalize"}, + "search_many_by_vector": {"normalize"}, + "count": {"filter"}, +} + + +def _shape(method) -> tuple[list[str], set[str]]: + """A method's call shape: ordered positional-or-keyword names (sans self) + and the set of keyword-only names.""" + parameters = inspect.signature(method).parameters.values() + positional = [ + parameter.name + for parameter in parameters + if parameter.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + and parameter.name != "self" + ] + keyword_only = { + parameter.name + for parameter in parameters + if parameter.kind is inspect.Parameter.KEYWORD_ONLY + } + return positional, keyword_only + + +@pytest.mark.parametrize("verb", SHARED_VERBS) +def test_shared_verb_keeps_the_local_call_shape(verb): + """CloudStore's version of each shared verb accepts the local call shape: + identical positional arguments, and keyword drift only within the named + cloud-only / local-only allowances.""" + local_positional, local_keywords = _shape(getattr(LodeDB, verb)) + cloud_positional, cloud_keywords = _shape(getattr(CloudStore, verb)) + + assert cloud_positional == local_positional, ( + f"{verb}: positional shape drifted — local {local_positional}, " + f"cloud {cloud_positional}" + ) + unexplained_cloud = cloud_keywords - local_keywords - CLOUD_ONLY_KEYWORDS + assert not unexplained_cloud, ( + f"{verb}: cloud grew keywords outside the additive allowance: " + f"{sorted(unexplained_cloud)}" + ) + unexplained_local = ( + local_keywords - cloud_keywords - LOCAL_ONLY_KEYWORDS.get(verb, set()) + ) + assert not unexplained_local, ( + f"{verb}: cloud is missing local keywords with no recorded reason: " + f"{sorted(unexplained_local)}" + ) diff --git a/tests/test_cloud_sync.py b/tests/test_cloud_sync.py new file mode 100644 index 00000000..32193ac3 --- /dev/null +++ b/tests/test_cloud_sync.py @@ -0,0 +1,242 @@ +"""Sync tests through the real engine: the `lodedb cloud sync` CLI verb and the +`lodedb.cloud.sync`/`lodedb.cloud.status` lineage surface, driven by actual LodeDB +commits (so epochs, generation numbers, and the epoch GC behave exactly as a +user's database does).""" + +import json + +import pytest +from typer.testing import CliRunner + +from lodedb import cloud + +# Collection must skip, not error, without the [cloud] extra installed +# (the modules below import httpx / pynacl at module level). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb.cloud.cli import app # noqa: E402 +from lodedb.engine.embedding_backends import HashEmbeddingBackend # noqa: E402 +from lodedb.local.db import LodeDB # noqa: E402 + +runner = CliRunner() + + +def add_documents(path, documents) -> None: + """One further committed generation in `path` through the real engine.""" + db = LodeDB( + path=path, + model="minilm", + commit_mode="generation", + _embedding_backend=HashEmbeddingBackend(native_dim=384), + ) + try: + db.add_many(documents) + finally: + db.close() + + +def docs(tag: str, count: int = 2): + return [{"text": f"{tag} document {i}", "id": f"{tag}-{i}"} for i in range(count)] + + +def test_sync_lifecycle_two_clones(committed_store, tmp_path): + source, key = committed_store + remote = tmp_path / "remote" + clone = tmp_path / "clone" + clone.mkdir() + + # Fresh push. + result = runner.invoke(app, ["sync", str(source), str(remote), key]) + assert result.exit_code == 0 + outcome = json.loads(result.output) + assert outcome["classification"] == "local_ahead" + assert outcome["action"] == "push" + assert (source / f"{key}.orecloud").exists() + + # Converged: a second sync moves nothing. + result = runner.invoke(app, ["sync", str(source), str(remote), key]) + assert result.exit_code == 0 + assert "action" in result.output and "none" in result.output + + # A new commit fast-forwards. + add_documents(source, docs("second")) + report = cloud.status(str(source), str(remote), key) + assert report["sidecar_present"] + assert report["classification"] == "local_ahead" + result = runner.invoke(app, ["sync", str(source), str(remote), key]) + assert result.exit_code == 0 + assert "push" in result.output + + # A second clone pulls (and the pull verify-opens: counts are reported). + result = runner.invoke(app, ["sync", str(clone), str(remote), key]) + assert result.exit_code == 0 + outcome = json.loads(result.output) + assert outcome["classification"] == "remote_ahead" + assert "document_count" in outcome + + +def test_diverged_requires_force_and_force_resolves(committed_store, tmp_path): + source, key = committed_store + remote = tmp_path / "remote" + clone = tmp_path / "clone" + clone.mkdir() + + runner.invoke(app, ["sync", str(source), str(remote), key]) + runner.invoke(app, ["sync", str(clone), str(remote), key]) + + # The clone advances once and publishes; the source advances twice on its + # own lineage. (Different commit counts keep the two lineages on different + # base epochs — same-epoch artifact-name collisions across divergent + # lineages are the fork-collision case the Phase-2 content-addressed + # layout absorbs; a dumb remote fails them closed at the + # immutable-artifact check.) + add_documents(clone, docs("clone-side")) + result = runner.invoke(app, ["sync", str(clone), str(remote), key]) + assert result.exit_code == 0 + add_documents(source, docs("source-side-a")) + add_documents(source, docs("source-side-b")) + + # Diverged: sync refuses, names the classification and both force flags, + # and exits with the documented "refused" class (5) — the same class the + # managed sync's 409 maps to, not the generic 1. + result = runner.invoke(app, ["sync", str(source), str(remote), key]) + assert result.exit_code == 5 + assert "diverged" in result.output + assert "--force-push" in result.output and "--force-pull" in result.output + + # The API surface raises the same refusal. + with pytest.raises(RuntimeError, match="diverged"): + cloud.sync(str(source), str(remote), key) + + # Both force flags at once is a usage error, not a transfer. + result = runner.invoke( + app, ["sync", str(source), str(remote), key, "--force-push", "--force-pull"] + ) + assert result.exit_code == 2 + + # Force-push keeps the source's lineage; the clone then fast-forwards. + result = runner.invoke(app, ["sync", str(source), str(remote), key, "--force-push"]) + assert result.exit_code == 0 + assert json.loads(result.output)["forced"] is True + result = runner.invoke(app, ["sync", str(clone), str(remote), key]) + assert result.exit_code == 0 + assert json.loads(result.output)["classification"] == "remote_ahead" + + +def test_sidecar_survives_engine_epoch_gc(committed_store, tmp_path): + """The engine's local epoch GC must never eat the sync base: after enough + commits to cycle the retained-epoch window, the sidecar still classifies + the directory as trusted local_ahead (an eaten/corrupt sidecar would read + as unknown).""" + source, key = committed_store + remote = tmp_path / "remote" + runner.invoke(app, ["sync", str(source), str(remote), key]) + + for round_ in range(4): + add_documents(source, docs(f"round-{round_}")) + + assert (source / f"{key}.orecloud").exists() + report = cloud.status(str(source), str(remote), key) + assert report["sidecar_present"] + assert report["classification"] == "local_ahead" + + # And the recorded base still fast-forwards. + outcome = cloud.sync(str(source), str(remote), key) + assert outcome["action"] == "push" + assert outcome["classification"] == "local_ahead" + + +def test_sidecar_is_invisible_to_the_engine(committed_store, tmp_path): + """The engine's load path treats unknown top-level `*.json` files as legacy + index snapshots, so the sidecar must not look like one: after a sync, the + directory must still open (and commit) as a plain LodeDB.""" + source, key = committed_store + remote = tmp_path / "remote" + result = runner.invoke(app, ["sync", str(source), str(remote), key]) + assert result.exit_code == 0 + assert (source / f"{key}.orecloud").exists() + + # Reopen through the engine and commit — this is exactly what broke when + # the sidecar was named `.orecloud.json`. + add_documents(source, docs("after-sync")) + (found_key,) = cloud.keys(str(source)) + assert found_key == key + + +def test_status_lineage_fields_without_a_sidecar(committed_store, tmp_path): + """A never-synced directory reports its lineage as untrusted, not a guess.""" + source, key = committed_store + remote = tmp_path / "remote" + + report = cloud.status(str(source), str(remote), key) + assert not report["sidecar_present"] + assert report["base_generation"] is None + # Local exists, remote absent: direction is unambiguous even without a base. + assert report["classification"] == "local_ahead" + + # Push via the plain verb also records the base (push and sync share it). + cloud.push(str(source), str(remote), key) + report = cloud.status(str(source), str(remote), key) + assert report["sidecar_present"] + assert report["classification"] == "in_sync" + + +def test_rescore_tvvf_store_round_trips(tmp_path): + """A rescore-enabled store's committed generation pins the tvvf sidecar + under the engine's own naming (`vf.tvvf`, an epoch counter + independent of the generation's base_epoch). Push must ship it, pull must + restore a copy the engine reopens (it refuses a rescore store without its + sidecar), and searches must work on the restored copy.""" + from conftest import read_pointer_body + + source = tmp_path / "source" + remote = tmp_path / "remote" + restored = tmp_path / "restored" + db = LodeDB.open_vector_store( + source, vector_dim=8, commit_mode="generation", rescore="original" + ) + try: + db.add_vectors_many( + [ + { + "vector": [1.0 if j == i else 0.1 for j in range(8)], + "id": f"doc-{i}", + "text": f"document {i}", + } + for i in range(4) + ] + ) + finally: + db.close() + (key,) = cloud.keys(str(source)) + assert read_pointer_body(source, key)["tvvf"], "a rescore store must pin its sidecar" + + result = cloud.push(str(source), str(remote), key) + assert result["pointer_published"] + shipped = {p.name for p in (remote / f"{key}.gen").iterdir()} + assert any(name.startswith("vf") and name.endswith(".tvvf") for name in shipped), shipped + + outcome = cloud.pull(str(remote), str(restored), key) + assert outcome["document_count"] == 4 + + db = LodeDB.open_vector_store(restored, vector_dim=8) + try: + hits = db.search_by_vector([1.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], k=2) + assert hits and hits[0].id == "doc-0" + finally: + db.close() + + +def test_engine_kind_wire_map_covers_every_pushable_kind(): + """The managed wire contract labels every blob by kind; an engine kind the + map does not know must fail loudly (never silently ship as \"state\"), and + every kind the local inventory can emit must be mapped.""" + from lodedb.cloud.transfer import _ENGINE_KIND_TO_WIRE, CloudError, _wire_kind + + for kind in ("json", "tvim", "tvtext", "tvlex", "tvmv", "tvann", "tvvf"): + assert kind in _ENGINE_KIND_TO_WIRE, kind + for vector_kind in ("tvim", "tvann", "tvvf"): + assert _ENGINE_KIND_TO_WIRE[vector_kind] == "vector" + with pytest.raises(CloudError, match="tvfuture"): + _wire_kind("tvfuture") diff --git a/tests/test_cloud_target_dispatch.py b/tests/test_cloud_target_dispatch.py new file mode 100644 index 00000000..d12097d0 --- /dev/null +++ b/tests/test_cloud_target_dispatch.py @@ -0,0 +1,158 @@ +"""The two constructor front doors to a managed-cloud store: `LodeDB.cloud()` +(the human-facing form, accepting a bare store id) and the +`LodeDB("orecloud://…")` config-string dispatch (one string field expressing +either a local path or a cloud store). Both funnel through +`lodedb.cloud.open_cloud_target`: a lazy import of the first-party client +with a clear install hint when the [cloud] extra's dependencies are absent, +targeted rejection of local-only options, and every other target constructing +the local database exactly as before. The client's `connect` is faked (or its +module blocked), so the suite runs without any network.""" + +from __future__ import annotations + +import sys + +import pytest + +# The tests below import (or monkeypatch into) the client modules, which +# pull httpx/pynacl — skip cleanly without the [cloud] extra installed. +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +from lodedb import LodeDB + +CLOUD_TARGET = "orecloud://acme/prod/user-42" + + +def _fake_connect(monkeypatch, captured: dict, handle: object): + """Replaces `lodedb.cloud.serving.connect` with a recorder mirroring its + real keyword surface; the funnel re-fetches the attribute per call.""" + + def connect( + target: str, + *, + token: str | None = None, + host: str | None = None, + key: str | None = None, + warm: bool = True, + timeout: float = 30.0, + read_your_writes: bool = True, + transport: object | None = None, + ) -> object: + captured["target"] = target + captured["token"] = token + captured["warm"] = warm + return handle + + monkeypatch.setattr("lodedb.cloud.serving.connect", connect) + + +def test_missing_cloud_deps_raise_install_hint(monkeypatch): + # A None entry makes `import lodedb.cloud.serving` raise ImportError even + # though the [cloud] extra's dependencies are installed in the dev venv. + monkeypatch.setitem(sys.modules, "lodedb.cloud.serving", None) + with pytest.raises(ImportError, match=r'pip install "lodedb\[cloud\]"'): + LodeDB(CLOUD_TARGET) + + +def test_cloud_target_returns_the_client_handle(monkeypatch): + """A cloud target hands back whatever the client's connect() opens — + LodeDB.__init__ never runs on it — with the options forwarded.""" + captured: dict[str, object] = {} + handle = object() + _fake_connect(monkeypatch, captured, handle) + + db = LodeDB(CLOUD_TARGET, token="tok-1", warm=False) + + assert db is handle + assert not isinstance(db, LodeDB) + assert captured == {"target": CLOUD_TARGET, "token": "tok-1", "warm": False} + + +def test_cloud_target_accepts_path_keyword(monkeypatch): + """`LodeDB(path="orecloud://…")` (the documented keyword spelling) + dispatches too, and the `path` keyword itself is not forwarded.""" + captured: dict[str, object] = {} + handle = object() + _fake_connect(monkeypatch, captured, handle) + + assert LodeDB(path=CLOUD_TARGET) is handle + assert captured["target"] == CLOUD_TARGET + + +def test_local_only_options_are_rejected_up_front(monkeypatch): + """Local construction options have no cloud meaning: the error names the + offending keyword and lists what cloud targets do accept.""" + _fake_connect(monkeypatch, {}, object()) + + with pytest.raises(TypeError, match=r"do not accept model.*available options.*token"): + LodeDB(CLOUD_TARGET, model="minilm") + + +def test_cloud_classmethod_accepts_a_bare_store_id(monkeypatch): + """`LodeDB.cloud("user-42")` forwards the short target verbatim — the + client resolves org/environment from the credential — and returns the + handle through the same funnel as the config-string form.""" + captured: dict[str, object] = {} + handle = object() + _fake_connect(monkeypatch, captured, handle) + + db = LodeDB.cloud("user-42", token="tok-1") + + assert db is handle + assert captured["target"] == "user-42" + assert captured["token"] == "tok-1" + + +def test_cloud_classmethod_accepts_the_url_form(monkeypatch): + """The classmethod takes the full triple and the orecloud:// spelling too.""" + captured: dict[str, object] = {} + handle = object() + _fake_connect(monkeypatch, captured, handle) + + assert LodeDB.cloud(CLOUD_TARGET) is handle + assert captured["target"] == CLOUD_TARGET + + +def test_cloud_classmethod_missing_deps_raise_install_hint(monkeypatch): + monkeypatch.setitem(sys.modules, "lodedb.cloud.serving", None) + with pytest.raises(ImportError, match=r'pip install "lodedb\[cloud\]"'): + LodeDB.cloud("user-42") + + +def test_cloud_classmethod_rejects_local_only_options(monkeypatch): + _fake_connect(monkeypatch, {}, object()) + with pytest.raises(TypeError, match=r"do not accept model"): + LodeDB.cloud("user-42", model="minilm") + + +def test_bare_store_ids_do_not_dispatch_from_the_constructor(tmp_path, monkeypatch): + """Only `LodeDB.cloud()` accepts scheme-less short forms: a bare id passed + to the constructor is an ordinary (relative) local path, never a cloud + target — so the client's connect must not be consulted.""" + + def _exploding_connect(target, **options): + raise AssertionError("the constructor must not dispatch scheme-less targets") + + monkeypatch.setattr("lodedb.cloud.serving.connect", _exploding_connect) + monkeypatch.chdir(tmp_path) + db = LodeDB("user-42", vector_dim=8) + try: + assert isinstance(db, LodeDB) + finally: + db.close() + + +def test_local_paths_still_construct_locally(tmp_path): + """The dispatch leaves ordinary construction untouched: a filesystem path + (str or Path) yields a real local LodeDB instance.""" + db = LodeDB(tmp_path / "store", vector_dim=8) + try: + assert isinstance(db, LodeDB) + finally: + db.close() + db = LodeDB(str(tmp_path / "store"), vector_dim=8) + try: + assert isinstance(db, LodeDB) + finally: + db.close() diff --git a/tests/test_cloud_trampoline.py b/tests/test_cloud_trampoline.py new file mode 100644 index 00000000..783e1f1b --- /dev/null +++ b/tests/test_cloud_trampoline.py @@ -0,0 +1,64 @@ +"""The `lodedb cloud` trampoline: forwards argv to the first-party cloud CLI +(`lodedb.cloud.cli`, whose modules pull the [cloud] extra's dependencies), +and answers a clear install hint — not a traceback — when those dependencies +are absent. Both paths are forced deterministically, so the suite never +depends on the venv's extras.""" + +from __future__ import annotations + +import sys + +import pytest +import typer +from typer.testing import CliRunner + +from lodedb.local.cli import app + +# The forwarding test monkeypatches into `lodedb.cloud.cli`, which pulls +# httpx/pynacl — skip cleanly without the [cloud] extra installed (the hint +# path blocks the module itself, so it needs no deps either way). +pytest.importorskip("httpx", reason="needs the [cloud] extra's dependencies") +pytest.importorskip("nacl", reason="needs the [cloud] extra's dependencies") + +runner = CliRunner() + + +def _all_output(result) -> str: + """stdout + stderr regardless of how this click version splits them.""" + output = result.output + try: + output += result.stderr + except (ValueError, AttributeError): + pass + return output + + +def test_missing_cloud_deps_print_install_hint(monkeypatch): + # A None entry makes `import lodedb.cloud.cli` raise ImportError even + # though the extra's dependencies are installed in the dev venv. + monkeypatch.setitem(sys.modules, "lodedb.cloud.cli", None) + result = runner.invoke(app, ["cloud", "sync", "./somewhere"]) + assert result.exit_code == 1 + assert 'pip install "lodedb[cloud]"' in _all_output(result) + + +def test_argv_forwards_to_the_cloud_cli(monkeypatch): + """Everything after `lodedb cloud` reaches the cloud app untouched — + subcommand, positionals, and options the trampoline knows nothing about.""" + captured: dict[str, object] = {} + fake_app = typer.Typer() + + @fake_app.command("sync") + def sync(target: str, note: str = typer.Option("", "--note")) -> None: + captured["target"] = target + captured["note"] = note + + @fake_app.command("status") + def status() -> None: # second command keeps the fake app a group + captured["status"] = True + + monkeypatch.setattr("lodedb.cloud.cli.app", fake_app) + + result = runner.invoke(app, ["cloud", "sync", "./my-store", "--note", "hi"]) + assert result.exit_code == 0, _all_output(result) + assert captured == {"target": "./my-store", "note": "hi"} diff --git a/tests/test_import_boundary.py b/tests/test_import_boundary.py index 954db3d4..36dfbd54 100644 --- a/tests/test_import_boundary.py +++ b/tests/test_import_boundary.py @@ -57,11 +57,16 @@ # imports function-local; none of these may load on a plain import. _OPTIONAL_INTEGRATION_PROBE = """ import importlib, sys -for _m in ("lodedb", "lodedb.local.cli"): +for _m in ("lodedb", "lodedb.local.cli", "lodedb.cloud"): importlib.import_module(_m) _FORBIDDEN = ( "langchain", "langchain_core", "llama_index", "mem0", "cognee", "psycopg", "psycopg2", "asyncpg", "qdrant_client", "chromadb", "lancedb", + # The [cloud] extra's dependencies (httpx; pynacl imports as `nacl`): the + # first-party cloud client (lodedb.cloud) reaches them only through its + # lazy PEP 562 exports and the CLI trampoline, so a plain import — even of + # lodedb.cloud itself — must stay network-free. + "httpx", "nacl", ) _loaded = {_name.split(".", 1)[0] for _name in sys.modules} for _root in _FORBIDDEN: diff --git a/third_party/turbovec/Cargo.lock b/third_party/turbovec/Cargo.lock index 0e65c443..116c16c2 100644 --- a/third_party/turbovec/Cargo.lock +++ b/third_party/turbovec/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "approx" version = "0.5.1" @@ -11,12 +20,35 @@ dependencies = [ "num-traits", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.11.0" @@ -32,6 +64,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytemuck" version = "1.25.0" @@ -58,6 +96,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cblas-sys" version = "0.1.4" @@ -85,12 +129,57 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "coe-rs" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -100,6 +189,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -166,6 +264,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dyn-stack" version = "0.11.0" @@ -249,6 +358,12 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -256,7 +371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -281,7 +396,7 @@ dependencies = [ "num-complex", "num-traits", "paste", - "rand", + "rand 0.8.5", "rand_distr", "rayon", "reborrow", @@ -303,12 +418,121 @@ dependencies = [ "reborrow", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "gemm" version = "0.18.2" @@ -451,8 +675,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -463,10 +689,43 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -480,6 +739,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -487,170 +752,498 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "indoc" -version = "2.0.7" +name = "http" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ - "rustversion", + "bytes", + "itoa", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "http-body" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] [[package]] -name = "jobserver" -version = "0.1.34" +name = "http-body-util" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ - "getrandom 0.3.4", - "libc", + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", ] [[package]] -name = "libc" -version = "0.2.184" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "libloading" -version = "0.9.0" +name = "humantime" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ - "cfg-if", - "windows-link", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", ] [[package]] -name = "libm" -version = "0.2.16" +name = "hyper-rustls" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] [[package]] -name = "linux-raw-sys" -version = "0.12.1" +name = "hyper-util" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] [[package]] -name = "lodedb-core" -version = "1.3.0" +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ - "lodedb-gpu", - "rayon", - "rustix", - "serde", - "serde_json", - "sha2", - "turbovec", - "zstd", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", ] [[package]] -name = "lodedb-gpu" -version = "1.3.0" +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cudarc", + "cc", ] [[package]] -name = "matrixcompare" -version = "0.3.0" +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "matrixcompare-core", - "num-traits", + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "matrixcompare-core" -version = "0.1.0" +name = "icu_locale_core" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] [[package]] -name = "matrixmultiply" -version = "0.3.10" +name = "icu_normalizer" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ - "autocfg", - "rawpointer", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", ] [[package]] -name = "memchr" -version = "2.8.0" +name = "icu_normalizer_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] -name = "memoffset" -version = "0.9.1" +name = "icu_properties" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "autocfg", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] [[package]] -name = "nalgebra" -version = "0.32.6" +name = "icu_properties_data" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ - "approx", - "matrixmultiply", - "nalgebra-macros", - "num-complex", - "num-rational", - "num-traits", - "rand", - "rand_distr", - "simba", - "typenum", + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", ] [[package]] -name = "nalgebra-macros" -version = "0.2.2" +name = "idna" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "proc-macro2", - "quote", - "syn", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "nano-gemm" -version = "0.1.3" +name = "idna_adapter" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb5ba2bea1c00e53de11f6ab5bd0761ba87dc0045d63b0c87ee471d2d3061376" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ - "equator 0.2.2", - "nano-gemm-c32", - "nano-gemm-c64", - "nano-gemm-codegen", - "nano-gemm-core", - "nano-gemm-f32", - "nano-gemm-f64", - "num-complex", + "icu_normalizer", + "icu_properties", ] [[package]] -name = "nano-gemm-c32" -version = "0.1.0" +name = "indexmap" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40449e57a5713464c3a1208c4c3301c8d29ee1344711822cf022bc91373a91b" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lodedb-cloud-core" +version = "1.3.2" +dependencies = [ + "lodedb-core", + "object_store", + "serde_json", + "sha2", + "tempfile", + "tokio", +] + +[[package]] +name = "lodedb-core" +version = "1.3.2" +dependencies = [ + "lodedb-gpu", + "rayon", + "rustix", + "serde", + "serde_json", + "sha2", + "turbovec", + "zstd", +] + +[[package]] +name = "lodedb-gpu" +version = "1.3.2" +dependencies = [ + "cudarc", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matrixcompare" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37832ba820e47c93d66b4360198dccb004b43c74abc3ac1ce1fed54e65a80445" +dependencies = [ + "matrixcompare-core", + "num-traits", +] + +[[package]] +name = "matrixcompare-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nalgebra" +version = "0.32.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +dependencies = [ + "approx", + "matrixmultiply", + "nalgebra-macros", + "num-complex", + "num-rational", + "num-traits", + "rand 0.8.5", + "rand_distr", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nano-gemm" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5ba2bea1c00e53de11f6ab5bd0761ba87dc0045d63b0c87ee471d2d3061376" +dependencies = [ + "equator 0.2.2", + "nano-gemm-c32", + "nano-gemm-c64", + "nano-gemm-codegen", + "nano-gemm-core", + "nano-gemm-f32", + "nano-gemm-f64", + "num-complex", +] + +[[package]] +name = "nano-gemm-c32" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40449e57a5713464c3a1208c4c3301c8d29ee1344711822cf022bc91373a91b" dependencies = [ "nano-gemm-codegen", "nano-gemm-core", @@ -746,7 +1339,7 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "bytemuck", "num-traits", - "rand", + "rand 0.8.5", ] [[package]] @@ -794,12 +1387,54 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.5", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "ordered-float" version = "4.6.0" @@ -809,12 +1444,41 @@ dependencies = [ "num-traits", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pest" version = "2.8.6" @@ -858,6 +1522,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.33" @@ -879,6 +1549,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -998,13 +1677,79 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.45" +name = "quick-xml" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ - "proc-macro2", -] + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] [[package]] name = "r-efi" @@ -1012,6 +1757,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -1019,8 +1770,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1030,7 +1802,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1042,6 +1824,21 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.4.3" @@ -1049,7 +1846,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand", + "rand 0.8.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -1093,6 +1899,71 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1109,7 +1980,54 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1118,6 +2036,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "safe_arch" version = "0.7.4" @@ -1136,6 +2060,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "seq-macro" version = "0.3.6" @@ -1185,6 +2147,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1192,7 +2166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1215,6 +2189,34 @@ dependencies = [ "wide", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "statrs" version = "0.17.1" @@ -1224,9 +2226,15 @@ dependencies = [ "approx", "nalgebra", "num-traits", - "rand", + "rand 0.8.5", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -1238,6 +2246,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sysctl" version = "0.6.0" @@ -1248,7 +2276,7 @@ dependencies = [ "byteorder", "enum-as-inner", "libc", - "thiserror", + "thiserror 1.0.69", "walkdir", ] @@ -1258,13 +2286,35 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1278,6 +2328,173 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "turbovec" version = "0.9.0" @@ -1285,8 +2502,8 @@ dependencies = [ "faer", "ndarray", "ordered-float", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_distr", "rayon", "statrs", @@ -1296,6 +2513,7 @@ dependencies = [ name = "turbovec-python" version = "0.8.0" dependencies = [ + "lodedb-cloud-core", "lodedb-core", "numpy", "pyo3", @@ -1329,6 +2547,30 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.5" @@ -1345,6 +2587,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1360,6 +2611,94 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wide" version = "0.7.33" @@ -1376,7 +2715,42 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1385,6 +2759,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1394,12 +2795,105 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -1420,6 +2914,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/third_party/turbovec/turbovec-python/Cargo.toml b/third_party/turbovec/turbovec-python/Cargo.toml index 7ba1476c..4affe93c 100644 --- a/third_party/turbovec/turbovec-python/Cargo.toml +++ b/third_party/turbovec/turbovec-python/Cargo.toml @@ -10,6 +10,11 @@ crate-type = ["cdylib"] [dependencies] lodedb-core = { path = "../../../crates/lodedb-core" } +# The managed-cloud transfer plane, exposed as the `cloud` submodule of the +# bundled extension (see src/cloud.rs). Compiled in unconditionally: pip extras +# can only add other distributions, never gate compiled contents, and the +# [cloud] extra's Python client needs these ops without a second wheel. +lodedb-cloud-core = { path = "../../../crates/lodedb-cloud-core" } turbovec-core = { package = "turbovec", path = "../turbovec" } pyo3 = { version = "0.27.0", features = ["extension-module", "abi3-py39"] } numpy = "0.27.0" diff --git a/third_party/turbovec/turbovec-python/src/cloud.rs b/third_party/turbovec/turbovec-python/src/cloud.rs new file mode 100644 index 00000000..d0d46ea8 --- /dev/null +++ b/third_party/turbovec/turbovec-python/src/cloud.rs @@ -0,0 +1,455 @@ +//! `lodedb._turbovec.cloud`: the Python face of the managed-cloud transfer plane. +//! +//! A pure translator over `lodedb_cloud_core::client_ops` — every function here +//! converts Python arguments in, calls the corresponding (Rust-tested) client +//! operation, and converts the report out as a plain dict. No logic lives at +//! this seam. The dict fields are mapped by hand, deliberately: a struct field +//! rename then fails to compile here instead of silently changing a dict key +//! under the CLI. +//! +//! Errors map onto stdlib exceptions: a missing artifact or generation raises +//! `FileNotFoundError`, a filesystem failure raises `OSError`, and every +//! integrity/conflict/backend failure raises `RuntimeError` with the library's +//! own diagnostic message. +//! +//! The GIL is released around every operation (`detach`) because a push +//! or pull can spend seconds-to-minutes in network and disk I/O. + +use lodedb_cloud_core::generation_inventory::ArtifactRef; +use lodedb_cloud_core::{client_ops, managed}; +use lodedb_cloud_core::{ + ArtifactStoreError, StatusReport, SyncForce, SyncOutcome, TransferPolicy, TransferResult, + VerifyReport, +}; +use pyo3::exceptions::{PyFileNotFoundError, PyOSError, PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList}; + +pyo3::create_exception!( + cloud, + SyncConflictError, + PyRuntimeError, + "A sync/restore refused because resolving it needs an explicit decision \ + (diverged/unknown lineage, or a destination WAL holding acknowledged \ + writes) — re-run with a force flag or checkpoint first. Subclasses \ + RuntimeError, so pre-existing handlers keep working." +); + +/// Maps a library error onto the matching stdlib Python exception. +/// +/// `SyncConflict` and `PendingWal` get the dedicated `SyncConflictError` (a +/// RuntimeError subclass): both are refusals the caller resolves with a force +/// flag or a checkpoint, and the CLI maps them to its "refused" exit class +/// instead of "unexpected". +fn to_py_err(error: ArtifactStoreError) -> PyErr { + match &error { + ArtifactStoreError::NotFound(_) => PyFileNotFoundError::new_err(error.to_string()), + ArtifactStoreError::Io(_) => PyOSError::new_err(error.to_string()), + ArtifactStoreError::SyncConflict { .. } | ArtifactStoreError::PendingWal { .. } => { + SyncConflictError::new_err(error.to_string()) + } + _ => PyRuntimeError::new_err(error.to_string()), + } +} + +/// Builds a `TransferPolicy` from the two CLI-level opt-in flags. +fn policy(include_text: bool, include_lexical: bool) -> TransferPolicy { + TransferPolicy { + include_text, + include_lexical, + } +} + +/// Renders a `TransferResult` as a dict. +fn transfer_dict<'py>(py: Python<'py>, result: &TransferResult) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("index_key", &result.index_key)?; + dict.set_item("generation", result.generation)?; + dict.set_item("artifacts_written", result.artifacts_written)?; + dict.set_item("artifacts_skipped", result.artifacts_skipped)?; + dict.set_item("bytes_written", result.bytes_written)?; + dict.set_item("pointer_published", result.pointer_published)?; + Ok(dict) +} + +/// Renders a `StatusReport` as a dict (absent sides become `None`). +fn status_dict<'py>(py: Python<'py>, report: &StatusReport) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("index_key", &report.index_key)?; + dict.set_item("local_generation", report.local_generation)?; + dict.set_item("remote_generation", report.remote_generation)?; + dict.set_item("local_document_count", report.local_document_count)?; + dict.set_item("remote_document_count", report.remote_document_count)?; + dict.set_item("local_chunk_count", report.local_chunk_count)?; + dict.set_item("remote_chunk_count", report.remote_chunk_count)?; + dict.set_item("artifacts_to_upload", report.artifacts_to_upload)?; + dict.set_item("bytes_to_upload", report.bytes_to_upload)?; + dict.set_item("ships_base", report.ships_base)?; + dict.set_item("in_sync", report.in_sync)?; + dict.set_item("sidecar_present", report.sidecar_present)?; + dict.set_item("sidecar_corrupt", report.sidecar_corrupt)?; + dict.set_item("base_generation", report.base_generation)?; + dict.set_item("classification", report.classification.as_deref())?; + Ok(dict) +} + +/// Renders a `SyncOutcome` as one flat dict: the decision fields, then the +/// transfer metrics when data moved, then the opened counts for a pull. +fn sync_dict<'py>(py: Python<'py>, outcome: &SyncOutcome) -> PyResult> { + let dict = match &outcome.transfer { + Some(transfer) => transfer_dict(py, transfer)?, + None => { + let dict = PyDict::new(py); + dict.set_item("index_key", &outcome.index_key)?; + dict + } + }; + dict.set_item("classification", &outcome.classification)?; + dict.set_item("action", &outcome.action)?; + dict.set_item("forced", outcome.forced)?; + dict.set_item("sidecar_corrupt", outcome.sidecar_corrupt)?; + if let Some(open) = &outcome.open { + dict.set_item("document_count", open.document_count)?; + dict.set_item("chunk_count", open.chunk_count)?; + } + Ok(dict) +} + +/// Renders a `VerifyReport` as a dict. +fn verify_dict<'py>(py: Python<'py>, report: &VerifyReport) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("index_key", &report.index_key)?; + dict.set_item("generation", report.generation)?; + dict.set_item("artifacts_verified", report.artifacts_verified)?; + dict.set_item("bytes_verified", report.bytes_verified)?; + Ok(dict) +} + +/// Lists the index keys with a committed generation in a local LodeDB directory. +#[pyfunction] +fn keys(py: Python<'_>, dir: &str) -> PyResult> { + py.detach(|| client_ops::keys(dir)).map_err(to_py_err) +} + +/// Compares `dir`'s committed generation against `remote` for a push under the +/// given policy flags. Read-only on both ends. +#[pyfunction] +#[pyo3(signature = (dir, remote, key, include_text = false, include_lexical = false))] +fn status<'py>( + py: Python<'py>, + dir: &str, + remote: &str, + key: &str, + include_text: bool, + include_lexical: bool, +) -> PyResult> { + let report = py + .detach(|| client_ops::status(dir, remote, key, policy(include_text, include_lexical))) + .map_err(to_py_err)?; + status_dict(py, &report) +} + +/// Pushes `dir`'s committed generation to `remote`. Redacted by default: the +/// payload-bearing text/lexical stores ship only with the explicit opt-ins. +#[pyfunction] +#[pyo3(signature = (dir, remote, key, include_text = false, include_lexical = false))] +fn push<'py>( + py: Python<'py>, + dir: &str, + remote: &str, + key: &str, + include_text: bool, + include_lexical: bool, +) -> PyResult> { + let result = py + .detach(|| client_ops::push(dir, remote, key, policy(include_text, include_lexical))) + .map_err(to_py_err)?; + transfer_dict(py, &result) +} + +/// Restores `key`'s committed generation from `remote` into the local `dir`. +/// The restore verifies the copy opens through the engine before reporting +/// success (see `client_ops::pull`); the returned dict is the transfer report +/// plus the opened `document_count`/`chunk_count`. +#[pyfunction] +fn pull<'py>(py: Python<'py>, remote: &str, dir: &str, key: &str) -> PyResult> { + let outcome = py + .detach(|| client_ops::pull(remote, dir, key)) + .map_err(to_py_err)?; + let dict = transfer_dict(py, &outcome.transfer)?; + dict.set_item("document_count", outcome.open.document_count)?; + dict.set_item("chunk_count", outcome.open.chunk_count)?; + Ok(dict) +} + +/// Synchronizes `key` between the local `dir` and `remote`: classifies local +/// vs the recorded sidecar base vs remote, then runs at most one fast-forward +/// transfer. Diverged/unknown lineage raises `RuntimeError` unless exactly one +/// of `force_push`/`force_pull` overrides it; setting both is a `ValueError`. +#[pyfunction] +#[pyo3(signature = ( + dir, remote, key, + include_text = false, include_lexical = false, + force_push = false, force_pull = false +))] +#[allow(clippy::too_many_arguments)] +fn sync<'py>( + py: Python<'py>, + dir: &str, + remote: &str, + key: &str, + include_text: bool, + include_lexical: bool, + force_push: bool, + force_pull: bool, +) -> PyResult> { + let force = match (force_push, force_pull) { + (true, true) => { + return Err(PyValueError::new_err( + "force_push and force_pull are mutually exclusive", + )) + } + (true, false) => SyncForce::Push, + (false, true) => SyncForce::Pull, + (false, false) => SyncForce::None, + }; + let outcome = py + .detach(|| { + client_ops::sync( + dir, + remote, + key, + policy(include_text, include_lexical), + force, + ) + }) + .map_err(to_py_err)?; + sync_dict(py, &outcome) +} + +/// Re-hashes every artifact `key`'s committed generation pins in `target` (a +/// local directory or object-store URL) against the manifest's checksums. +#[pyfunction] +fn verify<'py>(py: Python<'py>, target: &str, key: &str) -> PyResult> { + let report = py + .detach(|| client_ops::verify(target, key)) + .map_err(to_py_err)?; + verify_dict(py, &report) +} + +/// Parses a JSON body string handed across the FFI boundary. +fn parse_body(label: &str, body_json: &str) -> PyResult { + serde_json::from_str(body_json) + .map_err(|error| PyValueError::new_err(format!("{label} is not valid JSON: {error}"))) +} + +/// Renders one inventory artifact as a dict. +fn artifact_dict<'py>(py: Python<'py>, artifact: &ArtifactRef) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("name", &artifact.name)?; + dict.set_item("sha256", &artifact.sha256)?; + dict.set_item("size_bytes", artifact.size_bytes)?; + dict.set_item("kind", &artifact.kind)?; + dict.set_item("is_base", artifact.is_base)?; + Ok(dict) +} + +/// Renders a list of inventory artifacts. +fn artifact_list<'py>(py: Python<'py>, artifacts: &[ArtifactRef]) -> PyResult> { + let items = artifacts + .iter() + .map(|artifact| artifact_dict(py, artifact)) + .collect::>>()?; + PyList::new(py, items) +} + +/// Renders one side's identity/payload masks as a dict. +fn side_dict<'py>(py: Python<'py>, side: &managed::ManagedSide) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("snapshot_id", &side.snapshot_id)?; + dict.set_item("logical_id", &side.logical_id)?; + dict.set_item("generation", side.generation)?; + dict.set_item("has_text", side.has_text)?; + dict.set_item("has_lexical", side.has_lexical)?; + Ok(dict) +} + +/// Builds the decision context for one managed (`orecloud://`) index: local +/// identity + pointer document + artifact inventory, the remote head parsed +/// from `remote_body_json`, the sidecar base trusted against `remote_id` +/// (compared verbatim), and the three-pointer classification. Read-only. +#[pyfunction] +#[pyo3(signature = (dir, key, remote_id, remote_body_json = None, include_text = false, include_lexical = false))] +fn managed_plan<'py>( + py: Python<'py>, + dir: &str, + key: &str, + remote_id: &str, + remote_body_json: Option<&str>, + include_text: bool, + include_lexical: bool, +) -> PyResult> { + let remote_body = remote_body_json + .map(|json| parse_body("remote body", json)) + .transpose()?; + let plan = py + .detach(|| { + managed::managed_plan( + dir, + key, + remote_id, + remote_body, + policy(include_text, include_lexical), + ) + }) + .map_err(to_py_err)?; + + let dict = status_dict(py, &plan.report)?; + match &plan.local { + Some(local) => { + let local_dict = side_dict(py, &local.side)?; + let body_json = serde_json::to_string(&local.body).map_err(|error| { + PyRuntimeError::new_err(format!("failed to serialize local body: {error}")) + })?; + local_dict.set_item("body_json", body_json)?; + local_dict.set_item("pointer_document", &local.pointer_document)?; + local_dict.set_item("artifacts", artifact_list(py, &local.artifacts)?)?; + dict.set_item("local", local_dict)?; + } + None => dict.set_item("local", py.None())?, + } + match &plan.remote { + Some(remote) => dict.set_item("remote", side_dict(py, remote)?)?, + None => dict.set_item("remote", py.None())?, + } + match &plan.base { + Some(base) => { + let base_dict = PyDict::new(py); + base_dict.set_item("snapshot_id", &base.snapshot_id)?; + base_dict.set_item("logical_id", &base.logical_id)?; + base_dict.set_item("generation", base.generation)?; + dict.set_item("base", base_dict)?; + } + None => dict.set_item("base", py.None())?, + } + dict.set_item("base_is_current", plan.base_is_current)?; + dict.set_item( + "local_raw_snapshot_id", + plan.local_raw_snapshot_id.as_deref(), + )?; + Ok(dict) +} + +/// The number of valid, replayable records in `dir`'s WAL for `key` (0 when +/// the WAL is absent or empty). A pull-direction sync consults this BEFORE +/// downloading blobs; deliberately not part of `managed_plan`, so push/status +/// planning never pays an O(WAL bytes) scan. +#[pyfunction] +fn local_wal_ops(py: Python<'_>, dir: &str, key: &str) -> PyResult { + py.detach(|| client_ops::pending_wal_ops(dir, key)) + .map_err(to_py_err) +} + +/// Records `body_json` as the sidecar base for `remote_id` (stored verbatim) +/// after a successful managed transfer. +#[pyfunction] +fn managed_record_base( + py: Python<'_>, + dir: &str, + key: &str, + remote_id: &str, + body_json: &str, +) -> PyResult<()> { + let body = parse_body("body", body_json)?; + py.detach(|| managed::managed_record_base(dir, key, remote_id, &body)) + .map_err(to_py_err) +} + +/// The artifacts a pull of `body_json` must download into staging: everything +/// the remote body pins that `dir` does not already hold byte-identically. +#[pyfunction] +fn managed_pull_requirements<'py>( + py: Python<'py>, + dir: &str, + key: &str, + body_json: &str, +) -> PyResult> { + let body = parse_body("body", body_json)?; + let artifacts = py + .detach(|| managed::managed_pull_requirements(dir, key, &body)) + .map_err(to_py_err)?; + artifact_list(py, &artifacts) +} + +/// Restores a managed pull from `staging_dir` (holding `` blob files) +/// into `dir`, verifies the candidate opens through the engine BEFORE the +/// pointer moves, and records the sidecar base against `remote_id`. Runs +/// under the engine's single-writer lock; refuses when the destination WAL +/// holds acknowledged operations unless `discard_pending_wal` (the +/// force-pull semantics) truncates them. `expected_local_snapshot_id` pins +/// the local state the caller classified (`""` = classified as absent; +/// `None` = no pin, the plain-pull semantics): a local commit landing after +/// classification refuses with `SyncConflictError` instead of being +/// overwritten. Returns the transfer report plus the opened document/chunk +/// counts. +#[pyfunction] +#[pyo3(signature = ( + dir, key, remote_id, body_json, staging_dir, + discard_pending_wal = false, expected_local_snapshot_id = None +))] +#[allow(clippy::too_many_arguments)] +fn managed_materialize<'py>( + py: Python<'py>, + dir: &str, + key: &str, + remote_id: &str, + body_json: &str, + staging_dir: &str, + discard_pending_wal: bool, + expected_local_snapshot_id: Option<&str>, +) -> PyResult> { + let body = parse_body("body", body_json)?; + let outcome = py + .detach(|| { + managed::managed_materialize( + dir, + key, + remote_id, + body, + staging_dir, + discard_pending_wal, + expected_local_snapshot_id, + ) + }) + .map_err(to_py_err)?; + let dict = transfer_dict(py, &outcome.transfer)?; + dict.set_item("document_count", outcome.open.document_count)?; + dict.set_item("chunk_count", outcome.open.chunk_count)?; + Ok(dict) +} + +/// Registers the transfer plane as the `cloud` submodule of `_turbovec`. +/// +/// Reached as an attribute (`from lodedb._turbovec import cloud` / +/// `_turbovec.cloud.push(...)`), never as a dotted import — extension +/// submodules are not importable through the module finder. +pub(crate) fn register(parent: &Bound<'_, PyModule>) -> PyResult<()> { + let module = PyModule::new(parent.py(), "cloud")?; + module.add( + "SyncConflictError", + parent.py().get_type::(), + )?; + module.add_function(wrap_pyfunction!(keys, &module)?)?; + module.add_function(wrap_pyfunction!(status, &module)?)?; + module.add_function(wrap_pyfunction!(push, &module)?)?; + module.add_function(wrap_pyfunction!(pull, &module)?)?; + module.add_function(wrap_pyfunction!(sync, &module)?)?; + module.add_function(wrap_pyfunction!(verify, &module)?)?; + module.add_function(wrap_pyfunction!(local_wal_ops, &module)?)?; + module.add_function(wrap_pyfunction!(managed_plan, &module)?)?; + module.add_function(wrap_pyfunction!(managed_record_base, &module)?)?; + module.add_function(wrap_pyfunction!(managed_pull_requirements, &module)?)?; + module.add_function(wrap_pyfunction!(managed_materialize, &module)?)?; + parent.add_submodule(&module)?; + Ok(()) +} diff --git a/third_party/turbovec/turbovec-python/src/lib.rs b/third_party/turbovec/turbovec-python/src/lib.rs index 03f3b130..466656ba 100644 --- a/third_party/turbovec/turbovec-python/src/lib.rs +++ b/third_party/turbovec/turbovec-python/src/lib.rs @@ -10,6 +10,11 @@ use lodedb_core::{ use numpy::{IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyType}; + +// The managed-cloud transfer plane (push/pull/sync over the open commit +// format), exposed as the `cloud` submodule so the [cloud] extra's Python +// client rides the same bundled extension as the engine. +mod cloud; use serde::de::DeserializeOwned; use serde_json::Value; @@ -1725,5 +1730,6 @@ fn _turbovec(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(build_embedded_documents_payload, m)?)?; m.add_function(wrap_pyfunction!(encode_wal_segment, m)?)?; m.add_function(wrap_pyfunction!(decode_wal_segment, m)?)?; + cloud::register(m)?; Ok(()) } diff --git a/uv.lock b/uv.lock index d9675788..b0a0e28d 100644 --- a/uv.lock +++ b/uv.lock @@ -808,34 +808,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-runtime" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cufft" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-cupti" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-curand" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusolver" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusparse" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvtx" }, ] [[package]] @@ -1876,7 +1876,7 @@ dependencies = [ { name = "deprecation" }, { name = "lance-namespace" }, { name = "numpy" }, - { name = "overrides", marker = "python_full_version < '3.12'" }, + { name = "overrides", marker = "python_full_version < '3.12' or python_full_version >= '3.15'" }, { name = "packaging" }, { name = "pyarrow" }, { name = "pydantic" }, @@ -2075,6 +2075,10 @@ all = [ { name = "sentence-transformers" }, { name = "transformers" }, ] +cloud = [ + { name = "httpx" }, + { name = "pynacl" }, +] cognee = [ { name = "cognee", marker = "python_full_version < '3.15'" }, ] @@ -2116,6 +2120,7 @@ torch = [ requires-dist = [ { name = "cognee", marker = "python_full_version < '3.15' and extra == 'cognee'", specifier = ">=1.1.0,<2" }, { name = "cupy-cuda12x", marker = "sys_platform == 'linux' and extra == 'gpu'", specifier = ">=13.0.0" }, + { name = "httpx", marker = "extra == 'cloud'", specifier = ">=0.27" }, { name = "langchain-core", marker = "extra == 'langchain'", specifier = ">=0.3.0" }, { name = "llama-index-core", marker = "extra == 'llama-index'", specifier = ">=0.11.0" }, { name = "lodedb", extras = ["embeddings", "torch", "image", "mcp", "langchain", "llama-index", "mem0"], marker = "extra == 'all'" }, @@ -2126,6 +2131,7 @@ requires-dist = [ { name = "onnxruntime", marker = "extra == 'embeddings'", specifier = ">=1.20.0,<2" }, { name = "optimum", extras = ["exporters"], marker = "extra == 'onnx-export'", specifier = ">=1.23.0,<2.0.0" }, { name = "pillow", marker = "extra == 'image'", specifier = ">=10.0.0" }, + { name = "pynacl", marker = "extra == 'cloud'", specifier = ">=1.5" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, { name = "pyyaml", specifier = ">=6.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.17" }, @@ -2133,7 +2139,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'embeddings'", specifier = ">=4.40.0,<5" }, { name = "typer", specifier = ">=0.12.0" }, ] -provides-extras = ["embeddings", "torch", "gpu", "onnx-export", "mcp", "langchain", "llama-index", "mem0", "cognee", "image", "all", "dev"] +provides-extras = ["embeddings", "torch", "gpu", "onnx-export", "mcp", "langchain", "llama-index", "mem0", "cognee", "cloud", "image", "all", "dev"] [[package]] name = "lupa" @@ -3563,6 +3569,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/4f/a6a2e2b202d7fd97eadfe90979845b8706676b41cbd3b42ba75adf329d1f/Pympler-1.1-py3-none-any.whl", hash = "sha256:5b223d6027d0619584116a0cbc28e8d2e378f7a79c1e5e024f9ff3b673c58506", size = 165766, upload-time = "2024-06-28T19:56:05.087Z" }, ] +[[package]] +name = "pynacl" +version = "1.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/9a/4019b524b03a13438637b11538c82781a5eda427394380381af8f04f467a/pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c", size = 3511692, upload-time = "2026-01-01T17:48:10.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/79/0e3c34dc3c4671f67d251c07aa8eb100916f250ee470df230b0ab89551b4/pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594", size = 390064, upload-time = "2026-01-01T17:31:57.264Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1c/23a26e931736e13b16483795c8a6b2f641bf6a3d5238c22b070a5112722c/pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0", size = 809370, upload-time = "2026-01-01T17:31:59.198Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/8d4b718f8a22aea9e8dcc8b95deb76d4aae380e2f5b570cc70b5fd0a852d/pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9", size = 1408304, upload-time = "2026-01-01T17:32:01.162Z" }, + { url = "https://files.pythonhosted.org/packages/fd/73/be4fdd3a6a87fe8a4553380c2b47fbd1f7f58292eb820902f5c8ac7de7b0/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574", size = 844871, upload-time = "2026-01-01T17:32:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/55/ad/6efc57ab75ee4422e96b5f2697d51bbcf6cdcc091e66310df91fbdc144a8/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634", size = 1446356, upload-time = "2026-01-01T17:32:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/78/b7/928ee9c4779caa0a915844311ab9fb5f99585621c5d6e4574538a17dca07/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88", size = 826814, upload-time = "2026-01-01T17:32:06.078Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a9/1bdba746a2be20f8809fee75c10e3159d75864ef69c6b0dd168fc60e485d/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14", size = 1411742, upload-time = "2026-01-01T17:32:07.651Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2f/5e7ea8d85f9f3ea5b6b87db1d8388daa3587eed181bdeb0306816fdbbe79/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444", size = 801714, upload-time = "2026-01-01T17:32:09.558Z" }, + { url = "https://files.pythonhosted.org/packages/06/ea/43fe2f7eab5f200e40fb10d305bf6f87ea31b3bbc83443eac37cd34a9e1e/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b", size = 1372257, upload-time = "2026-01-01T17:32:11.026Z" }, + { url = "https://files.pythonhosted.org/packages/4d/54/c9ea116412788629b1347e415f72195c25eb2f3809b2d3e7b25f5c79f13a/pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145", size = 231319, upload-time = "2026-01-01T17:32:12.46Z" }, + { url = "https://files.pythonhosted.org/packages/ce/04/64e9d76646abac2dccf904fccba352a86e7d172647557f35b9fe2a5ee4a1/pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590", size = 244044, upload-time = "2026-01-01T17:32:13.781Z" }, + { url = "https://files.pythonhosted.org/packages/33/33/7873dc161c6a06f43cda13dec67b6fe152cb2f982581151956fa5e5cdb47/pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2", size = 188740, upload-time = "2026-01-01T17:32:15.083Z" }, + { url = "https://files.pythonhosted.org/packages/be/7b/4845bbf88e94586ec47a432da4e9107e3fc3ce37eb412b1398630a37f7dd/pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465", size = 388458, upload-time = "2026-01-01T17:32:16.829Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b4/e927e0653ba63b02a4ca5b4d852a8d1d678afbf69b3dbf9c4d0785ac905c/pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0", size = 800020, upload-time = "2026-01-01T17:32:18.34Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/d60984052df5c97b1d24365bc1e30024379b42c4edcd79d2436b1b9806f2/pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4", size = 1399174, upload-time = "2026-01-01T17:32:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/68/f7/322f2f9915c4ef27d140101dd0ed26b479f7e6f5f183590fd32dfc48c4d3/pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87", size = 835085, upload-time = "2026-01-01T17:32:22.24Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d0/f301f83ac8dbe53442c5a43f6a39016f94f754d7a9815a875b65e218a307/pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c", size = 1437614, upload-time = "2026-01-01T17:32:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/c4/58/fc6e649762b029315325ace1a8c6be66125e42f67416d3dbd47b69563d61/pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130", size = 818251, upload-time = "2026-01-01T17:32:25.69Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a8/b917096b1accc9acd878819a49d3d84875731a41eb665f6ebc826b1af99e/pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6", size = 1402859, upload-time = "2026-01-01T17:32:27.215Z" }, + { url = "https://files.pythonhosted.org/packages/85/42/fe60b5f4473e12c72f977548e4028156f4d340b884c635ec6b063fe7e9a5/pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e", size = 791926, upload-time = "2026-01-01T17:32:29.314Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/e40e318c604259301cc091a2a63f237d9e7b424c4851cafaea4ea7c4834e/pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577", size = 1363101, upload-time = "2026-01-01T17:32:31.263Z" }, + { url = "https://files.pythonhosted.org/packages/48/47/e761c254f410c023a469284a9bc210933e18588ca87706ae93002c05114c/pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa", size = 227421, upload-time = "2026-01-01T17:32:33.076Z" }, + { url = "https://files.pythonhosted.org/packages/41/ad/334600e8cacc7d86587fe5f565480fde569dfb487389c8e1be56ac21d8ac/pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0", size = 239754, upload-time = "2026-01-01T17:32:34.557Z" }, + { url = "https://files.pythonhosted.org/packages/29/7d/5945b5af29534641820d3bd7b00962abbbdfee84ec7e19f0d5b3175f9a31/pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c", size = 184801, upload-time = "2026-01-01T17:32:36.309Z" }, +] + [[package]] name = "pyparsing" version = "3.3.2" @@ -3791,7 +3832,7 @@ name = "redis" version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, + { name = "async-timeout", marker = "python_full_version < '3.11.3' or python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" } wheels = [