From f0a27ddfc066737b6e7198400f2ea8989e8964c1 Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Tue, 5 Sep 2023 11:44:44 -0700 Subject: [PATCH] feat: Introduce RocksDbStorage, genericize storage throughout. * Introduce `RocksDbStorage` * Rename NativeStorage -> SledStorage, WebStorage -> IndexedDbStorage. Make storage generic over gateway. * Added `PerformanceStorage`, `trait noosphere_storage::Space` for benchmarking storage backends. --- .github/workflows/noosphere_apple_build.yaml | 2 +- .github/workflows/run_test_suite.yaml | 6 +- Cargo.lock | 537 ++++++++++++------ rust/noosphere-cli/Cargo.toml | 5 +- .../src/native/helpers/workspace.rs | 13 +- rust/noosphere-cli/src/native/workspace.rs | 8 +- rust/noosphere-common/Cargo.toml | 2 +- .../dht => noosphere-common/src}/channel.rs | 53 +- rust/noosphere-common/src/lib.rs | 1 + rust/noosphere-core/Cargo.toml | 2 +- .../src/context/authority/read.rs | 10 +- .../src/context/authority/write.rs | 13 +- rust/noosphere-core/src/context/context.rs | 16 +- rust/noosphere-core/src/context/cursor.rs | 19 +- rust/noosphere-core/src/context/mod.rs | 11 +- rust/noosphere-core/src/context/walker.rs | 12 +- rust/noosphere-core/src/helpers/context.rs | 42 +- rust/noosphere-core/src/stream/mod.rs | 21 +- rust/noosphere-gateway/src/authority.rs | 21 +- .../src/handlers/v0alpha1/fetch.rs | 2 +- .../src/handlers/v0alpha1/identify.rs | 2 +- .../src/handlers/v0alpha1/push.rs | 2 +- .../src/handlers/v0alpha1/replicate.rs | 16 +- .../src/handlers/v0alpha2/push.rs | 2 +- .../src/worker/name_system.rs | 6 +- rust/noosphere-into/src/into/html/sphere.rs | 7 +- rust/noosphere-ns/Cargo.toml | 1 + rust/noosphere-ns/src/dht/errors.rs | 2 +- rust/noosphere-ns/src/dht/mod.rs | 1 - rust/noosphere-ns/src/dht/node.rs | 4 +- rust/noosphere-ns/src/dht/rpc.rs | 2 +- rust/noosphere-storage/Cargo.toml | 24 +- .../examples/bench/Cargo.toml | 27 + rust/noosphere-storage/examples/bench/main.rs | 270 +++++++++ rust/noosphere-storage/src/helpers.rs | 13 +- .../implementation/{web.rs => indexed_db.rs} | 134 ++++- .../src/implementation/memory.rs | 16 + .../src/implementation/mod.rs | 20 +- .../src/implementation/performance.rs | 208 +++++++ .../src/implementation/rocks_db.rs | 148 +++++ .../src/implementation/{native.rs => sled.rs} | 52 +- rust/noosphere-storage/src/key_value.rs | 41 +- rust/noosphere-storage/src/lib.rs | 5 + rust/noosphere-storage/src/space.rs | 36 ++ rust/noosphere-storage/src/storage.rs | 15 +- rust/noosphere-storage/src/store.rs | 31 +- rust/noosphere/Cargo.toml | 6 +- rust/noosphere/build.rs | 23 + rust/noosphere/src/ffi/noosphere.rs | 3 + rust/noosphere/src/platform.rs | 64 +-- rust/noosphere/src/sphere/builder.rs | 4 +- rust/noosphere/src/storage.rs | 33 +- 52 files changed, 1518 insertions(+), 496 deletions(-) rename rust/{noosphere-ns/src/dht => noosphere-common/src}/channel.rs (75%) create mode 100644 rust/noosphere-storage/examples/bench/Cargo.toml create mode 100644 rust/noosphere-storage/examples/bench/main.rs rename rust/noosphere-storage/src/implementation/{web.rs => indexed_db.rs} (50%) create mode 100644 rust/noosphere-storage/src/implementation/performance.rs create mode 100644 rust/noosphere-storage/src/implementation/rocks_db.rs rename rust/noosphere-storage/src/implementation/{native.rs => sled.rs} (61%) create mode 100644 rust/noosphere-storage/src/space.rs create mode 100644 rust/noosphere/build.rs diff --git a/.github/workflows/noosphere_apple_build.yaml b/.github/workflows/noosphere_apple_build.yaml index a2894e56a..43fd3df2a 100644 --- a/.github/workflows/noosphere_apple_build.yaml +++ b/.github/workflows/noosphere_apple_build.yaml @@ -72,7 +72,7 @@ jobs: with: brew: protobuf cmake - name: 'Cargo Build' - run: cargo build --package noosphere --release --locked --target ${{ matrix.target }} + run: cargo build --package noosphere --release --locked --target ${{ matrix.target }} --features rocksdb - run: | cd target/${{ matrix.target }}/release - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml index 928c12b18..48505ab2f 100644 --- a/.github/workflows/run_test_suite.yaml +++ b/.github/workflows/run_test_suite.yaml @@ -55,7 +55,7 @@ jobs: ipfs_version: v0.17.0 run_daemon: true - name: 'Run Rust native target tests' - run: cargo test --features test-kubo,helpers + run: cargo test --features test-kubo,helpers,performance env: NOOSPHERE_LOG: deafening @@ -96,7 +96,7 @@ jobs: ipfs_version: v0.17.0 run_daemon: true - name: 'Run Rust native target tests' - run: NOOSPHERE_LOG=deafening cargo test --features test-kubo,headers + run: NOOSPHERE_LOG=deafening cargo test --features test-kubo,headers,performance run-test-suite-linux-c: runs-on: ubuntu-latest steps: @@ -160,7 +160,7 @@ jobs: shell: bash - name: 'Run Rust headless browser tests' working-directory: ./rust - run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown + run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown --features performance shell: bash run-test-suite-web-typescript: diff --git a/Cargo.lock b/Cargo.lock index d82ecf8da..675c5aad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -190,9 +190,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "anstyle-parse" @@ -224,9 +224,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arc-swap" @@ -363,13 +363,13 @@ checksum = "5b49bd4c5b769125ea6323601c39815848972880efd33ffb2d01f9f909adc699" [[package]] name = "async-recursion" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -391,7 +391,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -402,7 +402,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -502,14 +502,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -565,6 +565,27 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease 0.2.15", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.32", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -601,24 +622,24 @@ dependencies = [ [[package]] name = "blake2b_simd" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" +checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" dependencies = [ "arrayref", "arrayvec", - "constant_time_eq 0.2.6", + "constant_time_eq", ] [[package]] name = "blake2s_simd" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" +checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" dependencies = [ "arrayref", "arrayvec", - "constant_time_eq 0.2.6", + "constant_time_eq", ] [[package]] @@ -631,7 +652,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.3.0", + "constant_time_eq", ] [[package]] @@ -685,9 +706,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" dependencies = [ "memchr", "serde", @@ -707,13 +728,24 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" dependencies = [ "serde", ] +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cbor4ii" version = "0.2.14" @@ -725,10 +757,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -743,12 +776,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chacha20" version = "0.8.2" @@ -776,9 +824,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.28" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" +checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" dependencies = [ "android-tzdata", "iana-time-zone", @@ -828,11 +876,22 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive", @@ -859,7 +918,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -911,15 +970,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "795bc6e66a8e340f075fcf6227e417a2dc976b92b91f3cdc778bb858778b6747" - -[[package]] -name = "constant_time_eq" -version = "0.2.6" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "constant_time_eq" @@ -1027,9 +1080,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4c2f4e1afd912bc40bfd6fed5d9dc1f288e0ba01bfcc835cc5bc3eb13efe15" +checksum = "740fe28e594155f10cfc383984cbefd529d7396050557148f79cb0f621204124" dependencies = [ "generic-array", "rand_core 0.6.4", @@ -1110,9 +1163,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" +checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" dependencies = [ "cfg-if", "cpufeatures", @@ -1133,7 +1186,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1220,9 +1273,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ed52955ce76b1554f509074bb357d3fb8ac9b51288a65a3fd480d1dfba946" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", "pem-rfc7468 0.7.0", @@ -1259,9 +1312,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "derive_builder" @@ -1364,7 +1417,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1391,7 +1444,7 @@ version = "0.16.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4b1e0c257a9e9f25f90ff76d7a68360ed497ee519c8e428d1825ef0000799d4" dependencies = [ - "der 0.7.7", + "der 0.7.8", "digest 0.10.7", "elliptic-curve 0.13.5", "rfc6979 0.4.0", @@ -1415,7 +1468,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ - "curve25519-dalek 4.0.0", + "curve25519-dalek 4.1.0", "ed25519", "rand_core 0.6.4", "serde", @@ -1473,7 +1526,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" dependencies = [ "base16ct 0.2.0", - "crypto-bigint 0.5.2", + "crypto-bigint 0.5.3", "digest 0.10.7", "ff 0.13.0", "generic-array", @@ -1612,9 +1665,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.1.20" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" [[package]] name = "fnv" @@ -1739,7 +1792,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1870,14 +1923,20 @@ checksum = "ba330b70a5341d3bc730b8e205aaee97ddab5d9c448c4f51a7c2d924266fa8f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" @@ -2375,7 +2434,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.3", + "socket2 0.5.4", "widestring", "windows-sys", "winreg", @@ -2442,6 +2501,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.64" @@ -2469,6 +2537,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.147" @@ -2565,6 +2639,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libm" version = "0.2.7" @@ -2962,6 +3046,33 @@ dependencies = [ "yamux", ] +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2976,9 +3087,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -2992,9 +3103,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" @@ -3014,6 +3125,16 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "mac" version = "0.1.1" @@ -3074,9 +3195,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" @@ -3338,6 +3459,7 @@ dependencies = [ "async-stream", "async-trait", "bytes", + "cfg_aliases", "cid", "gloo-timers", "instant", @@ -3647,6 +3769,7 @@ dependencies = [ "libipld-cbor", "libp2p", "noosphere", + "noosphere-common", "noosphere-core", "noosphere-ipfs", "noosphere-storage", @@ -3657,7 +3780,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "toml 0.7.6", + "toml 0.7.8", "tower-http", "tracing", "ucan", @@ -3706,19 +3829,25 @@ dependencies = [ "async-trait", "base64 0.21.2", "cid", + "instant", "js-sys", "libipld-cbor", "libipld-core", "noosphere-common", + "noosphere-core", + "rand", "rexie", + "rocksdb", "serde", "sled", + "tempfile", "tokio", "tokio-stream", "tracing", "ucan", "url", "wasm-bindgen", + "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", "witty-phrase-generator", @@ -3814,9 +3943,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -3972,6 +4101,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "1.1.1" @@ -4022,14 +4157,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "pin-project-lite" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -4043,7 +4178,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.7", + "der 0.7.8", "pkcs8 0.10.2", "spki 0.7.2", ] @@ -4064,10 +4199,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.7", + "der 0.7.8", "spki 0.7.2", ] +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "pkg-version" version = "1.0.0" @@ -4160,6 +4301,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn 2.0.32", +] + [[package]] name = "primeorder" version = "0.13.2" @@ -4238,7 +4389,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -4289,9 +4440,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -4397,14 +4548,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.8", + "regex-syntax 0.7.5", ] [[package]] @@ -4418,13 +4569,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -4435,9 +4586,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" @@ -4541,6 +4692,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rsa" version = "0.9.2" @@ -4649,14 +4810,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.5", + "linux-raw-sys 0.4.7", "windows-sys", ] @@ -4708,9 +4869,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" dependencies = [ "ring", "untrusted", @@ -4741,9 +4902,9 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "safer-ffi" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f47f1d2f33598dab2baa9517fffa1cf722f2e3a30633f2a230f20f9da67c564" +checksum = "e9c1d19b288ca9898cd421c7b105fb7269918a7f8e9253a991e228981ca421ad" dependencies = [ "inventory", "libc", @@ -4758,12 +4919,12 @@ dependencies = [ [[package]] name = "safer_ffi-proc_macros" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f58cf71a58bda5734758eb20051cdb66c06c9243badbc45092ced1be834df" +checksum = "e2d7a04caa3ca2224f5ea4ddd850e2629c3b36b2b83621f87a8303bf41020110" dependencies = [ "macro_rules_attribute", - "prettyplease", + "prettyplease 0.1.25", "proc-macro2", "quote", "syn 1.0.109", @@ -4843,7 +5004,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", - "der 0.7.7", + "der 0.7.8", "generic-array", "pkcs8 0.10.2", "subtle", @@ -4858,9 +5019,9 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "sentry-core" -version = "0.31.5" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7163491708804a74446642ff2c80b3acd668d4b9e9f497f85621f3d250fd012b" +checksum = "8339474f587f36cb110fa1ed1b64229eea6d47b0b886375579297b7e47aeb055" dependencies = [ "once_cell", "rand", @@ -4871,9 +5032,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.31.5" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca8b88978677a27ee1a91beafe4052306c474c06f582321fde72d2e2cc2f7f" +checksum = "89feead9bdd116f8035e89567651340fc382db29240b6c55ef412078b08d1aa3" dependencies = [ "sentry-core", "tracing-core", @@ -4882,9 +5043,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.31.5" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7a88e0c1922d19b3efee12a8215f6a8a806e442e665ada71cc222cab72985f" +checksum = "99dc599bd6646884fc403d593cdcb9816dd67c50cff3271c01ff123617908dcd" dependencies = [ "debugid", "getrandom 0.2.10", @@ -4899,9 +5060,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] @@ -4926,20 +5087,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "serde_ipld_dagcbor" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace39c1b7526be78c755a4c698313f699cf44e62408c0029bf9ab9450fe836da" +checksum = "8e880e0b1f9c7a8db874642c1217f7e19b29e325f24ab9f0fcb11818adec7f01" dependencies = [ "cbor4ii", "cid", @@ -4949,9 +5110,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" dependencies = [ "itoa", "ryu", @@ -5056,6 +5217,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5087,9 +5254,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -5125,7 +5292,7 @@ dependencies = [ "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0", + "curve25519-dalek 4.1.0", "rand_core 0.6.4", "ring", "rustc_version", @@ -5145,9 +5312,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -5176,7 +5343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der 0.7.7", + "der 0.7.8", ] [[package]] @@ -5213,7 +5380,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -5285,9 +5452,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2", "quote", @@ -5348,7 +5515,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.11", + "rustix 0.38.13", "windows-sys", ] @@ -5365,22 +5532,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -5395,9 +5562,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa", @@ -5414,9 +5581,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -5467,11 +5634,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -5480,7 +5646,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.4.9", + "socket2 0.5.4", "tokio-macros", "windows-sys", ] @@ -5493,7 +5659,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -5543,9 +5709,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", @@ -5564,9 +5730,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.0.0", "serde", @@ -5649,7 +5815,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -5926,9 +6092,9 @@ dependencies = [ [[package]] name = "unsigned-varint" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" dependencies = [ "asynchronous-codec", "bytes", @@ -5948,9 +6114,9 @@ checksum = "0976c77def3f1f75c4ef892a292c31c0bbe9e3d0702c63044d7c76db298171a3" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna 0.4.0", @@ -5986,6 +6152,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -6015,9 +6187,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -6051,6 +6223,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -6065,7 +6239,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -6099,7 +6273,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6476,24 +6650,24 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.48.0", + "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -6503,9 +6677,9 @@ checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -6515,9 +6689,9 @@ checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -6527,9 +6701,9 @@ checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -6539,15 +6713,15 @@ checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -6557,9 +6731,9 @@ checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" @@ -6678,7 +6852,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ - "curve25519-dalek 4.0.0", + "curve25519-dalek 4.1.0", "rand_core 0.6.4", "serde", "zeroize", @@ -6723,9 +6897,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" +checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" [[package]] name = "yamux" @@ -6767,5 +6941,16 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", ] diff --git a/rust/noosphere-cli/Cargo.toml b/rust/noosphere-cli/Cargo.toml index d32fc4a6b..7ac7249b7 100644 --- a/rust/noosphere-cli/Cargo.toml +++ b/rust/noosphere-cli/Cargo.toml @@ -17,7 +17,9 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] +default = [] helpers = ["tracing-subscriber", "noosphere-ns"] +rocksdb = ["noosphere/rocksdb"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -56,8 +58,7 @@ cid = { workspace = true } symlink = { workspace = true } pathdiff = { workspace = true } subtext = "0.3.2" -rand = {workspace = true } - +rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } libipld-core = { workspace = true } diff --git a/rust/noosphere-cli/src/native/helpers/workspace.rs b/rust/noosphere-cli/src/native/helpers/workspace.rs index 9f938b8db..28ed613cd 100644 --- a/rust/noosphere-cli/src/native/helpers/workspace.rs +++ b/rust/noosphere-cli/src/native/helpers/workspace.rs @@ -1,18 +1,15 @@ use anyhow::Result; use noosphere_ipfs::{IpfsStore, KuboClient}; -use noosphere_storage::{BlockStoreRetry, MemoryStore, NativeStorage, UcanStore}; +use noosphere_storage::{BlockStoreRetry, MemoryStore, UcanStore}; use std::{net::TcpListener, sync::Arc}; use tempfile::TempDir; use crate::{ cli::ConfigSetCommand, commands::{key::key_create, sphere::config_set, sphere::sphere_create}, - workspace::Workspace, -}; -use noosphere_core::{ - context::{HasSphereContext, SphereContext}, - data::Did, + workspace::{CliSphereContext, Workspace}, }; +use noosphere_core::{context::HasSphereContext, data::Did}; use noosphere_gateway::{start_gateway, GatewayScope}; use noosphere_ns::{helpers::NameSystemNetwork, server::start_name_system_api_server}; use tokio::{sync::Mutex, task::JoinHandle}; @@ -216,7 +213,7 @@ impl SpherePair { } /// Returns a [SphereContext] for the client sphere. - pub async fn sphere_context(&self) -> Result>>> { + pub async fn sphere_context(&self) -> Result>> { self.client.workspace.sphere_context().await } @@ -226,7 +223,7 @@ impl SpherePair { pub async fn spawn(&self, f: F) -> Result where T: Send + 'static, - F: FnOnce(Arc>>) -> Fut + Send + 'static, + F: FnOnce(Arc>) -> Fut + Send + 'static, Fut: std::future::Future> + Send + 'static, { let context = self.sphere_context().await?; diff --git a/rust/noosphere-cli/src/native/workspace.rs b/rust/noosphere-cli/src/native/workspace.rs index 0c4680b66..5d7912ca4 100644 --- a/rust/noosphere-cli/src/native/workspace.rs +++ b/rust/noosphere-cli/src/native/workspace.rs @@ -3,13 +3,13 @@ use anyhow::{anyhow, Result}; use cid::Cid; use directories::ProjectDirs; -use noosphere::sphere::SphereContextBuilder; +use noosphere::{platform::PlatformStorage, sphere::SphereContextBuilder}; use noosphere_core::authority::Author; use noosphere_core::context::{ SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL, }; use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld}; -use noosphere_storage::{KeyValueStore, NativeStorage, SphereDb}; +use noosphere_storage::{KeyValueStore, SphereDb}; use serde_json::Value; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -26,7 +26,7 @@ use super::paths::SpherePaths; use super::render::SphereRenderer; /// The flavor of [SphereContext] used through the CLI -pub type CliSphereContext = SphereContext; +pub type CliSphereContext = SphereContext; /// Metadata about a given sphere, including the sphere ID, a [Link] /// to it and a corresponding [LinkRecord] (if one is available). @@ -70,7 +70,7 @@ impl Workspace { /// Get an owned referenced to the [SphereDb] that backs the local sphere. /// Note that this will initialize the [SphereContext] if it has not been /// already. - pub async fn db(&self) -> Result> { + pub async fn db(&self) -> Result> { let context = self.sphere_context().await?; let context = context.lock().await; Ok(context.db().clone()) diff --git a/rust/noosphere-common/Cargo.toml b/rust/noosphere-common/Cargo.toml index 43c752f95..aead7452d 100644 --- a/rust/noosphere-common/Cargo.toml +++ b/rust/noosphere-common/Cargo.toml @@ -31,4 +31,4 @@ tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["sync", "macros"] } futures = { workspace = true } wasm-bindgen = { workspace = true } -wasm-bindgen-futures = { workspace = true } \ No newline at end of file +wasm-bindgen-futures = { workspace = true } diff --git a/rust/noosphere-ns/src/dht/channel.rs b/rust/noosphere-common/src/channel.rs similarity index 75% rename from rust/noosphere-ns/src/dht/channel.rs rename to rust/noosphere-common/src/channel.rs index 8a4f382cb..3b4295922 100644 --- a/rust/noosphere-ns/src/dht/channel.rs +++ b/rust/noosphere-common/src/channel.rs @@ -1,3 +1,7 @@ +//! Utility wrapper around [tokio::sync::mpsc] channels, enabling multiple +//! producers to send messages to a single subscriber, with each message +//! able to be responded to by the subscriber. + use core::{fmt, result::Result}; use tokio; use tokio::sync::{mpsc, mpsc::error::SendError, oneshot, oneshot::error::RecvError}; @@ -15,7 +19,11 @@ impl fmt::Display for ChannelError { /// and distinguish between user-land respond errors. #[derive(Debug)] pub enum ChannelError { + /// An occurred during sending a message. + /// From [tokio::sync::mpsc::error::SendError]. SendError, + /// An occurred during the receiving of a message. + /// From [tokio::sync::mpsc::error::RecvError]. RecvError, } @@ -31,14 +39,17 @@ impl From for ChannelError { } } -/// Represents a request to be processed in `MessageProcessor`, -/// sent from the associated `MessageClient`. +/// Represents a request to be processed in [MessageProcessor], +/// sent from the associated [MessageClient]. pub struct Message { + /// The initial request the [Message] is wrapping. pub request: Q, sender: oneshot::Sender>, } impl Message { + /// Send `response` to the originator of this [Message]. + /// Each message can only be responded to once. pub fn respond(self, response: Result) -> bool { self.sender.send(response).map_or_else(|_| false, |_| true) } @@ -56,20 +67,24 @@ impl fmt::Debug for Message { /// /// Instances are created by the /// [`message_channel`](message_channel) function. +#[derive(Debug)] pub struct MessageClient { tx: mpsc::UnboundedSender>, } impl MessageClient { - // TBD if/how "synchronous" requests will work. + /// Sends a one-way request to the corresponding receiver. Use + /// [MessageClient::send] if the receiver should be able to respond. #[allow(dead_code)] - pub fn send_request(&self, request: Q) -> Result<(), ChannelError> { + pub fn send_oneshot(&self, request: Q) -> Result<(), ChannelError> { self.send_request_impl(request) .map(|_| Ok(())) .map_err(ChannelError::from)? } - pub async fn send_request_async(&self, request: Q) -> Result, ChannelError> { + /// Sends a request to the corresponding receiver where it can be + /// responded to. + pub async fn send(&self, request: Q) -> Result, ChannelError> { let rx = self .send_request_impl(request) .map_err(ChannelError::from)?; @@ -91,16 +106,27 @@ impl MessageClient { } } +// Manually implement `Clone` so that the generics do not need +// also implement. +impl Clone for MessageClient { + fn clone(&self) -> Self { + MessageClient { + tx: self.tx.clone(), + } + } +} + /// Receives requests from the associated `MessageClient`, /// and optionally sends a response. /// -/// Instances are created by the -/// [`message_channel`](message_channel) function. +/// Instances are created by the [message_channel] function. pub struct MessageProcessor { rx: mpsc::UnboundedReceiver>, } impl MessageProcessor { + /// Awaits until it can return a new message to process, or + /// [None] if all senders have been terminated. pub async fn pull_message(&mut self) -> Option> { self.rx.recv().await } @@ -171,14 +197,17 @@ mod tests { } }); - let res = client.send_request_async(Request::Ping()).await?; - matches!(res, Ok(Response::Pong())); + let res = client.send(Request::Ping()).await?; + assert!(match res { + Ok(Response::Pong()) => true, + _ => false, + }); for n in 0..10 { - client.send_request(Request::SetFlag(n))?; + client.send_oneshot(Request::SetFlag(n))?; } - let res = client.send_request_async(Request::Throw()).await?; + let res = client.send(Request::Throw()).await?; assert!( match res { Ok(_) => false, @@ -190,7 +219,7 @@ mod tests { "User Error propagates to client." ); - let res = client.send_request_async(Request::Shutdown()).await?; + let res = client.send(Request::Shutdown()).await?; assert!( match res { Ok(Response::GenericResult(success)) => success, diff --git a/rust/noosphere-common/src/lib.rs b/rust/noosphere-common/src/lib.rs index 51b7cebae..405d433a7 100644 --- a/rust/noosphere-common/src/lib.rs +++ b/rust/noosphere-common/src/lib.rs @@ -4,6 +4,7 @@ #[macro_use] extern crate tracing; +pub mod channel; mod sync; mod task; mod unshared; diff --git a/rust/noosphere-core/Cargo.toml b/rust/noosphere-core/Cargo.toml index 03a90473b..d6dca3c13 100644 --- a/rust/noosphere-core/Cargo.toml +++ b/rust/noosphere-core/Cargo.toml @@ -90,4 +90,4 @@ tracing-wasm = "~0.2" workspace = true features = [ "ReadableStream" -] \ No newline at end of file +] diff --git a/rust/noosphere-core/src/context/authority/read.rs b/rust/noosphere-core/src/context/authority/read.rs index 01cd92f4a..03c487461 100644 --- a/rust/noosphere-core/src/context/authority/read.rs +++ b/rust/noosphere-core/src/context/authority/read.rs @@ -125,7 +125,7 @@ where #[cfg(test)] mod tests { - use crate::data::Did; + use crate::{authority::Access, data::Did}; use anyhow::Result; use ucan::crypto::KeyMaterial; @@ -137,14 +137,13 @@ mod tests { use crate::{ context::{HasSphereContext, SphereAuthorityRead}, - helpers::{simulated_sphere_context, SimulationAccess}, + helpers::simulated_sphere_context, }; #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_get_an_authorization_by_did() -> Result<()> { - let (sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let author_did = Did(sphere_context .sphere_context() @@ -169,8 +168,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_verify_an_authorization_to_write_to_a_sphere() -> Result<()> { - let (sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let author_did = Did(sphere_context .sphere_context() diff --git a/rust/noosphere-core/src/context/authority/write.rs b/rust/noosphere-core/src/context/authority/write.rs index db62725a7..f6bf1582a 100644 --- a/rust/noosphere-core/src/context/authority/write.rs +++ b/rust/noosphere-core/src/context/authority/write.rs @@ -226,7 +226,7 @@ where mod tests { use std::sync::Arc; - use crate::authority::{generate_ed25519_key, Author}; + use crate::authority::{generate_ed25519_key, Access, Author}; use crate::data::Did; use anyhow::Result; @@ -243,14 +243,13 @@ mod tests { HasMutableSphereContext, HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereContextKey, }, - helpers::{simulated_sphere_context, SimulationAccess}, + helpers::simulated_sphere_context, }; #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_allows_an_authorized_key_to_authorize_other_keys() -> Result<()> { - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let other_key = generate_ed25519_key(); let other_did = Did(other_key.get_did().await?); @@ -270,7 +269,7 @@ mod tests { #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_implicitly_revokes_transitive_authorizations() -> Result<()> { let (mut sphere_context, mnemonic) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + simulated_sphere_context(Access::ReadWrite, None).await?; let other_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key())); let other_did = Did(other_key.get_did().await?); @@ -316,7 +315,7 @@ mod tests { #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_catches_revoked_authorizations_when_verifying() -> Result<()> { let (mut sphere_context, mnemonic) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + simulated_sphere_context(Access::ReadWrite, None).await?; let other_key = generate_ed25519_key(); let other_did = Did(other_key.get_did().await?); @@ -343,7 +342,7 @@ mod tests { #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_perform_access_recovery_given_a_mnemonic() -> Result<()> { let (mut sphere_context, mnemonic) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + simulated_sphere_context(Access::ReadWrite, None).await?; let owner = sphere_context.sphere_context().await?.author().clone(); diff --git a/rust/noosphere-core/src/context/context.rs b/rust/noosphere-core/src/context/context.rs index 53ade732a..c8f62e1f6 100644 --- a/rust/noosphere-core/src/context/context.rs +++ b/rust/noosphere-core/src/context/context.rs @@ -254,12 +254,12 @@ mod tests { use anyhow::Result; use crate::{ - authority::{generate_capability, generate_ed25519_key, SphereAbility}, + authority::{generate_capability, generate_ed25519_key, Access, SphereAbility}, context::{ HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite, }, data::{ContentType, LinkRecord, LINK_RECORD_FACT_NAME}, - helpers::{make_valid_link_record, simulated_sphere_context, SimulationAccess}, + helpers::{make_valid_link_record, simulated_sphere_context}, tracing::initialize_tracing, view::Sphere, }; @@ -279,8 +279,7 @@ mod tests { let valid_names: &[&str] = &["j@__/_大", "/"]; let invalid_names: &[&str] = &[""]; - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; for invalid_name in invalid_names { assert!(sphere_context @@ -306,8 +305,7 @@ mod tests { let valid_names: &[&str] = &["j@__/_大"]; let invalid_names: &[&str] = &["", "did:key:foo"]; - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut db = sphere_context.sphere_context().await?.db().clone(); let (other_identity, link_record, _) = make_valid_link_record(&mut db).await?; @@ -342,8 +340,7 @@ mod tests { async fn it_disallows_adding_self_as_petname() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let db = sphere_context.sphere_context().await?.db().clone(); let sphere_identity = sphere_context.identity().await?; @@ -387,8 +384,7 @@ mod tests { async fn it_disallows_adding_outdated_records() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut store = sphere_context.sphere_context().await?.db().clone(); // Generate two LinkRecords, the first one having a later expiry diff --git a/rust/noosphere-core/src/context/cursor.rs b/rust/noosphere-core/src/context/cursor.rs index 4e1aa6528..9e6612e3c 100644 --- a/rust/noosphere-core/src/context/cursor.rs +++ b/rust/noosphere-core/src/context/cursor.rs @@ -284,6 +284,7 @@ mod tests { wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); use crate::{ + authority::Access, context::{ HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, @@ -291,7 +292,6 @@ mod tests { data::{ContentType, Header}, helpers::{ make_sphere_context_with_peer_chain, make_valid_link_record, simulated_sphere_context, - SimulationAccess, }, tracing::initialize_tracing, }; @@ -301,7 +301,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_unlink_slugs_from_the_content_space() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -329,7 +329,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_flushes_on_every_save() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let initial_stats = { @@ -386,7 +386,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_does_not_allow_writes_when_an_author_has_read_only_access() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::Readonly, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadOnly, None) .await .unwrap(); @@ -407,7 +407,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_write_a_file_and_read_it_back() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -439,7 +439,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_overwrite_a_file_with_new_contents_and_preserve_history() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -496,7 +496,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_throws_an_error_when_saving_without_changes() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -510,7 +510,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_throws_an_error_when_saving_with_empty_mutation_and_empty_headers() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -524,8 +524,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_get_all_petnames_assigned_to_an_identity() -> Result<()> { - let (sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut db = UcanStore(sphere_context.sphere_context().await?.db().clone()); diff --git a/rust/noosphere-core/src/context/mod.rs b/rust/noosphere-core/src/context/mod.rs index c95cd0ab9..c54313828 100644 --- a/rust/noosphere-core/src/context/mod.rs +++ b/rust/noosphere-core/src/context/mod.rs @@ -12,15 +12,15 @@ //! //! ```rust //! # use anyhow::Result; -//! # use noosphere_core::context::{SphereCursor, HasMutableSphereContext, SphereContentWrite}; +//! # use noosphere_core::{authority::Access, context::{SphereCursor, HasMutableSphereContext, SphereContentWrite}}; //! # //! # #[cfg(feature = "helpers")] -//! # use noosphere_core::helpers::{simulated_sphere_context,SimulationAccess}; +//! # use noosphere_core::helpers::simulated_sphere_context; //! # //! # #[cfg(feature = "helpers")] //! # #[tokio::main(flavor = "multi_thread")] //! # async fn main() -> Result<()> { -//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; //! # //! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?; //! sphere_context.save(None).await?; @@ -38,7 +38,8 @@ //! # use anyhow::Result; //! # #[cfg(feature = "helpers")] //! # use noosphere_core::{ -//! # helpers::{simulated_sphere_context,SimulationAccess}, +//! # authority::Access, +//! # helpers::simulated_sphere_context, //! # data::Did, //! # context::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite} //! # }; @@ -46,7 +47,7 @@ //! # #[cfg(feature = "helpers")] //! # #[tokio::main(flavor = "multi_thread")] //! # async fn main() -> Result<()> { -//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; //! # //! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?; //! sphere_context.save(None).await?; diff --git a/rust/noosphere-core/src/context/walker.rs b/rust/noosphere-core/src/context/walker.rs index ef5e1a898..05277e44c 100644 --- a/rust/noosphere-core/src/context/walker.rs +++ b/rust/noosphere-core/src/context/walker.rs @@ -454,18 +454,19 @@ mod tests { wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); use crate::{ + authority::Access, context::{ HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SphereCursor, SphereWalker, }, data::{ContentType, Did}, - helpers::{simulated_sphere_context, SimulationAccess}, + helpers::simulated_sphere_context, }; #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_be_initialized_with_a_context_or_a_cursor() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context.clone()); @@ -501,7 +502,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_list_all_slugs_currently_in_a_sphere() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); @@ -541,8 +542,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_list_all_authorizations_currently_in_a_sphere() -> Result<()> { - let (sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut cursor = SphereCursor::latest(sphere_context); let authorizations_to_add = 10; @@ -565,7 +565,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_stream_the_whole_index() { - let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(sphere_context); diff --git a/rust/noosphere-core/src/helpers/context.rs b/rust/noosphere-core/src/helpers/context.rs index 05e06dbf2..6d2bd8734 100644 --- a/rust/noosphere-core/src/helpers/context.rs +++ b/rust/noosphere-core/src/helpers/context.rs @@ -4,12 +4,12 @@ use std::sync::Arc; use crate::{ - authority::{generate_capability, generate_ed25519_key, Author, SphereAbility}, + authority::{generate_capability, generate_ed25519_key, Access, Author, SphereAbility}, data::{ContentType, Did, LinkRecord, Mnemonic, LINK_RECORD_FACT_NAME}, view::Sphere, }; use anyhow::Result; -use noosphere_storage::{BlockStore, MemoryStorage, SphereDb, TrackingStorage, UcanStore}; +use noosphere_storage::{BlockStore, MemoryStorage, SphereDb, Storage, TrackingStorage, UcanStore}; use tokio::{io::AsyncReadExt, sync::Mutex}; use ucan::{builder::UcanBuilder, crypto::KeyMaterial}; @@ -21,27 +21,19 @@ use crate::{ stream::{walk_versioned_map_elements, walk_versioned_map_elements_and}, }; -/// Access levels available when simulating a [SphereContext] -pub enum SimulationAccess { - /// Access to the related [SphereContext] is read-only - Readonly, - /// Access to the related [SphereContext] is read+write - ReadWrite, -} - /// Create a temporary, non-persisted [SphereContext] that tracks usage /// internally. This is intended for use in docs and tests, and should otherwise -/// be ignored. When creating the simulated [SphereContext], you can pass a -/// [SimulationAccess] to control the kind of access the emphemeral credentials +/// be ignored. When creating the simulated [SphereContext], you can pass an +/// [Access] to control the kind of access the emphemeral credentials /// have to the [SphereContext]. pub async fn simulated_sphere_context( - profile: SimulationAccess, + profile: Access, db: Option>>, ) -> Result<( Arc>>>, Mnemonic, )> { - let mut db = match db { + let db = match db { Some(db) => db, None => { let storage_provider = TrackingStorage::wrap(MemoryStorage::default()); @@ -49,6 +41,15 @@ pub async fn simulated_sphere_context( } }; + generate_sphere_context(profile, db).await +} + +/// Generate a [SphereContext] using the storage provided, intended for tests and +/// benchmarks. You can pass a [Access] to control access. +pub async fn generate_sphere_context( + profile: Access, + mut db: SphereDb, +) -> Result<(Arc>>, Mnemonic)> { let owner_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key())); let owner_did = owner_key.get_did().await?; @@ -58,8 +59,8 @@ pub async fn simulated_sphere_context( let author = Author { key: owner_key, authorization: match profile { - SimulationAccess::Readonly => None, - SimulationAccess::ReadWrite => Some(proof), + Access::ReadOnly => None, + Access::ReadWrite => Some(proof), }, }; @@ -130,7 +131,7 @@ pub type TrackedHasMutableSphereContext = Arc Result<(TrackedHasMutableSphereContext, Vec)> { - let (origin_sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (origin_sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); @@ -144,10 +145,9 @@ pub async fn make_sphere_context_with_peer_chain( let mut contexts = vec![origin_sphere_context.clone()]; for name in peer_chain.iter() { - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, Some(db.clone())) - .await - .unwrap(); + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, Some(db.clone())) + .await + .unwrap(); sphere_context .write("my-name", &ContentType::Subtext, name.as_bytes(), None) diff --git a/rust/noosphere-core/src/stream/mod.rs b/rust/noosphere-core/src/stream/mod.rs index 36496667a..e8f16676f 100644 --- a/rust/noosphere-core/src/stream/mod.rs +++ b/rust/noosphere-core/src/stream/mod.rs @@ -18,14 +18,12 @@ mod tests { use ucan::store::UcanJwtStore; use crate::{ + authority::Access, context::{ HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite, }, data::{BodyChunkIpld, ContentType, LinkRecord, MemoIpld}, - helpers::{ - make_valid_link_record, simulated_sphere_context, touch_all_sphere_blocks, - SimulationAccess, - }, + helpers::{make_valid_link_record, simulated_sphere_context, touch_all_sphere_blocks}, stream::{from_car_stream, memo_body_stream, memo_history_stream, to_car_stream}, tracing::initialize_tracing, view::{BodyChunkDecoder, Sphere}, @@ -45,8 +43,7 @@ mod tests { async fn it_includes_all_link_records_and_proofs_from_the_address_book() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut db = sphere_context.sphere_context().await?.db().clone(); let (foo_did, foo_link_record, foo_link_record_link) = @@ -90,8 +87,7 @@ mod tests { async fn it_can_stream_all_blocks_in_a_sphere_version() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let changes = vec![ (vec!["dogs", "birds"], vec!["alice", "bob"]), @@ -169,8 +165,7 @@ mod tests { async fn it_can_stream_all_delta_blocks_for_a_range_of_history() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let changes = vec![ (vec!["dogs", "birds"], vec!["alice", "bob"]), @@ -256,8 +251,7 @@ mod tests { async fn it_can_stream_all_blocks_in_some_sphere_content() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let mut db = sphere_context.sphere_context().await?.db_mut().clone(); let chunks = [b"foo", b"bar", b"baz"]; @@ -312,8 +306,7 @@ mod tests { async fn it_can_stream_all_blocks_in_a_sphere_version_as_a_car() -> Result<()> { initialize_tracing(None); - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let changes = vec![ (vec!["dogs", "birds"], vec!["alice", "bob"]), diff --git a/rust/noosphere-gateway/src/authority.rs b/rust/noosphere-gateway/src/authority.rs index 7b81988df..696eca6c9 100644 --- a/rust/noosphere-gateway/src/authority.rs +++ b/rust/noosphere-gateway/src/authority.rs @@ -11,8 +11,7 @@ use axum::{ use libipld_core::cid::Cid; use noosphere_core::authority::{SphereAbility, SphereReference, SPHERE_SEMANTICS}; use noosphere_core::context::SphereContext; -use noosphere_storage::NativeStorage; - +use noosphere_storage::Storage; use tokio::sync::Mutex; use ucan::{capability::CapabilityView, chain::ProofChain, store::UcanJwtStore}; @@ -23,12 +22,16 @@ use super::GatewayScope; /// represented by a UCAN. Any request handler can use a GatewayAuthority /// to test if a required capability is satisfied by the authorization /// presented by the maker of the request. -pub struct GatewayAuthority { +pub struct GatewayAuthority { proof: ProofChain, scope: GatewayScope, + marker: std::marker::PhantomData, } -impl GatewayAuthority { +impl GatewayAuthority +where + S: Storage + 'static, +{ pub fn try_authorize( &self, capability: &CapabilityView, @@ -52,16 +55,17 @@ impl GatewayAuthority { } #[async_trait] -impl FromRequestParts for GatewayAuthority +impl FromRequestParts for GatewayAuthority where - S: Send + Sync, + State: Send + Sync, + S: Storage + 'static, { type Rejection = StatusCode; - async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + async fn from_request_parts(parts: &mut Parts, state: &State) -> Result { let sphere_context = parts .extensions - .get::>>>() + .get::>>>() .ok_or_else(|| { error!("Could not find DidParser in extensions"); StatusCode::INTERNAL_SERVER_ERROR @@ -141,6 +145,7 @@ where Ok(GatewayAuthority { scope: gateway_scope.clone(), proof: proof_chain, + marker: std::marker::PhantomData, }) } } diff --git a/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs index 8067394da..37acb6572 100644 --- a/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs @@ -20,7 +20,7 @@ use crate::{authority::GatewayAuthority, GatewayScope}; #[instrument(level = "debug", skip(authority, scope, sphere_context, ipfs_client))] pub async fn fetch_route( - authority: GatewayAuthority, + authority: GatewayAuthority, Query(FetchParameters { since }): Query, Extension(scope): Extension, Extension(ipfs_client): Extension, diff --git a/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs index 7d522d21f..818f3a15b 100644 --- a/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs @@ -8,7 +8,7 @@ use noosphere_storage::Storage; pub async fn identify_route( Extension(scope): Extension, Extension(sphere_context): Extension, - authority: GatewayAuthority, + authority: GatewayAuthority, ) -> Result where C: HasSphereContext, diff --git a/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs index 858f5a5d6..26d11da01 100644 --- a/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs @@ -36,7 +36,7 @@ use crate::{ ) )] pub async fn push_route( - authority: GatewayAuthority, + authority: GatewayAuthority, Extension(sphere_context): Extension, Extension(gateway_scope): Extension, Extension(syndication_tx): Extension>>, diff --git a/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs index 0d213fc11..5fb8cd533 100644 --- a/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs @@ -34,7 +34,7 @@ pub type ReplicationCarStreamBody = /// fetch from the gateway. #[instrument(level = "debug", skip(authority, scope, sphere_context,))] pub async fn replicate_route( - authority: GatewayAuthority, + authority: GatewayAuthority, // NOTE: Cannot go from string to CID via serde Path(memo_version): Path, Query(ReplicateParameters { since }): Query, @@ -158,10 +158,11 @@ mod tests { use super::is_allowed_to_replicate_incrementally; use anyhow::Result; + use noosphere_core::authority::Access; use noosphere_core::context::{ HasMutableSphereContext, HasSphereContext, SphereContext, SphereContextKey, SphereCursor, }; - use noosphere_core::helpers::{simulated_sphere_context, SimulationAccess}; + use noosphere_core::helpers::simulated_sphere_context; use noosphere_core::{ authority::{ generate_capability, generate_ed25519_key, Author, Authorization, SphereAbility, @@ -173,8 +174,7 @@ mod tests { #[tokio::test] async fn it_only_allows_incremental_replication_of_causally_ordered_revisions() -> Result<()> { - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let db = sphere_context.sphere_context().await?.db().clone(); let version_1 = sphere_context @@ -204,8 +204,7 @@ mod tests { #[tokio::test] async fn it_only_allows_incremental_replication_of_revisions_from_same_sphere() -> Result<()> { - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let db = sphere_context.sphere_context().await?.db().clone(); let version_1 = sphere_context @@ -220,7 +219,7 @@ mod tests { let memo_2 = version_2.load_from(&db).await?; let (mut other_sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, Some(db.clone())).await?; + simulated_sphere_context(Access::ReadWrite, Some(db.clone())).await?; let other_version_1 = other_sphere_context .save(Some(vec![("V".into(), "1".into())])) @@ -264,8 +263,7 @@ mod tests { let to_be_revoked_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key())); let to_be_revoked_did = to_be_revoked_key.get_did().await?; - let (mut sphere_context, _) = - simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (mut sphere_context, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let db = sphere_context.sphere_context().await?.db().clone(); let _ = sphere_context diff --git a/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs index 09b5f110c..03d2d8030 100644 --- a/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs @@ -42,7 +42,7 @@ use crate::{ ) )] pub async fn push_route( - authority: GatewayAuthority, + authority: GatewayAuthority, Extension(sphere_context): Extension, Extension(gateway_scope): Extension, Extension(syndication_tx): Extension>>, diff --git a/rust/noosphere-gateway/src/worker/name_system.rs b/rust/noosphere-gateway/src/worker/name_system.rs index ea255cce7..8fc7b649a 100644 --- a/rust/noosphere-gateway/src/worker/name_system.rs +++ b/rust/noosphere-gateway/src/worker/name_system.rs @@ -511,9 +511,9 @@ where #[cfg(test)] mod tests { use noosphere_core::{ - authority::{generate_capability, SphereAbility}, + authority::{generate_capability, Access, SphereAbility}, data::LINK_RECORD_FACT_NAME, - helpers::{simulated_sphere_context, SimulationAccess}, + helpers::simulated_sphere_context, }; use noosphere_ns::helpers::KeyValueNameResolver; use ucan::builder::UcanBuilder; @@ -523,7 +523,7 @@ mod tests { #[tokio::test] async fn it_publishes_to_the_name_system() -> Result<()> { let ipfs_url: Url = "http://127.0.0.1:5000".parse()?; - let (sphere, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let (sphere, _) = simulated_sphere_context(Access::ReadWrite, None).await?; let record: LinkRecord = { let context = sphere.lock().await; let identity: &str = context.identity().into(); diff --git a/rust/noosphere-into/src/into/html/sphere.rs b/rust/noosphere-into/src/into/html/sphere.rs index 86c6c7800..ef54e89c2 100644 --- a/rust/noosphere-into/src/into/html/sphere.rs +++ b/rust/noosphere-into/src/into/html/sphere.rs @@ -167,9 +167,10 @@ pub mod tests { use crate::write::MemoryWriteTarget; use noosphere_core::{ + authority::Access, context::{HasMutableSphereContext, SphereContentWrite, SphereCursor}, data::{ContentType, Header}, - helpers::{simulated_sphere_context, SimulationAccess}, + helpers::simulated_sphere_context, }; use super::sphere_into_html; @@ -177,7 +178,7 @@ pub mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_writes_a_file_from_the_sphere_to_the_target_as_html() { - let (context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(context); @@ -246,7 +247,7 @@ pub mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_symlinks_a_file_slug_to_the_latest_file_version() { - let (context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + let (context, _) = simulated_sphere_context(Access::ReadWrite, None) .await .unwrap(); let mut cursor = SphereCursor::latest(context); diff --git a/rust/noosphere-ns/Cargo.toml b/rust/noosphere-ns/Cargo.toml index 78993dcbe..faa7a7462 100644 --- a/rust/noosphere-ns/Cargo.toml +++ b/rust/noosphere-ns/Cargo.toml @@ -38,6 +38,7 @@ ucan-key-support = { workspace = true } tokio = { workspace = true, features = ["io-util", "io-std", "sync", "macros", "rt", "rt-multi-thread"] } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } noosphere-core = { version = "0.15.2", path = "../noosphere-core" } +noosphere-common = { version = "0.1.0", path = "../noosphere-common" } libp2p = { version = "0.51.3", default-features = false, features = [ "ed25519", "identify", "dns", "kad", "macros", "noise", "serde", "tcp", "tokio", "yamux" ] } void = { workspace = true } diff --git a/rust/noosphere-ns/src/dht/errors.rs b/rust/noosphere-ns/src/dht/errors.rs index e8fba16f2..cb4db64fc 100644 --- a/rust/noosphere-ns/src/dht/errors.rs +++ b/rust/noosphere-ns/src/dht/errors.rs @@ -1,6 +1,6 @@ -use crate::dht::channel::ChannelError; use anyhow; use libp2p::{kad, kad::record::store::Error as KadStorageError, TransportError}; +use noosphere_common::channel::ChannelError; use std::fmt; use std::io; diff --git a/rust/noosphere-ns/src/dht/mod.rs b/rust/noosphere-ns/src/dht/mod.rs index 471b3ec06..7ab5a9e30 100644 --- a/rust/noosphere-ns/src/dht/mod.rs +++ b/rust/noosphere-ns/src/dht/mod.rs @@ -1,4 +1,3 @@ -mod channel; mod config; mod errors; mod node; diff --git a/rust/noosphere-ns/src/dht/node.rs b/rust/noosphere-ns/src/dht/node.rs index e3f39ea7f..5c5899173 100644 --- a/rust/noosphere-ns/src/dht/node.rs +++ b/rust/noosphere-ns/src/dht/node.rs @@ -1,5 +1,4 @@ use crate::dht::{ - channel::message_channel, errors::DhtError, processor::DhtProcessor, rpc::{DhtMessageClient, DhtRequest, DhtResponse}, @@ -7,6 +6,7 @@ use crate::dht::{ DhtConfig, Validator, }; use libp2p::{identity::Keypair, Multiaddr, PeerId}; +use noosphere_common::channel::message_channel; use std::time::Duration; use tokio; @@ -208,7 +208,7 @@ impl DhtNode { async fn send_request(&self, request: DhtRequest) -> Result { self.client - .send_request_async(request) + .send(request) .await .map_err(DhtError::from) .and_then(|res| res) diff --git a/rust/noosphere-ns/src/dht/rpc.rs b/rust/noosphere-ns/src/dht/rpc.rs index 748f48990..eccd8679e 100644 --- a/rust/noosphere-ns/src/dht/rpc.rs +++ b/rust/noosphere-ns/src/dht/rpc.rs @@ -1,7 +1,7 @@ -use crate::dht::channel::{Message, MessageClient, MessageProcessor}; use crate::dht::errors::DhtError; use crate::dht::types::{DhtRecord, NetworkInfo, Peer}; use libp2p::Multiaddr; +use noosphere_common::channel::{Message, MessageClient, MessageProcessor}; use std::{fmt, str}; diff --git a/rust/noosphere-storage/Cargo.toml b/rust/noosphere-storage/Cargo.toml index 1a6042bdf..8655649e2 100644 --- a/rust/noosphere-storage/Cargo.toml +++ b/rust/noosphere-storage/Cargo.toml @@ -24,6 +24,7 @@ async-stream = { workspace = true } tokio-stream = { workspace = true } cid = { workspace = true } noosphere-common = { version = "0.1.0", path = "../noosphere-common" } +instant = { version = "0.1.12", features = ["wasm-bindgen"], optional = true } tracing = "~0.1" ucan = { workspace = true } libipld-core = { workspace = true } @@ -35,14 +36,24 @@ url = { version = "^2" } [dev-dependencies] witty-phrase-generator = "~0.2" wasm-bindgen-test = { workspace = true } +rand = { workspace = true } +# examples/bench +noosphere-core = { version = "0.15.2", path = "../noosphere-core", features = ["helpers"] } +# examples/bench +noosphere-common = { version = "0.1.0", path = "../noosphere-common", features = ["helpers"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tempfile = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] sled = "~0.34" tokio = { workspace = true, features = ["full"] } +rocksdb = { version = "0.21.0", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] tokio = { workspace = true, features = ["sync", "macros"] } -wasm-bindgen = { workspace = true } +wasm-bindgen = { workspace = true, features = ["serde-serialize"] } +wasm-bindgen-futures = { workspace = true } js-sys = { workspace = true } rexie = { version = "~0.4" } @@ -51,4 +62,15 @@ version = "~0.3" features = [ "Window", "DedicatedWorkerGlobalScope", + "StorageManager", + "Navigator", ] + +[features] +default = [] +rocksdb = ["dep:rocksdb"] +rocksdb-multi-thread = ["dep:rocksdb"] +performance = ["dep:instant"] + +[[example]] +name = "bench" diff --git a/rust/noosphere-storage/examples/bench/Cargo.toml b/rust/noosphere-storage/examples/bench/Cargo.toml new file mode 100644 index 000000000..abf460364 --- /dev/null +++ b/rust/noosphere-storage/examples/bench/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "noosphere-storage-benchmarks" +version = "1.0.0" +edition = "2021" +rust-version = "1.60.0" +license = "MIT OR Apache-2.0" +documentation = "https://docs.rs/noosphere-storage" +repository = "https://github.com/subconsciousnetwork/noosphere" +homepage = "https://github.com/subconsciousnetwork/noosphere" + +[dependencies] +noosphere-storage = { path = "../../", features = ["performance"] } +noosphere-core = { path = "../../../noosphere-core", features = ["helpers"] } +noosphere-common = { path = "../../../noosphere-common", features = ["helpers"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { workspace = true, features = ["full"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = { workspace = true } +wasm-bindgen-futures = { workspace = true } + +[features] +default = [] +rocksdb = ["noosphere-storage/rocksdb"] +rocksdb-mt = ["noosphere-storage/rocksdb-mt"] +sqlite = ["noosphere-storage/sqlite"] diff --git a/rust/noosphere-storage/examples/bench/main.rs b/rust/noosphere-storage/examples/bench/main.rs new file mode 100644 index 000000000..419e69a80 --- /dev/null +++ b/rust/noosphere-storage/examples/bench/main.rs @@ -0,0 +1,270 @@ +//! Benchmarking suite for comparing [Storage] providers. +//! Even though the `performance` feature is defined in the example manifest, +//! it still needs to be passed in when invoking from the parent crate. +//! +//! Run Sled: +//! `cargo run --example bench --features performance` +//! Run RocksDB: +//! `cargo run --example bench --features performance --features rocksdb` +//! Run IndexedDb (open `http://localhost:8000` in a browser) +//! `NO_HEADLESS=1 cargo run --example bench --features performance --target wasm32-unknown-unknown` + +use anyhow::Result; +use noosphere_common::helpers::TestEntropy; +use noosphere_core::{ + authority::Access, + context::{HasMutableSphereContext, SphereContentRead, SphereContentWrite}, + data::ContentType, + helpers::generate_sphere_context, + tracing::initialize_tracing, +}; +use noosphere_storage::{PerformanceStats, PerformanceStorage, SphereDb, Storage}; +use rand::Rng; +use tokio::io::AsyncReadExt; + +#[cfg(target_arch = "wasm32")] +macro_rules! output { + ($($v:expr),+) => { tracing::info!($($v,)*) } +} +#[cfg(not(target_arch = "wasm32"))] +macro_rules! output { + ($($v:expr),+) => { println!($($v,)*) } +} + +async fn create_sphere_with_long_history(db: SphereDb) -> Result<()> { + let (mut ctx, _) = generate_sphere_context(Access::ReadWrite, db).await?; + let entropy = TestEntropy::default(); + let rng_base = entropy.to_rng(); + let mut rng = rng_base.lock().await; + + // Long history, small-ish files + for _ in 0..100 { + let random_index = rng.gen_range(0..100); + let mut random_bytes = Vec::from(rng.gen::<[u8; 32]>()); + let slug = format!("slug{}", random_index); + + let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { + let mut file_bytes = Vec::new(); + file.contents.read_to_end(&mut file_bytes).await?; + file_bytes.append(&mut random_bytes); + file_bytes + } else { + random_bytes + }; + + ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) + .await?; + ctx.save(None).await?; + } + Ok(()) +} + +async fn create_sphere_with_large_files(db: SphereDb) -> Result<()> { + let (mut ctx, _) = generate_sphere_context(Access::ReadWrite, db).await?; + let entropy = TestEntropy::default(); + let rng_base = entropy.to_rng(); + let mut rng = rng_base.lock().await; + + // Modest history, large-ish files + for _ in 0..10 { + let mut random_bytes = (0..1000).fold(Vec::new(), |mut bytes, _| { + bytes.append(&mut Vec::from(rng.gen::<[u8; 32]>())); + bytes + }); + let random_index = rng.gen_range(0..10); + let slug = format!("slug{}", random_index); + + let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { + let mut file_bytes = Vec::new(); + file.contents.read_to_end(&mut file_bytes).await?; + file_bytes.append(&mut random_bytes); + file_bytes + } else { + random_bytes + }; + + ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) + .await?; + + ctx.save(None).await?; + } + Ok(()) +} + +#[cfg(all( + not(target_arch = "wasm32"), + not(any(feature = "rocksdb", feature = "sqlite")) +))] +type ActiveStoragePrimitive = noosphere_storage::SledStorage; +#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] +type ActiveStoragePrimitive = noosphere_storage::RocksDbStorage; +#[cfg(all( + not(target_arch = "wasm32"), + feature = "sqlite", + not(feature = "rocksdb") +))] +type ActiveStoragePrimitive = noosphere_storage::SqliteStorage; +#[cfg(target_arch = "wasm32")] +type ActiveStoragePrimitive = noosphere_storage::IndexedDbStorage; + +type ActiveStorageType = PerformanceStorage; + +struct BenchmarkStorage { + storage: ActiveStorageType, + #[cfg(not(target_arch = "wasm32"))] + _temp_dir: tempfile::TempDir, + name: String, +} +impl BenchmarkStorage { + pub async fn new() -> Result { + #[cfg(not(target_arch = "wasm32"))] + let temp_dir = tempfile::TempDir::new()?; + #[cfg(not(target_arch = "wasm32"))] + let storage_path = temp_dir.path(); + + #[cfg(all( + not(target_arch = "wasm32"), + not(any(feature = "rocksdb", feature = "sqlite")) + ))] + let (storage, storage_name) = { + ( + noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path( + storage_path.into(), + ))?, + "SledDbStorage", + ) + }; + + #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] + let (storage, storage_name) = { + ( + noosphere_storage::RocksDbStorage::new(storage_path.into())?, + "RocksDbStorage", + ) + }; + + #[cfg(target_arch = "wasm32")] + let (storage, storage_name) = { + let temp_name: String = witty_phrase_generator::WPGen::new() + .with_words(3) + .unwrap() + .into_iter() + .map(|word| String::from(word)) + .collect(); + ( + noosphere_storage::IndexedDbStorage::new(&temp_name).await?, + "IndexedDbStorage", + ) + }; + + let storage = PerformanceStorage::new(storage); + + #[cfg(not(target_arch = "wasm32"))] + { + Ok(BenchmarkStorage { + storage, + name: storage_name.to_owned(), + _temp_dir: temp_dir, + }) + } + #[cfg(target_arch = "wasm32")] + { + Ok(BenchmarkStorage { + storage, + name: storage_name.to_owned(), + }) + } + } + + pub async fn sphere_db(&self) -> Result> { + SphereDb::new(&self.storage).await + } + + pub async fn to_stats(&mut self) -> Result { + self.storage.to_stats().await + } + + /// Cleanup the storage. Tempdirs handle native implementations, + /// wipe any IndexedDb usage here. + pub async fn dispose(self) -> Result<()> { + #[cfg(target_arch = "wasm32")] + self.storage.to_inner().clear().await?; + Ok(()) + } +} + +async fn log_perf_stats(stats: &PerformanceStats) { + output!("reads: {} (avg {}us)", stats.reads.count, stats.reads.mean); + output!( + "writes: {} (avg {}us)", + stats.writes.count, + stats.writes.mean + ); + output!( + "removes: {} (avg {}us)", + stats.removes.count, + stats.removes.mean + ); + output!( + "flushes: {} (avg {}us)", + stats.flushes.count, + stats.flushes.mean + ); + output!("logical bytes: {}", stats.logical_bytes_stored); + output!("physical bytes: {}", stats.physical_bytes_stored); + + let space_amplification = if stats.logical_bytes_stored == 0 { + 0.0 + } else { + stats.physical_bytes_stored as f64 / stats.logical_bytes_stored as f64 + }; + output!("space amplification: {}", space_amplification); +} + +async fn bench_sphere_writing_long_history() { + let mut storage = BenchmarkStorage::new().await.unwrap(); + output!("Testing {}", storage.name); + + let db = storage.sphere_db().await.unwrap(); + create_sphere_with_long_history(db).await.unwrap(); + let stats = storage.to_stats().await.unwrap(); + log_perf_stats(&stats).await; + storage.dispose().await.unwrap(); +} + +async fn bench_sphere_writing_large_files() { + let mut storage = BenchmarkStorage::new().await.unwrap(); + output!("Testing {}", storage.name); + + let db = storage.sphere_db().await.unwrap(); + create_sphere_with_large_files(db).await.unwrap(); + let stats = storage.to_stats().await.unwrap(); + log_perf_stats(&stats).await; + storage.dispose().await.unwrap(); +} + +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::wasm_bindgen_test; +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + +#[cfg(target_arch = "wasm32")] +pub fn main() {} + +#[cfg(target_arch = "wasm32")] +#[wasm_bindgen_test] +async fn main_js() { + initialize_tracing(None); + + bench_sphere_writing_long_history().await; + bench_sphere_writing_large_files().await; +} + +#[cfg(not(target_arch = "wasm32"))] +#[tokio::main(flavor = "multi_thread")] +pub async fn main() { + initialize_tracing(None); + + bench_sphere_writing_long_history().await; + bench_sphere_writing_large_files().await; +} diff --git a/rust/noosphere-storage/src/helpers.rs b/rust/noosphere-storage/src/helpers.rs index 0e423a831..5c531ef3c 100644 --- a/rust/noosphere-storage/src/helpers.rs +++ b/rust/noosphere-storage/src/helpers.rs @@ -2,10 +2,10 @@ use crate::Storage; use anyhow::Result; #[cfg(not(target_arch = "wasm32"))] -use crate::{NativeStorage, NativeStorageInit, NativeStore}; +use crate::{SledStorage, SledStorageInit, SledStore}; #[cfg(not(target_arch = "wasm32"))] -pub async fn make_disposable_store() -> Result { +pub async fn make_disposable_store() -> Result { let temp_dir = std::env::temp_dir(); let temp_name: String = witty_phrase_generator::WPGen::new() .with_words(3) @@ -13,16 +13,15 @@ pub async fn make_disposable_store() -> Result { .into_iter() .map(String::from) .collect(); - let db = sled::open(temp_dir.join(temp_name)).unwrap(); - let provider = NativeStorage::new(NativeStorageInit::Db(db))?; + let provider = SledStorage::new(SledStorageInit::Path(temp_dir.join(temp_name)))?; provider.get_block_store("foo").await } #[cfg(target_arch = "wasm32")] -use crate::{WebStorage, WebStore}; +use crate::{IndexedDbStorage, IndexedDbStore}; #[cfg(target_arch = "wasm32")] -pub async fn make_disposable_store() -> Result { +pub async fn make_disposable_store() -> Result { let temp_name: String = witty_phrase_generator::WPGen::new() .with_words(3) .unwrap() @@ -30,6 +29,6 @@ pub async fn make_disposable_store() -> Result { .map(|word| String::from(word)) .collect(); - let provider = WebStorage::new(&temp_name).await?; + let provider = IndexedDbStorage::new(&temp_name).await?; provider.get_block_store(crate::db::BLOCK_STORE).await } diff --git a/rust/noosphere-storage/src/implementation/web.rs b/rust/noosphere-storage/src/implementation/indexed_db.rs similarity index 50% rename from rust/noosphere-storage/src/implementation/web.rs rename to rust/noosphere-storage/src/implementation/indexed_db.rs index bbaea4ea9..6327108da 100644 --- a/rust/noosphere-storage/src/implementation/web.rs +++ b/rust/noosphere-storage/src/implementation/indexed_db.rs @@ -1,6 +1,6 @@ use crate::store::Store; use crate::{db::SPHERE_DB_STORE_NAMES, storage::Storage}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Error, Result}; use async_trait::async_trait; use js_sys::Uint8Array; use rexie::{ @@ -12,17 +12,18 @@ use wasm_bindgen::{JsCast, JsValue}; pub const INDEXEDDB_STORAGE_VERSION: u32 = 1; #[derive(Clone)] -pub struct WebStorage { +pub struct IndexedDbStorage { db: Rc, + name: String, } -impl Debug for WebStorage { +impl Debug for IndexedDbStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WebStorage").finish() + f.debug_struct("IndexedDbStorage").finish() } } -impl WebStorage { +impl IndexedDbStorage { pub async fn new(db_name: &str) -> Result { Self::configure(INDEXEDDB_STORAGE_VERSION, db_name, SPHERE_DB_STORE_NAMES).await } @@ -39,10 +40,13 @@ impl WebStorage { .await .map_err(|error| anyhow!("{:?}", error))?; - Ok(WebStorage { db: Rc::new(db) }) + Ok(IndexedDbStorage { + db: Rc::new(db), + name: db_name.to_owned(), + }) } - async fn get_store(&self, name: &str) -> Result { + async fn get_store(&self, name: &str) -> Result { if self .db .store_names() @@ -53,18 +57,29 @@ impl WebStorage { return Err(anyhow!("No such store named {}", name)); } - Ok(WebStore { + Ok(IndexedDbStore { db: self.db.clone(), store_name: name.to_string(), }) } + + /// Closes and clears the database from origin storage. + pub async fn clear(self) -> Result<()> { + let name = self.name; + let db = Rc::into_inner(self.db) + .ok_or_else(|| anyhow!("Could not unwrap inner during database clear."))?; + db.close(); + Rexie::delete(&name) + .await + .map_err(|error| anyhow!("{:?}", error)) + } } #[async_trait(?Send)] -impl Storage for WebStorage { - type BlockStore = WebStore; +impl Storage for IndexedDbStorage { + type BlockStore = IndexedDbStore; - type KeyValueStore = WebStore; + type KeyValueStore = IndexedDbStore; async fn get_block_store(&self, name: &str) -> Result { self.get_store(name).await @@ -76,12 +91,12 @@ impl Storage for WebStorage { } #[derive(Clone)] -pub struct WebStore { +pub struct IndexedDbStore { db: Rc, store_name: String, } -impl WebStore { +impl IndexedDbStore { fn start_transaction(&self, mode: TransactionMode) -> Result<(IdbStore, Transaction)> { let tx = self .db @@ -116,7 +131,7 @@ impl WebStore { } async fn read(key: &JsValue, store: &IdbStore) -> Result>> { - Ok(match WebStore::contains(&key, &store).await? { + Ok(match IndexedDbStore::contains(&key, &store).await? { true => Some( store .get(&key) @@ -132,14 +147,14 @@ impl WebStore { } #[async_trait(?Send)] -impl Store for WebStore { +impl Store for IndexedDbStore { async fn read(&self, key: &[u8]) -> Result>> { let (store, tx) = self.start_transaction(TransactionMode::ReadOnly)?; - let key = WebStore::bytes_to_typed_array(key)?; + let key = IndexedDbStore::bytes_to_typed_array(key)?; - let maybe_dag = WebStore::read(&key, &store).await?; + let maybe_dag = IndexedDbStore::read(&key, &store).await?; - WebStore::finish_transaction(tx).await?; + IndexedDbStore::finish_transaction(tx).await?; Ok(maybe_dag) } @@ -147,17 +162,17 @@ impl Store for WebStore { async fn write(&mut self, key: &[u8], bytes: &[u8]) -> Result>> { let (store, tx) = self.start_transaction(TransactionMode::ReadWrite)?; - let key = WebStore::bytes_to_typed_array(key)?; - let value = WebStore::bytes_to_typed_array(bytes)?; + let key = IndexedDbStore::bytes_to_typed_array(key)?; + let value = IndexedDbStore::bytes_to_typed_array(bytes)?; - let old_bytes = WebStore::read(&key, &store).await?; + let old_bytes = IndexedDbStore::read(&key, &store).await?; store .put(&value, Some(&key)) .await .map_err(|error| anyhow!("{:?}", error))?; - WebStore::finish_transaction(tx).await?; + IndexedDbStore::finish_transaction(tx).await?; Ok(old_bytes) } @@ -165,17 +180,86 @@ impl Store for WebStore { async fn remove(&mut self, key: &[u8]) -> Result>> { let (store, tx) = self.start_transaction(TransactionMode::ReadWrite)?; - let key = WebStore::bytes_to_typed_array(key)?; + let key = IndexedDbStore::bytes_to_typed_array(key)?; - let old_value = WebStore::read(&key, &store).await?; + let old_value = IndexedDbStore::read(&key, &store).await?; store .delete(&key) .await .map_err(|error| anyhow!("{:?}", error))?; - WebStore::finish_transaction(tx).await?; + IndexedDbStore::finish_transaction(tx).await?; Ok(old_value) } } + +#[cfg(feature = "performance")] +struct SpaceUsageError(Error); + +#[cfg(feature = "performance")] +impl From for SpaceUsageError { + fn from(value: JsValue) -> SpaceUsageError { + if let Ok(js_string) = js_sys::JSON::stringify(&value) { + SpaceUsageError(anyhow!("{}", js_string.as_string().unwrap())) + } else { + SpaceUsageError(anyhow!("Could not parse JsValue error as string.")) + } + } +} + +#[cfg(feature = "performance")] +impl From for Error { + fn from(value: SpaceUsageError) -> Self { + value.0 + } +} + +#[cfg(feature = "performance")] +use serde; + +#[cfg(feature = "performance")] +#[derive(Debug, serde::Deserialize)] +pub struct StorageEstimate { + pub quota: u64, + pub usage: u64, + #[serde(rename = "usageDetails")] + pub usage_details: Option, +} + +#[cfg(feature = "performance")] +#[derive(Debug, serde::Deserialize)] +pub struct UsageDetails { + #[serde(rename = "indexedDB")] + pub indexed_db: u64, +} + +#[cfg(feature = "performance")] +#[async_trait(?Send)] +impl crate::Space for IndexedDbStorage { + /// Returns an estimate of disk usage of the IndexedDb instance. + /// Note this includes all storage usage for the active origin -- it is up + /// to the consumer to clear unwanted databases for more accurate reporting. + /// https://developer.mozilla.org/en-US/docs/Web/API/StorageManager/estimate + /// + /// A benign(?) warning is emitted by an underlying dependency from this: + /// https://github.com/DioxusLabs/cli/issues/62 + async fn get_space_usage(&self) -> Result { + let window = web_sys::window().ok_or_else(|| anyhow!("No window found."))?; + let storage = window.navigator().storage(); + let promise = storage + .estimate() + .map_err(|e| >::into(e))?; + let estimate_obj = wasm_bindgen_futures::JsFuture::from(promise) + .await + .map_err(|e| >::into(e))?; + let estimate: StorageEstimate = estimate_obj.into_serde()?; + + if let Some(details) = estimate.usage_details { + Ok(details.indexed_db) + } else { + Ok(estimate.usage) + } + } +} diff --git a/rust/noosphere-storage/src/implementation/memory.rs b/rust/noosphere-storage/src/implementation/memory.rs index 75e5335f6..8fd92e692 100644 --- a/rust/noosphere-storage/src/implementation/memory.rs +++ b/rust/noosphere-storage/src/implementation/memory.rs @@ -130,3 +130,19 @@ impl Store for MemoryStore { Ok(dags.remove(key)) } } + +#[cfg(feature = "performance")] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl crate::Space for MemoryStorage { + async fn get_space_usage(&self) -> Result { + let mut size = 0; + for (_, store) in self.stores.lock().await.iter() { + for (key, entry) in store.entries.lock().await.iter() { + size += key.len() as u64; + size += entry.len() as u64; + } + } + Ok(size) + } +} diff --git a/rust/noosphere-storage/src/implementation/mod.rs b/rust/noosphere-storage/src/implementation/mod.rs index f45005436..ab563f67e 100644 --- a/rust/noosphere-storage/src/implementation/mod.rs +++ b/rust/noosphere-storage/src/implementation/mod.rs @@ -5,13 +5,21 @@ pub use memory::*; pub use tracking::*; #[cfg(not(target_arch = "wasm32"))] -mod native; - +mod sled; #[cfg(not(target_arch = "wasm32"))] -pub use native::*; +pub use self::sled::*; -#[cfg(target_arch = "wasm32")] -mod web; +#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] +mod rocks_db; +#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] +pub use rocks_db::*; #[cfg(target_arch = "wasm32")] -pub use web::*; +mod indexed_db; +#[cfg(target_arch = "wasm32")] +pub use indexed_db::*; + +#[cfg(feature = "performance")] +mod performance; +#[cfg(feature = "performance")] +pub use performance::*; diff --git a/rust/noosphere-storage/src/implementation/performance.rs b/rust/noosphere-storage/src/implementation/performance.rs new file mode 100644 index 000000000..5a763776e --- /dev/null +++ b/rust/noosphere-storage/src/implementation/performance.rs @@ -0,0 +1,208 @@ +use crate::{store::Store, Space, Storage}; +use anyhow::{Error, Result}; +use async_trait::async_trait; +use instant::{Duration, Instant}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::Mutex; + +#[derive(Debug, Clone, Default)] +pub struct PerformanceStats { + pub reads: PerformanceAnalysis, + pub writes: PerformanceAnalysis, + pub removes: PerformanceAnalysis, + pub flushes: PerformanceAnalysis, + pub logical_bytes_stored: u64, + pub physical_bytes_stored: u64, +} + +#[derive(Debug, Clone, Default)] +pub struct PerformanceAnalysis { + pub mean: f64, + pub count: usize, +} + +impl TryFrom> for PerformanceAnalysis { + type Error = Error; + fn try_from(value: Vec) -> Result { + let mut durations_us: Vec<_> = value + .into_iter() + .map(|d| TryInto::::try_into(d.as_micros())) + .collect::, std::num::TryFromIntError>>()?; + durations_us.sort(); + let count = durations_us.len(); + let mean = if count == 0 { + 0.0 + } else { + durations_us.iter().sum::() as f64 / count as f64 + }; + Ok(Self { count, mean }) + } +} + +impl TryFrom for PerformanceStats { + type Error = Error; + fn try_from(value: InternalStoreStats) -> Result { + let mut stats = PerformanceStats::default(); + stats.reads = value.reads.try_into()?; + stats.writes = value.writes.try_into()?; + stats.removes = value.removes.try_into()?; + stats.flushes = value.flushes.try_into()?; + stats.logical_bytes_stored = value.logical_bytes_stored; + Ok(stats) + } +} + +#[derive(Debug, Default)] +struct InternalStoreStats { + pub reads: Vec, + pub writes: Vec, + pub removes: Vec, + pub flushes: Vec, + pub logical_bytes_stored: u64, +} + +/// A wrapper for [Storage] types that tracks performance +/// of various operations. +/// If [Storage] is also [Space], [PerformanceStorage::to_stats] can be +/// called to get performance data as [PerformanceStats]. +#[derive(Clone, Debug)] +pub struct PerformanceStorage { + storage: S, + stats: Arc>>>>, +} + +impl PerformanceStorage +where + S: Storage, + S::KeyValueStore: Store, + S::BlockStore: Store, +{ + pub fn new(other: S) -> Self { + PerformanceStorage { + storage: other, + stats: Arc::new(Mutex::new(HashMap::default())), + } + } + + async fn get_store_stats(&self, name: &str) -> Arc> { + let mut storage_stats = self.stats.lock().await; + let store_name = name.to_owned(); + if let Some(stats) = storage_stats.get(&store_name) { + stats.to_owned() + } else { + let store_stats = Arc::new(Mutex::new(InternalStoreStats::default())); + storage_stats.insert(store_name, store_stats.clone()); + store_stats + } + } + + pub fn to_inner(self) -> S { + self.storage + } +} + +impl PerformanceStorage +where + S: Storage + Space, + S::KeyValueStore: Store, + S::BlockStore: Store, +{ + /// Drains the storage stats and returns a summary + /// of operations as [PerformanceStats]. + pub async fn to_stats(&mut self) -> Result { + let storage_stats = self.stats.lock().await; + let mut agg = InternalStoreStats::default(); + for (_, store_stats) in storage_stats.iter() { + let mut stats = store_stats.lock().await; + agg.reads.append(&mut stats.reads); + agg.writes.append(&mut stats.writes); + agg.removes.append(&mut stats.removes); + agg.flushes.append(&mut stats.flushes); + agg.logical_bytes_stored += stats.logical_bytes_stored; + } + + let mut perf_stats: PerformanceStats = agg.try_into()?; + perf_stats.physical_bytes_stored = self.storage.get_space_usage().await?; + + Ok(perf_stats) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl Storage for PerformanceStorage +where + S: Storage, + S::KeyValueStore: Store, + S::BlockStore: Store, +{ + type BlockStore = PerformanceStore; + type KeyValueStore = PerformanceStore; + + async fn get_block_store(&self, name: &str) -> Result { + let stats = self.get_store_stats(name).await; + let store = self.storage.get_block_store(name).await?; + let block_store = PerformanceStore { store, stats }; + Ok(block_store) + } + + async fn get_key_value_store(&self, name: &str) -> Result { + let stats = self.get_store_stats(name).await; + let store = self.storage.get_key_value_store(name).await?; + let key_value_store = PerformanceStore { store, stats }; + Ok(key_value_store) + } +} + +#[derive(Debug, Clone)] +pub struct PerformanceStore { + stats: Arc>, + store: S, +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl Store for PerformanceStore { + async fn read(&self, key: &[u8]) -> Result>> { + let start = Instant::now(); + let value = self.store.read(key).await?; + let duration = start.elapsed(); + + let mut stats = self.stats.lock().await; + stats.reads.push(duration); + Ok(value) + } + + async fn write(&mut self, key: &[u8], bytes: &[u8]) -> Result>> { + let start = Instant::now(); + let result = self.store.write(key, bytes).await; + let duration = start.elapsed(); + + let mut stats = self.stats.lock().await; + stats.writes.push(duration); + stats.logical_bytes_stored += bytes.len() as u64; + result + } + + async fn remove(&mut self, key: &[u8]) -> Result>> { + let start = Instant::now(); + let value = self.store.remove(key).await?; + let duration = start.elapsed(); + + let mut stats = self.stats.lock().await; + if let Some(bytes) = &value { + stats.logical_bytes_stored -= bytes.len() as u64; + } + stats.removes.push(duration); + Ok(value) + } + + async fn flush(&self) -> Result<()> { + let start = Instant::now(); + let result = self.store.flush().await; + let duration = start.elapsed(); + let mut stats = self.stats.lock().await; + stats.flushes.push(duration); + result + } +} diff --git a/rust/noosphere-storage/src/implementation/rocks_db.rs b/rust/noosphere-storage/src/implementation/rocks_db.rs new file mode 100644 index 000000000..2a787e66f --- /dev/null +++ b/rust/noosphere-storage/src/implementation/rocks_db.rs @@ -0,0 +1,148 @@ +use crate::{storage::Storage, store::Store, SPHERE_DB_STORE_NAMES}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use rocksdb::{ColumnFamilyDescriptor, DBWithThreadMode, Options}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +#[cfg(not(feature = "rocksdb-multi-thread"))] +type DbInner = DBWithThreadMode; +#[cfg(not(feature = "rocksdb-multi-thread"))] +type ColumnType<'a> = &'a rocksdb::ColumnFamily; +#[cfg(feature = "rocksdb-multi-thread")] +type DbInner = DBWithThreadMode; +#[cfg(feature = "rocksdb-multi-thread")] +type ColumnType<'a> = Arc>; + +/// A RocksDB implementation of [Storage]. +/// +/// Caveats: +/// * Values are limited to 4GB(?) [https://github.com/facebook/rocksdb/wiki/Basic-Operations#reads] +/// TODO(#631): Further improvements to the implementation. +#[derive(Clone, Debug)] +pub struct RocksDbStorage { + db: Arc, + #[allow(unused)] + path: PathBuf, +} + +impl RocksDbStorage { + pub fn new(path: PathBuf) -> Result { + std::fs::create_dir_all(&path)?; + let canonicalized = path.canonicalize()?; + let db = Arc::new(RocksDbStorage::init_db(canonicalized.clone())?); + Ok(RocksDbStorage { + db, + path: canonicalized, + }) + } + + async fn get_store(&self, name: &str) -> Result { + if SPHERE_DB_STORE_NAMES + .iter() + .find(|val| **val == name) + .is_none() + { + return Err(anyhow!("No such store named {}", name)); + } + + RocksDbStore::new(self.db.clone(), name.to_owned()) + } + + /// Configures a databasea at `path` and initializes the expected configurations. + fn init_db>(path: P) -> Result { + let mut cfs: Vec = Vec::with_capacity(SPHERE_DB_STORE_NAMES.len()); + + for store_name in SPHERE_DB_STORE_NAMES { + // https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + let cf_opts = Options::default(); + cfs.push(ColumnFamilyDescriptor::new(*store_name, cf_opts)); + } + + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + + Ok(DbInner::open_cf_descriptors(&db_opts, path, cfs)?) + } +} + +#[async_trait] +impl Storage for RocksDbStorage { + type BlockStore = RocksDbStore; + type KeyValueStore = RocksDbStore; + + async fn get_block_store(&self, name: &str) -> Result { + self.get_store(name).await + } + + async fn get_key_value_store(&self, name: &str) -> Result { + self.get_store(name).await + } +} + +#[derive(Clone)] +pub struct RocksDbStore { + name: String, + db: Arc, +} + +impl RocksDbStore { + pub fn new(db: Arc, name: String) -> Result { + Ok(RocksDbStore { db, name }) + } + + /// Returns the column family handle. Unfortunately generated on every call + /// due to not being `Sync`, potentially `unsafe` alternatives: + /// https://github.com/rust-rocksdb/rust-rocksdb/issues/407 + /// TODO(#631): Further improvements to the implementation. + fn cf_handle<'a>(&'a self) -> Result { + self.db + .cf_handle(&self.name) + .ok_or_else(move || anyhow!("Could not open handle for {}", self.name)) + } +} + +#[async_trait] +impl Store for RocksDbStore { + async fn read(&self, key: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-multi-thread")] + let cf = &cf; + Ok(self.db.get_cf(cf, key)?) + } + + async fn write(&mut self, key: &[u8], bytes: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-multi-thread")] + let cf = &cf; + let old_bytes = self.db.get_cf(cf, key)?; + self.db.put_cf(cf, key, bytes)?; + Ok(old_bytes) + } + + async fn remove(&mut self, key: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-multi-thread")] + let cf = &cf; + let old_bytes = self.db.get_cf(cf, key)?; + self.db.delete_cf(cf, key)?; + Ok(old_bytes) + } + + async fn flush(&self) -> Result<()> { + // With the use of WAL, we do not want to actively flush on every sync, + // and instead allow RocksDB to determine when to flush to OS. + Ok(()) + } +} + +#[cfg(feature = "performance")] +#[async_trait] +impl crate::Space for RocksDbStorage { + async fn get_space_usage(&self) -> Result { + crate::get_dir_size(&self.path).await + } +} diff --git a/rust/noosphere-storage/src/implementation/native.rs b/rust/noosphere-storage/src/implementation/sled.rs similarity index 61% rename from rust/noosphere-storage/src/implementation/native.rs rename to rust/noosphere-storage/src/implementation/sled.rs index 62bf1658c..f1a116bf8 100644 --- a/rust/noosphere-storage/src/implementation/native.rs +++ b/rust/noosphere-storage/src/implementation/sled.rs @@ -7,39 +7,43 @@ use anyhow::Result; use async_trait::async_trait; use sled::{Db, Tree}; -pub enum NativeStorageInit { +pub enum SledStorageInit { Path(PathBuf), Db(Db), } #[derive(Clone, Debug)] -pub struct NativeStorage { +pub struct SledStorage { db: Db, + #[allow(unused)] + path: Option, } -impl NativeStorage { - pub fn new(init: NativeStorageInit) -> Result { +impl SledStorage { + pub fn new(init: SledStorageInit) -> Result { + let mut db_path = None; let db: Db = match init { - NativeStorageInit::Path(path) => { + SledStorageInit::Path(path) => { std::fs::create_dir_all(&path)?; + db_path = Some(path.clone().canonicalize()?); sled::open(path)? } - NativeStorageInit::Db(db) => db, + SledStorageInit::Db(db) => db, }; - Ok(NativeStorage { db }) + Ok(SledStorage { db, path: db_path }) } - async fn get_store(&self, name: &str) -> Result { - Ok(NativeStore::new(&self.db.open_tree(name)?)) + async fn get_store(&self, name: &str) -> Result { + Ok(SledStore::new(&self.db.open_tree(name)?)) } } #[async_trait] -impl Storage for NativeStorage { - type BlockStore = NativeStore; +impl Storage for SledStorage { + type BlockStore = SledStore; - type KeyValueStore = NativeStore; + type KeyValueStore = SledStore; async fn get_block_store(&self, name: &str) -> Result { self.get_store(name).await @@ -51,18 +55,18 @@ impl Storage for NativeStorage { } #[derive(Clone)] -pub struct NativeStore { +pub struct SledStore { db: Tree, } -impl NativeStore { +impl SledStore { pub fn new(db: &Tree) -> Self { - NativeStore { db: db.clone() } + SledStore { db: db.clone() } } } #[async_trait] -impl Store for NativeStore { +impl Store for SledStore { async fn read(&self, key: &[u8]) -> Result>> { Ok(self.db.get(key)?.map(|entry| entry.to_vec())) } @@ -94,8 +98,22 @@ impl Store for NativeStore { } } -impl Drop for NativeStorage { +impl Drop for SledStorage { fn drop(&mut self) { let _ = self.db.flush(); } } + +#[cfg(feature = "performance")] +#[async_trait] +impl crate::Space for SledStorage { + async fn get_space_usage(&self) -> Result { + if let Some(path) = &self.path { + crate::get_dir_size(path).await + } else { + Err(anyhow::anyhow!( + "Could not calculate storage space, requires usage of a path constructor." + )) + } + } +} diff --git a/rust/noosphere-storage/src/key_value.rs b/rust/noosphere-storage/src/key_value.rs index 87f2d5b44..0f218af6e 100644 --- a/rust/noosphere-storage/src/key_value.rs +++ b/rust/noosphere-storage/src/key_value.rs @@ -2,63 +2,40 @@ use std::fmt::Display; use anyhow::{anyhow, Result}; use async_trait::async_trait; +use noosphere_common::{ConditionalSend, ConditionalSync}; use serde::{de::DeserializeOwned, Serialize}; -#[cfg(not(target_arch = "wasm32"))] -pub trait KeyValueSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl KeyValueSendSync for T where T: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait KeyValueSendSync {} - -#[cfg(target_arch = "wasm32")] -impl KeyValueSendSync for T {} - -#[cfg(not(target_arch = "wasm32"))] -pub trait KeyValueStoreSend: Send {} - -#[cfg(not(target_arch = "wasm32"))] -impl KeyValueStoreSend for T where T: Send {} - -#[cfg(target_arch = "wasm32")] -pub trait KeyValueStoreSend {} - -#[cfg(target_arch = "wasm32")] -impl KeyValueStoreSend for T {} - /// A [KeyValueStore] is a construct that is suitable for persisting generic /// key/value data to a storage backend. #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait KeyValueStore: Clone + KeyValueSendSync { +pub trait KeyValueStore: Clone + ConditionalSync { /// Given some key that can be realized as bytes, persist a serializable /// value to storage so that it can later be retrieved by that key async fn set_key(&mut self, key: K, value: V) -> Result<()> where - K: AsRef<[u8]> + KeyValueStoreSend, - V: Serialize + KeyValueStoreSend; + K: AsRef<[u8]> + ConditionalSend, + V: Serialize + ConditionalSend; /// Given some key that can be realized as bytes, retrieve some data that /// can be deserialized as the intended data structure async fn get_key(&self, key: K) -> Result> where - K: AsRef<[u8]> + KeyValueStoreSend, - V: DeserializeOwned + KeyValueStoreSend; + K: AsRef<[u8]> + ConditionalSend, + V: DeserializeOwned + ConditionalSend; /// Given some key that can be realized as bytes, unset the value stored /// against that key (if any) async fn unset_key(&mut self, key: K) -> Result<()> where - K: AsRef<[u8]> + KeyValueStoreSend; + K: AsRef<[u8]> + ConditionalSend; /// Same as get_key, but returns an error if no value is found to be stored /// against the key async fn require_key(&self, key: K) -> Result where - K: AsRef<[u8]> + KeyValueStoreSend + Display, - V: DeserializeOwned + KeyValueStoreSend, + K: AsRef<[u8]> + ConditionalSend + Display, + V: DeserializeOwned + ConditionalSend, { let required = key.to_string(); diff --git a/rust/noosphere-storage/src/lib.rs b/rust/noosphere-storage/src/lib.rs index 34ea6b44d..8d357a87a 100644 --- a/rust/noosphere-storage/src/lib.rs +++ b/rust/noosphere-storage/src/lib.rs @@ -29,6 +29,11 @@ pub use storage::*; pub use store::*; pub use tap::*; +#[cfg(feature = "performance")] +mod space; +#[cfg(feature = "performance")] +pub use space::*; + #[cfg(test)] pub mod helpers; diff --git a/rust/noosphere-storage/src/space.rs b/rust/noosphere-storage/src/space.rs new file mode 100644 index 000000000..393146d58 --- /dev/null +++ b/rust/noosphere-storage/src/space.rs @@ -0,0 +1,36 @@ +use anyhow::Result; +use async_trait::async_trait; +use noosphere_common::ConditionalSend; + +/// [Space] is a general trait for a storage provider to provide +/// a the size on disk, used to calculate space amplification. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait Space: ConditionalSend { + /// Get the underlying (e.g. disk) space usage of a storage provider. + async fn get_space_usage(&self) -> Result; +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) async fn get_dir_size(path: impl Into) -> Result { + use std::{future::Future, pin::Pin}; + + let path = path.into(); + fn dir_size(mut dir: tokio::fs::ReadDir) -> Pin> + Send>> { + Box::pin(async move { + let mut total_size = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + dir_size(tokio::fs::read_dir(entry.path()).await?).await? + } + data => data.len(), + }; + total_size += size; + } + Ok(total_size) + }) + } + + dir_size(tokio::fs::read_dir(path).await?).await +} diff --git a/rust/noosphere-storage/src/storage.rs b/rust/noosphere-storage/src/storage.rs index 4c104e090..86a585596 100644 --- a/rust/noosphere-storage/src/storage.rs +++ b/rust/noosphere-storage/src/storage.rs @@ -2,20 +2,9 @@ use crate::block::BlockStore; use crate::key_value::KeyValueStore; use anyhow::Result; use async_trait::async_trait; +use noosphere_common::ConditionalSync; use std::fmt::Debug; -#[cfg(not(target_arch = "wasm32"))] -pub trait StorageSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl StorageSendSync for T where T: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait StorageSendSync {} - -#[cfg(target_arch = "wasm32")] -impl StorageSendSync for T {} - /// [Storage] is a general trait for composite storage backends. It is often the /// case that we are able to use a single storage primitive for all forms of /// storage, but sometimes block storage and generic key/value storage come from @@ -24,7 +13,7 @@ impl StorageSendSync for T {} /// other Noosphere constructs. #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait Storage: Clone + StorageSendSync + Debug { +pub trait Storage: Clone + ConditionalSync + Debug { type BlockStore: BlockStore; type KeyValueStore: KeyValueStore; diff --git a/rust/noosphere-storage/src/store.rs b/rust/noosphere-storage/src/store.rs index 8d4893bc1..2cef23a5c 100644 --- a/rust/noosphere-storage/src/store.rs +++ b/rust/noosphere-storage/src/store.rs @@ -1,5 +1,6 @@ use std::io::Cursor; +use crate::{block::BlockStore, key_value::KeyValueStore}; use anyhow::Result; use async_trait::async_trait; use cid::Cid; @@ -9,25 +10,9 @@ use libipld_core::{ ipld::Ipld, serde::{from_ipld, to_ipld}, }; +use noosphere_common::{ConditionalSend, ConditionalSync}; use serde::{de::DeserializeOwned, Serialize}; -use crate::{ - block::BlockStore, - key_value::{KeyValueStore, KeyValueStoreSend}, -}; - -#[cfg(not(target_arch = "wasm32"))] -pub trait StoreConditionalSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl StoreConditionalSendSync for S where S: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait StoreConditionalSendSync {} - -#[cfg(target_arch = "wasm32")] -impl StoreConditionalSendSync for S {} - /// A primitive interface for storage backends. A storage backend does not /// necessarily need to implement this trait to be used in Noosphere, but if it /// does it automatically benefits from trait implementations for [BlockStore] @@ -35,7 +20,7 @@ impl StoreConditionalSendSync for S {} /// backend. #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait Store: Clone + StoreConditionalSendSync { +pub trait Store: Clone + ConditionalSync { /// Read the bytes stored against a given key async fn read(&self, key: &[u8]) -> Result>>; @@ -80,8 +65,8 @@ where { async fn set_key(&mut self, key: K, value: V) -> Result<()> where - K: AsRef<[u8]> + KeyValueStoreSend, - V: Serialize + KeyValueStoreSend, + K: AsRef<[u8]> + ConditionalSend, + V: Serialize + ConditionalSend, { let ipld = to_ipld(value)?; let codec = DagCborCodec; @@ -93,7 +78,7 @@ where async fn unset_key(&mut self, key: K) -> Result<()> where - K: AsRef<[u8]> + KeyValueStoreSend, + K: AsRef<[u8]> + ConditionalSend, { let key_bytes = K::as_ref(&key); self.remove(key_bytes).await?; @@ -102,8 +87,8 @@ where async fn get_key(&self, key: K) -> Result> where - K: AsRef<[u8]> + KeyValueStoreSend, - V: DeserializeOwned + KeyValueStoreSend, + K: AsRef<[u8]> + ConditionalSend, + V: DeserializeOwned + ConditionalSend, { let key_bytes = K::as_ref(&key); Ok(match self.read(key_bytes).await? { diff --git a/rust/noosphere/Cargo.toml b/rust/noosphere/Cargo.toml index ff5a4eee0..cc900b621 100644 --- a/rust/noosphere/Cargo.toml +++ b/rust/noosphere/Cargo.toml @@ -17,9 +17,10 @@ crate-type = ["rlib", "staticlib", "cdylib"] [features] default = [] +test-kubo = [] headers = ["safer-ffi/headers"] ipfs-storage = ["noosphere-ipfs"] -test-kubo = [] +rocksdb = ["noosphere-storage/rocksdb"] [dependencies] anyhow = { workspace = true } @@ -84,3 +85,6 @@ wasm-bindgen-test = { workspace = true } witty-phrase-generator = "~0.2" instant = { version = "0.1.12", features = ["wasm-bindgen"] } gloo-timers = { workspace = true } + +[build-dependencies] +cfg_aliases = "0.1.0" diff --git a/rust/noosphere/build.rs b/rust/noosphere/build.rs new file mode 100644 index 000000000..3747db77c --- /dev/null +++ b/rust/noosphere/build.rs @@ -0,0 +1,23 @@ +use cfg_aliases::cfg_aliases; + +fn main() { + cfg_aliases! { + // Platforms + wasm: { target_arch = "wasm32" }, + native: { not(target_arch = "wasm32") }, + apple: { + all( + target_vendor = "apple", + any(target_arch = "aarch64", target_arch = "x86_64") + ) + }, + + // Backends + rocksdb: { all(feature = "rocksdb", native) }, + sled: { all(not(any(rocksdb)), native) }, + indexeddb: { wasm }, + + // Other + ipfs_storage: { feature = "ipfs-storage" }, + } +} diff --git a/rust/noosphere/src/ffi/noosphere.rs b/rust/noosphere/src/ffi/noosphere.rs index c78baa61d..2e6b25b9d 100644 --- a/rust/noosphere/src/ffi/noosphere.rs +++ b/rust/noosphere/src/ffi/noosphere.rs @@ -94,6 +94,9 @@ impl Drop for NsNoosphere { /// You can also initialize the ns_noosphere_t with an optional third /// argument: a URL string that refers to a Noosphere Gateway API somewhere on /// the network that one or more local spheres may have access to. +/// +/// Note that storages (`global_storage_path`, `sphere_storage_path`) can only +/// be opened by a single `ns_noosphere` at a time. pub fn ns_initialize( global_storage_path: char_p::Ref<'_>, sphere_storage_path: char_p::Ref<'_>, diff --git a/rust/noosphere/src/platform.rs b/rust/noosphere/src/platform.rs index 71353adfa..f9f38e7f8 100644 --- a/rust/noosphere/src/platform.rs +++ b/rust/noosphere/src/platform.rs @@ -3,12 +3,8 @@ ///! secure key management. This module lays out the concrete strategies we will ///! use on a per-platform basis. -#[cfg(all( - any(target_arch = "aarch64", target_arch = "x86_64"), - target_vendor = "apple" -))] +#[cfg(apple)] mod inner { - use noosphere_storage::NativeStorage; use ucan_key_support::ed25519::Ed25519KeyMaterial; use crate::key::InsecureKeyStorage; @@ -18,14 +14,16 @@ mod inner { pub type PlatformKeyMaterial = Ed25519KeyMaterial; pub type PlatformKeyStorage = InsecureKeyStorage; - #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = NativeStorage; + #[cfg(sled)] + pub(crate) type PrimitiveStorage = noosphere_storage::SledStorage; + #[cfg(rocksdb)] + pub(crate) type PrimitiveStorage = noosphere_storage::RocksDbStorage; - #[cfg(feature = "ipfs-storage")] - use noosphere_ipfs::{IpfsStorage, KuboClient}; - - #[cfg(feature = "ipfs-storage")] - pub type PlatformStorage = IpfsStorage; + #[cfg(not(ipfs_storage))] + pub type PlatformStorage = PrimitiveStorage; + #[cfg(ipfs_storage)] + pub type PlatformStorage = + noosphere_ipfs::IpfsStorage; #[cfg(test)] use anyhow::Result; @@ -49,7 +47,7 @@ mod inner { } } -#[cfg(target_arch = "wasm32")] +#[cfg(wasm)] mod inner { use crate::key::WebCryptoKeyStorage; @@ -59,14 +57,16 @@ mod inner { pub type PlatformKeyMaterial = Arc; pub type PlatformKeyStorage = WebCryptoKeyStorage; - use noosphere_storage::WebStorage; + use noosphere_storage::IndexedDbStorage; - #[cfg(feature = "ipfs-storage")] + pub(crate) type PrimitiveStorage = IndexedDbStorage; + + #[cfg(ipfs_storage)] pub type PlatformStorage = - noosphere_ipfs::IpfsStorage; + noosphere_ipfs::IpfsStorage; - #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = WebStorage; + #[cfg(not(ipfs_storage))] + pub type PlatformStorage = PrimitiveStorage; #[cfg(test)] use anyhow::Result; @@ -96,30 +96,24 @@ mod inner { } } -#[cfg(all( - not(target_arch = "wasm32"), - not(all( - any(target_arch = "aarch64", target_arch = "x86_64"), - target_vendor = "apple" - )) -))] +#[cfg(all(native, not(apple)))] mod inner { - use noosphere_storage::NativeStorage; - use ucan_key_support::ed25519::Ed25519KeyMaterial; - use crate::key::InsecureKeyStorage; + use ucan_key_support::ed25519::Ed25519KeyMaterial; pub type PlatformKeyMaterial = Ed25519KeyMaterial; pub type PlatformKeyStorage = InsecureKeyStorage; - #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = NativeStorage; + #[cfg(sled)] + pub(crate) type PrimitiveStorage = noosphere_storage::SledStorage; + #[cfg(rocksdb)] + pub(crate) type PrimitiveStorage = noosphere_storage::RocksDbStorage; - #[cfg(feature = "ipfs-storage")] - use noosphere_ipfs::{IpfsStorage, KuboClient}; - - #[cfg(feature = "ipfs-storage")] - pub type PlatformStorage = IpfsStorage; + #[cfg(not(ipfs_storage))] + pub type PlatformStorage = PrimitiveStorage; + #[cfg(ipfs_storage)] + pub type PlatformStorage = + noosphere_ipfs::IpfsStorage; #[cfg(test)] use anyhow::Result; diff --git a/rust/noosphere/src/sphere/builder.rs b/rust/noosphere/src/sphere/builder.rs index ce401d0ed..7ea23d904 100644 --- a/rust/noosphere/src/sphere/builder.rs +++ b/rust/noosphere/src/sphere/builder.rs @@ -346,9 +346,9 @@ async fn generate_db( false => StorageLayout::Unscoped(storage_path), }; - #[cfg(not(all(target_arch = "wasm32", feature = "ipfs-storage")))] + #[cfg(not(all(wasm, ipfs_storage)))] let storage = storage_layout.to_storage().await?; - #[cfg(all(target_arch = "wasm32", feature = "ipfs-storage"))] + #[cfg(all(wasm, ipfs_storage))] let storage = IpfsStorage::new( storage_layout.to_storage().await?, ipfs_gateway_url.map(|url| GatewayClient::new(url)), diff --git a/rust/noosphere/src/storage.rs b/rust/noosphere/src/storage.rs index 449dbaa12..0caacef17 100644 --- a/rust/noosphere/src/storage.rs +++ b/rust/noosphere/src/storage.rs @@ -1,11 +1,11 @@ +use crate::platform::PrimitiveStorage; +use anyhow::Result; +use noosphere_core::data::Did; use std::{ fmt::Display, path::{Path, PathBuf}, }; -use anyhow::Result; -use noosphere_core::data::Did; - #[cfg(doc)] use noosphere_storage::Storage; @@ -41,23 +41,26 @@ impl From for PathBuf { } } -#[cfg(not(target_arch = "wasm32"))] -use noosphere_storage::{NativeStorage, NativeStorageInit}; - -#[cfg(not(target_arch = "wasm32"))] +#[cfg(native)] impl StorageLayout { - pub async fn to_storage(&self) -> Result { - NativeStorage::new(NativeStorageInit::Path(PathBuf::from(self))) + pub async fn to_storage(&self) -> Result { + #[cfg(sled)] + { + noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path( + PathBuf::from(self), + )) + } + #[cfg(rocksdb)] + { + noosphere_storage::RocksDbStorage::new(PathBuf::from(self)) + } } } -#[cfg(target_arch = "wasm32")] -use noosphere_storage::WebStorage; - -#[cfg(target_arch = "wasm32")] +#[cfg(wasm)] impl StorageLayout { - pub async fn to_storage(&self) -> Result { - WebStorage::new(&self.to_string()).await + pub async fn to_storage(&self) -> Result { + noosphere_storage::IndexedDbStorage::new(&self.to_string()).await } }