From fe7a8e49eddb9940470298ee4776dd41da5e986b Mon Sep 17 00:00:00 2001 From: Chris Joel <0xcda7a@gmail.com> Date: Thu, 31 Aug 2023 23:03:36 -0700 Subject: [PATCH] feat!: Replace `Bundle` with CAR streams in push --- .github/workflows/run_test_suite.yaml | 4 +- .vscode/settings.json | 2 +- Cargo.lock | 439 ++++++----- Cargo.toml | 10 + release-please-config.json | 15 +- rust/noosphere-cli/Cargo.toml | 8 +- .../src/native/commands/sphere/auth.rs | 4 +- .../src/native/commands/sphere/config.rs | 2 +- .../src/native/commands/sphere/follow.rs | 4 +- .../src/native/commands/sphere/history.rs | 2 +- .../src/native/commands/sphere/mod.rs | 2 +- .../src/native/commands/sphere/save.rs | 2 +- .../src/native/commands/sphere/status.rs | 2 +- .../src/native/commands/sphere/sync.rs | 2 +- rust/noosphere-cli/src/native/content.rs | 2 +- .../{tests => src/native}/helpers/cli.rs | 30 +- rust/noosphere-cli/src/native/helpers/mod.rs | 8 + .../native/helpers/workspace.rs} | 37 +- rust/noosphere-cli/src/native/mod.rs | 3 + .../noosphere-cli/src/native/render/buffer.rs | 2 +- rust/noosphere-cli/src/native/render/job.rs | 4 +- .../src/native/render/renderer.rs | 2 +- .../noosphere-cli/src/native/render/writer.rs | 2 +- rust/noosphere-cli/src/native/workspace.rs | 4 +- rust/noosphere-cli/src/web.rs | 1 + rust/noosphere-common/Cargo.toml | 34 + rust/noosphere-common/README.md | 5 + rust/noosphere-common/src/helpers/mod.rs | 8 + .../src}/helpers/random.rs | 27 +- rust/noosphere-common/src/helpers/wait.rs | 8 + rust/noosphere-common/src/lib.rs | 16 + rust/noosphere-common/src/sync.rs | 27 + rust/noosphere-common/src/task.rs | 128 +++ rust/noosphere-common/src/unshared.rs | 83 ++ rust/noosphere-core/Cargo.toml | 26 +- rust/noosphere-core/src/api/client.rs | 498 ++++++++++++ rust/noosphere-core/src/api/data.rs | 35 + rust/noosphere-core/src/api/mod.rs | 13 + rust/noosphere-core/src/api/route.rs | 54 ++ rust/noosphere-core/src/api/v0alpha1/data.rs | 243 ++++++ rust/noosphere-core/src/api/v0alpha1/mod.rs | 6 + rust/noosphere-core/src/api/v0alpha1/route.rs | 41 + rust/noosphere-core/src/api/v0alpha2/data.rs | 83 ++ rust/noosphere-core/src/api/v0alpha2/mod.rs | 6 + rust/noosphere-core/src/api/v0alpha2/route.rs | 25 + rust/noosphere-core/src/authority/author.rs | 4 + .../src/authority/authorization.rs | 7 +- .../src/authority/capability.rs | 8 + .../src/authority/key_material.rs | 13 +- rust/noosphere-core/src/authority/mod.rs | 6 + .../src/context/authority/mod.rs | 5 + .../src/context/authority/read.rs | 192 +++++ .../src/context/authority/write.rs | 387 +++++++++ .../src/context/content/decoder.rs | 29 + .../src/context/content/file.rs | 49 ++ .../noosphere-core/src/context/content/mod.rs | 13 + .../src/context/content/read.rs | 49 ++ .../src/context/content/write.rs | 183 +++++ rust/noosphere-core/src/context/context.rs | 445 +++++++++++ rust/noosphere-core/src/context/cursor.rs | 745 ++++++++++++++++++ rust/noosphere-core/src/context/has.rs | 223 ++++++ rust/noosphere-core/src/context/internal.rs | 105 +++ rust/noosphere-core/src/context/metadata.rs | 36 + rust/noosphere-core/src/context/mod.rs | 94 +++ .../noosphere-core/src/context/petname/mod.rs | 10 + .../src/context/petname/read.rs | 138 ++++ .../src/context/petname/write.rs | 192 +++++ .../src/context/replication/mod.rs | 3 + .../src/context/replication/read.rs | 18 + rust/noosphere-core/src/context/sync/error.rs | 29 + rust/noosphere-core/src/context/sync/mod.rs | 9 + .../src/context/sync/recovery.rs | 9 + .../src/context/sync/strategy.rs | 467 +++++++++++ rust/noosphere-core/src/context/sync/write.rs | 80 ++ rust/noosphere-core/src/context/walker.rs | 603 ++++++++++++++ rust/noosphere-core/src/data/address.rs | 5 + rust/noosphere-core/src/data/authority.rs | 18 + rust/noosphere-core/src/data/body_chunk.rs | 4 + rust/noosphere-core/src/data/bundle.rs | 3 + rust/noosphere-core/src/data/changelog.rs | 10 + .../src/data/headers/content_type.rs | 8 + .../noosphere-core/src/data/headers/header.rs | 11 + .../src/data/headers/version.rs | 3 + rust/noosphere-core/src/data/link.rs | 17 +- rust/noosphere-core/src/data/memo.rs | 6 +- rust/noosphere-core/src/data/mod.rs | 4 + rust/noosphere-core/src/data/sphere.rs | 17 +- rust/noosphere-core/src/data/versioned_map.rs | 57 +- rust/noosphere-core/src/error.rs | 8 + rust/noosphere-core/src/helpers/context.rs | 229 ++++++ .../src/{helpers.rs => helpers/link.rs} | 0 rust/noosphere-core/src/helpers/mod.rs | 8 + rust/noosphere-core/src/lib.rs | 13 + rust/noosphere-core/src/stream/block.rs | 42 + rust/noosphere-core/src/stream/car.rs | 160 ++++ rust/noosphere-core/src/stream/memo.rs | 283 +++++++ rust/noosphere-core/src/stream/mod.rs | 406 ++++++++++ rust/noosphere-core/src/stream/walk.rs | 72 ++ rust/noosphere-core/src/tracing.rs | 9 + rust/noosphere-core/src/view/authority.rs | 9 + rust/noosphere-core/src/view/content.rs | 29 + rust/noosphere-core/src/view/mod.rs | 5 + rust/noosphere-core/src/view/mutation.rs | 50 ++ rust/noosphere-core/src/view/sphere.rs | 10 + rust/noosphere-core/src/view/timeline.rs | 29 + rust/noosphere-core/src/view/versioned_map.rs | 26 + rust/noosphere-gateway/Cargo.toml | 7 +- rust/noosphere-gateway/src/authority.rs | 2 +- rust/noosphere-gateway/src/error.rs | 62 ++ rust/noosphere-gateway/src/gateway.rs | 35 +- rust/noosphere-gateway/src/handlers/mod.rs | 2 + .../src/{route => handlers/v0alpha1}/did.rs | 0 .../src/{route => handlers/v0alpha1}/fetch.rs | 14 +- .../{route => handlers/v0alpha1}/identify.rs | 6 +- .../src/{route => handlers/v0alpha1}/mod.rs | 0 .../src/{route => handlers/v0alpha1}/push.rs | 5 +- .../{route => handlers/v0alpha1}/replicate.rs | 21 +- .../src/handlers/v0alpha2/mod.rs | 3 + .../src/handlers/v0alpha2/push.rs | 376 +++++++++ rust/noosphere-gateway/src/lib.rs | 6 +- .../src/worker/name_system.rs | 25 +- .../src/worker/syndication.rs | 17 +- rust/noosphere-into/Cargo.toml | 5 +- .../examples/notes-to-html/implementation.rs | 4 +- rust/noosphere-into/src/into/html/sphere.rs | 9 +- .../noosphere-into/src/transcluder/content.rs | 2 +- rust/noosphere-into/src/transform/file.rs | 2 +- .../src/transform/sphere/html.rs | 2 +- .../src/transform/subtext/html/document.rs | 2 +- .../src/transform/subtext/html/fragment.rs | 2 +- .../src/transform/transform_implementation.rs | 2 +- rust/noosphere-ipfs/Cargo.toml | 5 +- rust/noosphere-ipfs/src/client/kubo.rs | 2 +- rust/noosphere-ipfs/src/storage.rs | 29 +- .../src/bin/orb-ns/cli/address.rs | 8 +- rust/noosphere-sphere/src/cursor.rs | 5 +- rust/noosphere-sphere/src/internal.rs | 5 +- rust/noosphere-sphere/src/lib.rs | 71 +- .../src/replication/stream.rs | 2 +- rust/noosphere-sphere/src/sync/strategy.rs | 3 +- rust/noosphere-storage/Cargo.toml | 3 +- rust/noosphere-storage/src/block.rs | 33 +- rust/noosphere-storage/src/db.rs | 61 +- rust/noosphere/Cargo.toml | 13 +- rust/noosphere/src/ffi/authority.rs | 4 +- rust/noosphere/src/ffi/context.rs | 2 +- rust/noosphere/src/ffi/petname.rs | 2 +- rust/noosphere/src/ffi/sphere.rs | 2 +- rust/noosphere/src/noosphere.rs | 2 +- rust/noosphere/src/platform.rs | 2 +- rust/noosphere/src/sphere/builder.rs | 16 +- rust/noosphere/src/sphere/channel.rs | 8 +- rust/noosphere/src/wasm/file.rs | 2 +- rust/noosphere/src/wasm/fs.rs | 2 +- rust/noosphere/src/wasm/sphere.rs | 2 +- rust/noosphere/tests/cli.rs | 155 ++++ .../tests/distributed_basic.rs} | 133 ++-- .../tests/distributed_stress.rs} | 252 +++--- .../tests/gateway.rs | 78 +- .../{integration.rs => sphere_channel.rs} | 5 +- 160 files changed, 8373 insertions(+), 806 deletions(-) rename rust/noosphere-cli/{tests => src/native}/helpers/cli.rs (76%) create mode 100644 rust/noosphere-cli/src/native/helpers/mod.rs rename rust/noosphere-cli/{tests/helpers/mod.rs => src/native/helpers/workspace.rs} (90%) create mode 100644 rust/noosphere-common/Cargo.toml create mode 100644 rust/noosphere-common/README.md create mode 100644 rust/noosphere-common/src/helpers/mod.rs rename rust/{noosphere-cli/tests => noosphere-common/src}/helpers/random.rs (51%) create mode 100644 rust/noosphere-common/src/helpers/wait.rs create mode 100644 rust/noosphere-common/src/lib.rs create mode 100644 rust/noosphere-common/src/sync.rs create mode 100644 rust/noosphere-common/src/task.rs create mode 100644 rust/noosphere-common/src/unshared.rs create mode 100644 rust/noosphere-core/src/api/client.rs create mode 100644 rust/noosphere-core/src/api/data.rs create mode 100644 rust/noosphere-core/src/api/mod.rs create mode 100644 rust/noosphere-core/src/api/route.rs create mode 100644 rust/noosphere-core/src/api/v0alpha1/data.rs create mode 100644 rust/noosphere-core/src/api/v0alpha1/mod.rs create mode 100644 rust/noosphere-core/src/api/v0alpha1/route.rs create mode 100644 rust/noosphere-core/src/api/v0alpha2/data.rs create mode 100644 rust/noosphere-core/src/api/v0alpha2/mod.rs create mode 100644 rust/noosphere-core/src/api/v0alpha2/route.rs create mode 100644 rust/noosphere-core/src/context/authority/mod.rs create mode 100644 rust/noosphere-core/src/context/authority/read.rs create mode 100644 rust/noosphere-core/src/context/authority/write.rs create mode 100644 rust/noosphere-core/src/context/content/decoder.rs create mode 100644 rust/noosphere-core/src/context/content/file.rs create mode 100644 rust/noosphere-core/src/context/content/mod.rs create mode 100644 rust/noosphere-core/src/context/content/read.rs create mode 100644 rust/noosphere-core/src/context/content/write.rs create mode 100644 rust/noosphere-core/src/context/context.rs create mode 100644 rust/noosphere-core/src/context/cursor.rs create mode 100644 rust/noosphere-core/src/context/has.rs create mode 100644 rust/noosphere-core/src/context/internal.rs create mode 100644 rust/noosphere-core/src/context/metadata.rs create mode 100644 rust/noosphere-core/src/context/mod.rs create mode 100644 rust/noosphere-core/src/context/petname/mod.rs create mode 100644 rust/noosphere-core/src/context/petname/read.rs create mode 100644 rust/noosphere-core/src/context/petname/write.rs create mode 100644 rust/noosphere-core/src/context/replication/mod.rs create mode 100644 rust/noosphere-core/src/context/replication/read.rs create mode 100644 rust/noosphere-core/src/context/sync/error.rs create mode 100644 rust/noosphere-core/src/context/sync/mod.rs create mode 100644 rust/noosphere-core/src/context/sync/recovery.rs create mode 100644 rust/noosphere-core/src/context/sync/strategy.rs create mode 100644 rust/noosphere-core/src/context/sync/write.rs create mode 100644 rust/noosphere-core/src/context/walker.rs create mode 100644 rust/noosphere-core/src/helpers/context.rs rename rust/noosphere-core/src/{helpers.rs => helpers/link.rs} (100%) create mode 100644 rust/noosphere-core/src/helpers/mod.rs create mode 100644 rust/noosphere-core/src/stream/block.rs create mode 100644 rust/noosphere-core/src/stream/car.rs create mode 100644 rust/noosphere-core/src/stream/memo.rs create mode 100644 rust/noosphere-core/src/stream/mod.rs create mode 100644 rust/noosphere-core/src/stream/walk.rs create mode 100644 rust/noosphere-core/src/view/content.rs create mode 100644 rust/noosphere-gateway/src/error.rs create mode 100644 rust/noosphere-gateway/src/handlers/mod.rs rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/did.rs (100%) rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/fetch.rs (92%) rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/identify.rs (92%) rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/mod.rs (100%) rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/push.rs (98%) rename rust/noosphere-gateway/src/{route => handlers/v0alpha1}/replicate.rs (96%) create mode 100644 rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs create mode 100644 rust/noosphere-gateway/src/handlers/v0alpha2/push.rs create mode 100644 rust/noosphere/tests/cli.rs rename rust/{noosphere-cli/tests/peer_to_peer.rs => noosphere/tests/distributed_basic.rs} (87%) rename rust/{noosphere-cli/tests/cli.rs => noosphere/tests/distributed_stress.rs} (67%) rename rust/{noosphere-cli => noosphere}/tests/gateway.rs (87%) rename rust/noosphere/tests/{integration.rs => sphere_channel.rs} (97%) diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml index 1bd9d838e..928c12b18 100644 --- a/.github/workflows/run_test_suite.yaml +++ b/.github/workflows/run_test_suite.yaml @@ -55,7 +55,7 @@ jobs: ipfs_version: v0.17.0 run_daemon: true - name: 'Run Rust native target tests' - run: cargo test --features test_kubo + run: cargo test --features test-kubo,helpers env: NOOSPHERE_LOG: deafening @@ -96,7 +96,7 @@ jobs: ipfs_version: v0.17.0 run_daemon: true - name: 'Run Rust native target tests' - run: NOOSPHERE_LOG=deafening cargo test --features test_kubo,headers + run: NOOSPHERE_LOG=deafening cargo test --features test-kubo,headers run-test-suite-linux-c: runs-on: ubuntu-latest steps: diff --git a/.vscode/settings.json b/.vscode/settings.json index 5876957c0..af68d05a2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,6 +11,6 @@ ] }, "rust-analyzer.cargo.features": [ - "test_kubo" + "test-kubo" ] } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 49de2def0..77ae6c161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,24 +176,23 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" @@ -215,9 +214,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys", @@ -349,9 +348,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ "event-listener", ] @@ -574,9 +573,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitvec" @@ -777,14 +776,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "winapi", + "windows-targets", ] [[package]] @@ -831,33 +830,31 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.19" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.19" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", "clap_lex", - "once_cell", "strsim", ] [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", @@ -867,9 +864,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "colorchoice" @@ -889,7 +886,7 @@ dependencies = [ "http", "mime", "mime_guess", - "rand 0.8.5", + "rand", "thiserror", ] @@ -1113,18 +1110,32 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16" +checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" dependencies = [ "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", "fiat-crypto", - "packed_simd_2", "platforms", + "rustc_version", "subtle", "zeroize", ] +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "darling" version = "0.14.4" @@ -1390,24 +1401,25 @@ dependencies = [ [[package]] name = "ed25519" -version = "1.5.3" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" dependencies = [ - "signature 1.6.4", + "pkcs8 0.10.2", + "signature 2.1.0", ] [[package]] name = "ed25519-dalek" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ - "curve25519-dalek 3.2.0", + "curve25519-dalek 4.0.0", "ed25519", - "rand 0.7.3", + "rand_core 0.6.4", "serde", - "sha2 0.9.9", + "sha2 0.10.7", "zeroize", ] @@ -1476,9 +1488,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -1503,9 +1515,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -1737,8 +1749,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" dependencies = [ "futures-io", - "rustls 0.20.8", - "webpki 0.22.0", + "rustls 0.20.9", + "webpki 0.22.1", ] [[package]] @@ -1880,6 +1892,27 @@ dependencies = [ "regex", ] +[[package]] +name = "gloo-net" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ac9e8288ae2c632fa9f8657ac70bfe38a1530f345282d7ba66a1f70b72b7dc4" +dependencies = [ + "futures-channel", + "futures-core", + "futures-sink", + "gloo-utils", + "http", + "js-sys", + "pin-project", + "serde", + "serde_json", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1892,6 +1925,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-utils" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "group" version = "0.12.1" @@ -1916,9 +1962,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -2096,9 +2142,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" @@ -2146,7 +2192,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.6", + "rustls 0.21.7", "tokio", "tokio-rustls", ] @@ -2280,7 +2326,7 @@ dependencies = [ "async-trait", "bytes", "log", - "rand 0.8.5", + "rand", "rtcp", "rtp", "thiserror", @@ -2332,7 +2378,7 @@ dependencies = [ "socket2 0.5.3", "widestring", "windows-sys", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -2381,17 +2427,6 @@ dependencies = [ "unsigned-varint", ] -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi", - "rustix 0.38.7", - "windows-sys", -] - [[package]] name = "itertools" version = "0.11.0" @@ -2530,12 +2565,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "libm" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" - [[package]] name = "libm" version = "0.2.7" @@ -2617,7 +2646,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project", "quick-protobuf", - "rand 0.8.5", + "rand", "rw-stream-sink", "serde", "smallvec", @@ -2662,7 +2691,7 @@ dependencies = [ "prometheus-client", "quick-protobuf", "quick-protobuf-codec", - "rand 0.8.5", + "rand", "regex", "serde", "sha2 0.10.7", @@ -2697,9 +2726,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2d584751cecb2aabaa56106be6be91338a60a0f4e420cf2af639204f596fc1" +checksum = "276bb57e7af15d8f100d3c11cbdd32c6752b7eef4ba7a18ecf464972c07abcce" dependencies = [ "bs58 0.4.0", "ed25519-dalek", @@ -2707,7 +2736,7 @@ dependencies = [ "multiaddr", "multihash 0.17.0", "quick-protobuf", - "rand 0.8.5", + "rand", "serde", "sha2 0.10.7", "thiserror", @@ -2733,7 +2762,7 @@ dependencies = [ "libp2p-swarm", "log", "quick-protobuf", - "rand 0.8.5", + "rand", "serde", "sha2 0.10.7", "smallvec", @@ -2756,7 +2785,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand 0.8.5", + "rand", "smallvec", "socket2 0.4.9", "tokio", @@ -2791,7 +2820,7 @@ dependencies = [ "log", "once_cell", "quick-protobuf", - "rand 0.8.5", + "rand", "sha2 0.10.7", "snow", "static_assertions", @@ -2816,8 +2845,8 @@ dependencies = [ "log", "parking_lot 0.12.1", "quinn-proto", - "rand 0.8.5", - "rustls 0.20.8", + "rand", + "rustls 0.20.9", "thiserror", "tokio", ] @@ -2837,7 +2866,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm-derive", "log", - "rand 0.8.5", + "rand", "smallvec", "tokio", "void", @@ -2882,9 +2911,9 @@ dependencies = [ "libp2p-identity", "rcgen 0.10.0", "ring", - "rustls 0.20.8", + "rustls 0.20.9", "thiserror", - "webpki 0.22.0", + "webpki 0.22.1", "x509-parser 0.14.0", "yasna", ] @@ -2909,7 +2938,7 @@ dependencies = [ "multihash 0.17.0", "quick-protobuf", "quick-protobuf-codec", - "rand 0.8.5", + "rand", "rcgen 0.9.3", "serde", "stun", @@ -3317,15 +3346,20 @@ dependencies = [ "lazy_static", "libipld-cbor", "libipld-core", - "noosphere-api", + "noosphere-cli", + "noosphere-common", "noosphere-core", + "noosphere-gateway", "noosphere-into", "noosphere-ipfs", - "noosphere-sphere", + "noosphere-ns", "noosphere-storage", "pkg-version", + "rand", + "reqwest", "rexie", "safer-ffi", + "serde_json", "subtext", "tempfile", "thiserror", @@ -3384,15 +3418,13 @@ dependencies = [ "libipld-core", "mime_guess", "noosphere", - "noosphere-api", "noosphere-core", "noosphere-gateway", "noosphere-ipfs", "noosphere-ns", - "noosphere-sphere", "noosphere-storage", "pathdiff", - "rand 0.8.5", + "rand", "reqwest", "serde", "serde_json", @@ -3436,6 +3468,20 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "noosphere-common" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures", + "futures-util", + "rand", + "tokio", + "tracing", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "noosphere-core" version = "0.15.2" @@ -3445,37 +3491,50 @@ dependencies = [ "async-recursion", "async-stream", "async-trait", + "axum", "base64 0.21.2", "byteorder", + "bytes", "cid", "console_error_panic_hook", "ed25519-zebra", "fastcdc", "futures", + "futures-util", "getrandom 0.2.10", + "gloo-net", + "iroh-car", + "js-sys", "libipld-cbor", "libipld-core", "noosphere-collections", + "noosphere-common", "noosphere-storage", "once_cell", - "rand 0.8.5", + "rand", + "reqwest", "sentry-tracing", "serde", "serde_bytes", "serde_json", + "serde_urlencoded", "strum 0.25.0", "strum_macros", "thiserror", "tiny-bip39", "tokio", "tokio-stream", + "tokio-util", "tracing", "tracing-subscriber", "tracing-wasm", "ucan", "ucan-key-support", "url", + "wasm-bindgen", "wasm-bindgen-test", + "wasm-streams", + "web-sys", ] [[package]] @@ -3492,11 +3551,9 @@ dependencies = [ "libipld-cbor", "libipld-core", "mime_guess", - "noosphere-api", "noosphere-core", "noosphere-ipfs", "noosphere-ns", - "noosphere-sphere", "noosphere-storage", "reqwest", "serde", @@ -3513,6 +3570,7 @@ dependencies = [ "ucan-key-support", "url", "wasm-bindgen", + "wnfs-common", "wnfs-namefilter", ] @@ -3532,7 +3590,6 @@ dependencies = [ "horrorshow", "libipld-cbor", "noosphere-core", - "noosphere-sphere", "noosphere-storage", "subtext", "tempfile", @@ -3562,9 +3619,10 @@ dependencies = [ "iroh-car", "libipld-cbor", "libipld-core", + "noosphere-common", "noosphere-core", "noosphere-storage", - "rand 0.8.5", + "rand", "reqwest", "serde", "serde_json", @@ -3593,7 +3651,7 @@ dependencies = [ "noosphere-core", "noosphere-ipfs", "noosphere-storage", - "rand 0.8.5", + "rand", "reqwest", "serde", "serde_json", @@ -3652,6 +3710,7 @@ dependencies = [ "js-sys", "libipld-cbor", "libipld-core", + "noosphere-common", "rexie", "serde", "sled", @@ -3687,9 +3746,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -3704,11 +3763,11 @@ checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" dependencies = [ "byteorder", "lazy_static", - "libm 0.2.7", + "libm", "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand", "smallvec", "zeroize", ] @@ -3741,7 +3800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", - "libm 0.2.7", + "libm", ] [[package]] @@ -3839,16 +3898,6 @@ dependencies = [ "sha2 0.10.7", ] -[[package]] -name = "packed_simd_2" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" -dependencies = [ - "cfg-if", - "libm 0.1.4", -] - [[package]] name = "parking" version = "2.1.0" @@ -4041,9 +4090,9 @@ dependencies = [ [[package]] name = "platforms" -version = "3.0.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" +checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" [[package]] name = "polling" @@ -4184,13 +4233,13 @@ dependencies = [ [[package]] name = "prometheus-client-derive-encode" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b6a5217beb0ad503ee7fa752d451c905113d70721b937126158f3106a48cc1" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.28", ] [[package]] @@ -4228,15 +4277,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f31999cfc7927c4e212e60fd50934ab40e8e8bfd2d493d6095d2d306bc0764d9" dependencies = [ "bytes", - "rand 0.8.5", + "rand", "ring", "rustc-hash", - "rustls 0.20.8", + "rustls 0.20.9", "slab", "thiserror", "tinyvec", "tracing", - "webpki 0.22.0", + "webpki 0.22.1", ] [[package]] @@ -4254,19 +4303,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -4274,20 +4310,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", + "rand_chacha", "rand_core 0.6.4", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -4316,15 +4342,6 @@ dependencies = [ "getrandom 0.2.10", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rcgen" version = "0.9.3" @@ -4425,9 +4442,9 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ "base64 0.21.2", "bytes", @@ -4446,7 +4463,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", @@ -4461,7 +4478,7 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots", - "winreg 0.10.1", + "winreg", ] [[package]] @@ -4581,7 +4598,7 @@ checksum = "a2a095411ff00eed7b12e4c6a118ba984d113e1079582570d56a5ee723f11f80" dependencies = [ "async-trait", "bytes", - "rand 0.8.5", + "rand", "serde", "thiserror", "webrtc-util", @@ -4633,11 +4650,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.7" +version = "0.38.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys 0.4.5", @@ -4659,21 +4676,21 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring", "sct 0.7.0", - "webpki 0.22.0", + "webpki 0.22.1", ] [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", @@ -4692,9 +4709,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.2" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -4800,7 +4817,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d22a5ef407871893fd72b4562ee15e4742269b173959db4b8df6f538c414e13" dependencies = [ - "rand 0.8.5", + "rand", "substring", "thiserror", "url", @@ -4847,7 +4864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7163491708804a74446642ff2c80b3acd668d4b9e9f497f85621f3d250fd012b" dependencies = [ "once_cell", - "rand 0.8.5", + "rand", "sentry-types", "serde", "serde_json", @@ -5102,14 +5119,14 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "snow" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733" +checksum = "0c9d1425eb528a21de2755c75af4c9b5d57f50a0d4c3b7f1828a4cd03f8ba155" dependencies = [ "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-rc.1", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "ring", "rustc_version", @@ -5210,7 +5227,7 @@ dependencies = [ "crc", "lazy_static", "md-5", - "rand 0.8.5", + "rand", "ring", "subtle", "thiserror", @@ -5325,14 +5342,14 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.7", + "rustix 0.38.11", "windows-sys", ] @@ -5415,7 +5432,7 @@ dependencies = [ "hmac 0.12.1", "once_cell", "pbkdf2", - "rand 0.8.5", + "rand", "rustc-hash", "sha2 0.10.7", "thiserror", @@ -5486,7 +5503,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.6", + "rustls 0.21.7", "tokio", ] @@ -5577,11 +5594,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "bytes", "futures-core", "futures-util", @@ -5715,7 +5732,7 @@ dependencies = [ "idna 0.2.3", "ipnet", "lazy_static", - "rand 0.8.5", + "rand", "smallvec", "socket2 0.4.9", "thiserror", @@ -5762,7 +5779,7 @@ dependencies = [ "futures", "log", "md-5", - "rand 0.8.5", + "rand", "ring", "stun", "thiserror", @@ -5794,7 +5811,7 @@ dependencies = [ "libipld-core", "libipld-json", "log", - "rand 0.8.5", + "rand", "serde", "serde_json", "strum 0.24.1", @@ -5839,9 +5856,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -6120,9 +6137,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -6168,9 +6185,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" dependencies = [ "ring", "untrusted", @@ -6178,12 +6195,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki 0.22.0", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "webrtc" @@ -6198,7 +6212,7 @@ dependencies = [ "interceptor", "lazy_static", "log", - "rand 0.8.5", + "rand", "rcgen 0.9.3", "regex", "ring", @@ -6243,9 +6257,9 @@ dependencies = [ [[package]] name = "webrtc-dtls" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942be5bd85f072c3128396f6e5a9bfb93ca8c1939ded735d177b7bcba9a13d05" +checksum = "c4a00f4242f2db33307347bd5be53263c52a0331c96c14292118c9a6bb48d267" dependencies = [ "aes 0.6.0", "aes-gcm 0.10.2", @@ -6260,12 +6274,11 @@ dependencies = [ "hkdf", "hmac 0.12.1", "log", - "oid-registry 0.6.1", "p256 0.11.1", "p384", - "rand 0.8.5", + "rand", "rand_core 0.6.4", - "rcgen 0.9.3", + "rcgen 0.10.0", "ring", "rustls 0.19.1", "sec1 0.3.0", @@ -6278,7 +6291,7 @@ dependencies = [ "tokio", "webpki 0.21.4", "webrtc-util", - "x25519-dalek 2.0.0-pre.1", + "x25519-dalek 2.0.0", "x509-parser 0.13.2", ] @@ -6292,7 +6305,7 @@ dependencies = [ "async-trait", "crc", "log", - "rand 0.8.5", + "rand", "serde", "serde_json", "stun", @@ -6327,7 +6340,7 @@ checksum = "f72e1650a8ae006017d1a5280efb49e2610c19ccc3c0905b03b648aee9554991" dependencies = [ "byteorder", "bytes", - "rand 0.8.5", + "rand", "rtp", "thiserror", ] @@ -6343,7 +6356,7 @@ dependencies = [ "bytes", "crc", "log", - "rand 0.8.5", + "rand", "thiserror", "tokio", "webrtc-util", @@ -6388,7 +6401,7 @@ dependencies = [ "libc", "log", "nix", - "rand 0.8.5", + "rand", "thiserror", "tokio", "winapi", @@ -6551,22 +6564,13 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.4" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - [[package]] name = "winreg" version = "0.50.0" @@ -6604,7 +6608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775ccf0bffa4cae6de1c534ffda97b032d7144d648bcc07ad56f73ac4b064b6b" dependencies = [ "getopts", - "rand 0.8.5", + "rand", ] [[package]] @@ -6671,12 +6675,13 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0-pre.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5da623d8af10a62342bcbbb230e33e58a63255a58012f8653c578e54bab48df" +checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ - "curve25519-dalek 3.2.0", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", + "serde", "zeroize", ] @@ -6733,7 +6738,7 @@ dependencies = [ "log", "nohash-hasher", "parking_lot 0.12.1", - "rand 0.8.5", + "rand", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index c7a5a0087..5af9faa43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "rust/noosphere-api", "rust/noosphere-cli", "rust/noosphere-collections", + "rust/noosphere-common", "rust/noosphere-core", "rust/noosphere-gateway", "rust/noosphere-into", @@ -20,9 +21,13 @@ resolver = "2" anyhow = { version = "1" } async-stream = { version = "0.3" } axum = { version = "^0.6.18" } +bytes = { version = "^1" } cid = { version = "0.10" } directories = { version = "5" } fastcdc = { version = "3.1" } +futures = { version = "0.3" } +futures-util = { version = "0.3" } +gloo-net = { version = "0.4" } gloo-timers = { version = "0.2", features = ["futures"] } ignore = { version = "0.4.20" } iroh-car = { version = "^0.3.0" } @@ -32,9 +37,11 @@ libipld-core = { version = "0.16" } libipld-cbor = { version = "0.16" } pathdiff = { version = "0.2.1" } rand = { version = "0.8" } +reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream"] } sentry-tracing = { version = "0.31.5" } serde = { version = "^1" } serde_json = { version = "^1" } +serde_urlencoded = { version = "~0.7" } strum = { version = "0.25" } strum_macros = { version = "0.25" } subtext = { version = "0.3.4" } @@ -43,6 +50,7 @@ tempfile = { version = "^3" } thiserror = { version = "1" } tokio = { version = "^1" } tokio-stream = { version = "~0.1" } +tokio-util = { version = "0.7" } tower = { version = "^0.4.13" } tower-http = { version = "^0.4.3" } tracing = { version = "0.1" } @@ -54,6 +62,8 @@ void = { version = "1" } wasm-bindgen = { version = "^0.2" } wasm-bindgen-test = { version = "^0.3" } wasm-bindgen-futures = { version = "^0.4" } +wasm-streams = { version = "0.3.0" } +web-sys = { version = "0.3" } wnfs-namefilter = { version = "0.1.21" } [profile.release] diff --git a/release-please-config.json b/release-please-config.json index dda8f6f76..b04d5105e 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -1,11 +1,16 @@ { - "plugins": ["cargo-workspace"], + "plugins": [ + "cargo-workspace" + ], "changelog-path": "CHANGELOG.md", "release-type": "rust", "bump-minor-pre-major": true, "bump-patch-for-minor-pre-major": true, "packages": { - "rust/noosphere-api": {}, + "rust/noosphere-api": { + "release-as": "0.13.0+deprecated" + }, + "rust/noosphere-common": {}, "rust/noosphere-cli": { "draft": true }, @@ -15,11 +20,13 @@ "rust/noosphere-ipfs": {}, "rust/noosphere-into": {}, "rust/noosphere-ns": {}, - "rust/noosphere-sphere": {}, + "rust/noosphere-sphere": { + "release-as": "0.11.0+deprecated" + }, "rust/noosphere-storage": {}, "rust/noosphere": { "draft": true } }, "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" -} +} \ No newline at end of file diff --git a/rust/noosphere-cli/Cargo.toml b/rust/noosphere-cli/Cargo.toml index afec7ad12..d32fc4a6b 100644 --- a/rust/noosphere-cli/Cargo.toml +++ b/rust/noosphere-cli/Cargo.toml @@ -17,14 +17,12 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] -test_kubo = [] +helpers = ["tracing-subscriber", "noosphere-ns"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -tracing-subscriber = { workspace = true } reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls", "stream"] } -noosphere-ns = { version = "0.10.2", path = "../noosphere-ns" } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tempfile = { workspace = true } @@ -37,6 +35,7 @@ tower = { workspace = true } tower-http = { workspace = true, features = ["cors", "trace"] } async-trait = "~0.1" tracing = { workspace = true } +tracing-subscriber = { workspace = true, optional = true } iroh-car = { workspace = true } url = { workspace = true, features = ["serde"] } @@ -47,10 +46,9 @@ globset = "~0.4" noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs" } noosphere-core = { version = "0.15.2", path = "../noosphere-core" } -noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } -noosphere-api = { version = "0.12.2", path = "../noosphere-api" } noosphere-gateway = { version = "0.8.2", path = "../noosphere-gateway" } +noosphere-ns = { version = "0.10.2", path = "../noosphere-ns", optional = true } noosphere = { version = "0.14.2", path = "../noosphere" } ucan = { workspace = true } ucan-key-support = { workspace = true } diff --git a/rust/noosphere-cli/src/native/commands/sphere/auth.rs b/rust/noosphere-cli/src/native/commands/sphere/auth.rs index a1d1f756d..977398f52 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/auth.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/auth.rs @@ -2,8 +2,10 @@ use std::{collections::BTreeMap, convert::TryFrom}; use anyhow::{anyhow, Result}; use cid::Cid; +use noosphere_core::context::{ + HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker, +}; use noosphere_core::data::{Did, Jwt, Link}; -use noosphere_sphere::{HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker}; use serde_json::{json, Value}; use ucan::{store::UcanJwtStore, Ucan}; diff --git a/rust/noosphere-cli/src/native/commands/sphere/config.rs b/rust/noosphere-cli/src/native/commands/sphere/config.rs index 18da8a625..ee43ca96a 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/config.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/config.rs @@ -1,6 +1,6 @@ use anyhow::Result; +use noosphere_core::context::metadata::{COUNTERPART, GATEWAY_URL}; use noosphere_core::data::Did; -use noosphere_sphere::metadata::{COUNTERPART, GATEWAY_URL}; use noosphere_storage::KeyValueStore; use url::Url; diff --git a/rust/noosphere-cli/src/native/commands/sphere/follow.rs b/rust/noosphere-cli/src/native/commands/sphere/follow.rs index b18a460c8..680cd2eee 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/follow.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/follow.rs @@ -2,10 +2,10 @@ use std::collections::BTreeMap; use crate::native::{commands::sphere::save, workspace::Workspace}; use anyhow::{anyhow, Result}; -use noosphere_core::data::Did; -use noosphere_sphere::{ +use noosphere_core::context::{ HasMutableSphereContext, SpherePetnameRead, SpherePetnameWrite, SphereWalker, }; +use noosphere_core::data::Did; use serde_json::{json, Value}; use tokio_stream::StreamExt; diff --git a/rust/noosphere-cli/src/native/commands/sphere/history.rs b/rust/noosphere-cli/src/native/commands/sphere/history.rs index 12e783be2..a8237f3fd 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/history.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/history.rs @@ -1,6 +1,6 @@ use anyhow::Result; +use noosphere_core::context::HasSphereContext; use noosphere_core::data::MapOperation; -use noosphere_sphere::HasSphereContext; use tokio_stream::StreamExt; use ucan::Ucan; diff --git a/rust/noosphere-cli/src/native/commands/sphere/mod.rs b/rust/noosphere-cli/src/native/commands/sphere/mod.rs index 2b39074d8..66e9de382 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/mod.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/mod.rs @@ -27,8 +27,8 @@ use noosphere::{ key::KeyStorage, sphere::{SphereContextBuilder, SphereContextBuilderArtifacts}, }; +use noosphere_core::context::{HasMutableSphereContext, SphereContext, SphereSync, SyncRecovery}; use noosphere_core::{authority::Authorization, data::Did}; -use noosphere_sphere::{HasMutableSphereContext, SphereContext, SphereSync, SyncRecovery}; use tokio::sync::Mutex; use ucan::crypto::KeyMaterial; diff --git a/rust/noosphere-cli/src/native/commands/sphere/save.rs b/rust/noosphere-cli/src/native/commands/sphere/save.rs index fc9134d10..6a860fb90 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/save.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/save.rs @@ -1,8 +1,8 @@ use anyhow::{anyhow, Result}; use cid::Cid; use libipld_cbor::DagCborCodec; +use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor}; use noosphere_core::data::Header; -use noosphere_sphere::{HasMutableSphereContext, SphereContentWrite, SphereCursor}; use noosphere_storage::BlockStore; use crate::native::{ diff --git a/rust/noosphere-cli/src/native/commands/sphere/status.rs b/rust/noosphere-cli/src/native/commands/sphere/status.rs index fd7eaf3e9..783b2c922 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/status.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/status.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use crate::native::{content::Content, Workspace}; use anyhow::Result; +use noosphere_core::context::{HasSphereContext, SphereCursor}; use noosphere_core::data::ContentType; -use noosphere_sphere::{HasSphereContext, SphereCursor}; fn status_section( name: &str, diff --git a/rust/noosphere-cli/src/native/commands/sphere/sync.rs b/rust/noosphere-cli/src/native/commands/sphere/sync.rs index 3bc8e1e13..b3d81ac51 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/sync.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/sync.rs @@ -1,6 +1,6 @@ use crate::native::{content::Content, workspace::Workspace}; use anyhow::{anyhow, Result}; -use noosphere_sphere::{SphereSync, SyncRecovery}; +use noosphere_core::context::{SphereSync, SyncRecovery}; /// Attempt to synchronize the local workspace with a configured gateway, /// optionally automatically retrying a fixed number of times in case a rebase diff --git a/rust/noosphere-cli/src/native/content.rs b/rust/noosphere-cli/src/native/content.rs index ab5377546..d11a8d6ea 100644 --- a/rust/noosphere-cli/src/native/content.rs +++ b/rust/noosphere-cli/src/native/content.rs @@ -11,7 +11,7 @@ use subtext::util::to_slug; use tokio::fs; use tokio_stream::StreamExt; -use noosphere_sphere::SphereWalker; +use noosphere_core::context::SphereWalker; use super::{extension::infer_content_type, paths::SpherePaths, workspace::Workspace}; diff --git a/rust/noosphere-cli/tests/helpers/cli.rs b/rust/noosphere-cli/src/native/helpers/cli.rs similarity index 76% rename from rust/noosphere-cli/tests/helpers/cli.rs rename to rust/noosphere-cli/src/native/helpers/cli.rs index c97a5c53f..a2f477b19 100644 --- a/rust/noosphere-cli/tests/helpers/cli.rs +++ b/rust/noosphere-cli/src/native/helpers/cli.rs @@ -1,18 +1,13 @@ -#![allow(dead_code)] - -use tracing::*; - use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::Mutex; use tempfile::TempDir; -use tracing::Level; -use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; -use tracing_subscriber::Layer; +use tracing::{field, Level, Subscriber}; +use tracing_subscriber::{prelude::*, Layer}; +use crate::{cli::Cli, invoke_cli, workspace::Workspace}; use clap::Parser; -use noosphere_cli::{cli::Cli, invoke_cli, workspace::Workspace}; #[derive(Default)] struct InfoCaptureVisitor { @@ -58,6 +53,8 @@ where } } +/// Poll a [Future] to completion, capturing the log output from work performed +/// in the span of that work. pub fn run_and_capture_info(fut: F) -> Result> where F: std::future::Future>, @@ -83,6 +80,10 @@ where Ok(lines) } +/// A helper that simulates using the `orb` CLI from a command line. When +/// initialized, it produces a temporary directory for a sphere and for global +/// Noosphere configuration, and ensures that any command that expects these +/// things is configured to use them. pub struct CliSimulator { current_working_directory: PathBuf, sphere_directory: TempDir, @@ -90,6 +91,7 @@ pub struct CliSimulator { } impl CliSimulator { + /// Initialize a new [CliSimulator] pub fn new() -> Result { let sphere_directory = TempDir::new()?; @@ -100,6 +102,9 @@ impl CliSimulator { }) } + /// Logs a command to quickly move into the simulator's temporary + /// directory and ensure its temporary credentials are available in your + /// global Noosphere key storage location pub fn print_debug_shell_command(&self) { info!( "cd {} && cp {}/keys/* $HOME/.config/noosphere/keys/", @@ -108,22 +113,30 @@ impl CliSimulator { ); } + /// The temporary root path for the [CliSimulator]'s sphere directory pub fn sphere_directory(&self) -> &Path { self.sphere_directory.path() } + /// The temporary root path for the [CliSimulator]'s global Noosphere + /// directory pub fn noosphere_directory(&self) -> &Path { self.noosphere_directory.path() } + /// Change the current working directory used by this [CliSimulator] when + /// simulating `orb` commands pub fn cd(&mut self, path: &Path) { self.current_working_directory = path.to_owned(); } + /// Change the current working directory to the temporary root sphere + /// directory in use by this [CliSimulator] pub fn cd_to_sphere_directory(&mut self) { self.current_working_directory = self.sphere_directory().to_owned(); } + /// Run an `orb` command, capturing and returning its logged output pub async fn orb_with_output(&self, command: &[&str]) -> Result> { Ok(self .run_orb_command(command, true) @@ -131,6 +144,7 @@ impl CliSimulator { .unwrap_or_default()) } + /// Run an `orb` command pub async fn orb(&self, command: &[&str]) -> Result<()> { self.run_orb_command(command, false).await?; Ok(()) diff --git a/rust/noosphere-cli/src/native/helpers/mod.rs b/rust/noosphere-cli/src/native/helpers/mod.rs new file mode 100644 index 000000000..1074556b3 --- /dev/null +++ b/rust/noosphere-cli/src/native/helpers/mod.rs @@ -0,0 +1,8 @@ +//! Generic helper utilities intended to be used exclusively in tests and hidden +//! setup for examples + +mod cli; +mod workspace; + +pub use cli::*; +pub use workspace::*; diff --git a/rust/noosphere-cli/tests/helpers/mod.rs b/rust/noosphere-cli/src/native/helpers/workspace.rs similarity index 90% rename from rust/noosphere-cli/tests/helpers/mod.rs rename to rust/noosphere-cli/src/native/helpers/workspace.rs index e8babb3a3..9f938b8db 100644 --- a/rust/noosphere-cli/tests/helpers/mod.rs +++ b/rust/noosphere-cli/src/native/helpers/workspace.rs @@ -1,26 +1,20 @@ -#![allow(dead_code)] - -mod cli; -mod random; - -pub use crate::helpers::random::*; -pub use cli::*; - use anyhow::Result; use noosphere_ipfs::{IpfsStore, KuboClient}; use noosphere_storage::{BlockStoreRetry, MemoryStore, NativeStorage, UcanStore}; -use std::{net::TcpListener, sync::Arc, time::Duration}; +use std::{net::TcpListener, sync::Arc}; use tempfile::TempDir; -use noosphere_cli::{ +use crate::{ cli::ConfigSetCommand, commands::{key::key_create, sphere::config_set, sphere::sphere_create}, workspace::Workspace, }; -use noosphere_core::data::Did; +use noosphere_core::{ + context::{HasSphereContext, SphereContext}, + data::Did, +}; use noosphere_gateway::{start_gateway, GatewayScope}; use noosphere_ns::{helpers::NameSystemNetwork, server::start_name_system_api_server}; -use noosphere_sphere::{HasSphereContext, SphereContext}; use tokio::{sync::Mutex, task::JoinHandle}; use url::Url; @@ -42,14 +36,6 @@ pub fn temporary_workspace() -> Result<(Workspace, (TempDir, TempDir))> { (root, global_root), )) } - -/// Wait for the specified number of seconds; uses [tokio::time::sleep], so this -/// will yield to the async runtime rather than block until the sleep time is -/// elapsed. -pub async fn wait(seconds: u64) { - tokio::time::sleep(Duration::from_secs(seconds)).await; -} - async fn start_gateway_for_workspace( workspace: &Workspace, client_sphere_identity: &Did, @@ -89,6 +75,7 @@ async fn start_gateway_for_workspace( Ok((gateway_url, join_handle)) } +/// Start a Noosphere Name System server for use in a test scenario pub async fn start_name_system_server(ipfs_url: &Url) -> Result<(Url, JoinHandle<()>)> { // TODO(#267) let use_validation = false; @@ -119,7 +106,9 @@ pub async fn start_name_system_server(ipfs_url: &Url) -> Result<(Url, JoinHandle /// Represents a single sphere and a corresponding workspace. pub struct SphereData { + /// The identity of the sphere pub identity: Did, + /// The temporary [Workspace] of the sphere pub workspace: Workspace, _temp_dirs: (tempfile::TempDir, tempfile::TempDir), } @@ -128,10 +117,15 @@ pub struct SphereData { /// and provides high-level utility methods for synchronizing between /// the two in order to support DSL-like integration tests. pub struct SpherePair { + /// The human-readable name of the [SpherePair] pub name: String, + /// The [SphereData] for the client half of the [SpherePair] pub client: SphereData, + /// The [SphereData] for the gateway half of the [SpherePair] pub gateway: SphereData, + /// The base [Url] for a separately-initialized Noosphere Name System REST API pub ns_url: Url, + /// The base [Url] for a separately-initialized IPFS Kubo RPC API pub ipfs_url: Url, gateway_url: Option, gateway_task: Option>, @@ -226,6 +220,9 @@ impl SpherePair { self.client.workspace.sphere_context().await } + /// Invoke a closure that returns a [Future]. The closure receives the + /// client's [SphereContext] as its argument, and polls the returned + /// [Future] until it finishes pub async fn spawn(&self, f: F) -> Result where T: Send + 'static, diff --git a/rust/noosphere-cli/src/native/mod.rs b/rust/noosphere-cli/src/native/mod.rs index 53313b13b..727304719 100644 --- a/rust/noosphere-cli/src/native/mod.rs +++ b/rust/noosphere-cli/src/native/mod.rs @@ -8,6 +8,9 @@ pub mod paths; pub mod render; pub mod workspace; +#[cfg(any(test, feature = "helpers"))] +pub mod helpers; + use anyhow::Result; use noosphere_core::tracing::initialize_tracing; diff --git a/rust/noosphere-cli/src/native/render/buffer.rs b/rust/noosphere-cli/src/native/render/buffer.rs index 65d2afee8..a13a27552 100644 --- a/rust/noosphere-cli/src/native/render/buffer.rs +++ b/rust/noosphere-cli/src/native/render/buffer.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use cid::Cid; +use noosphere_core::context::{AsyncFileBody, SphereFile}; use noosphere_core::data::Did; -use noosphere_sphere::{AsyncFileBody, SphereFile}; use std::{ collections::{BTreeMap, BTreeSet}, future::Future, diff --git a/rust/noosphere-cli/src/native/render/job.rs b/rust/noosphere-cli/src/native/render/job.rs index 3dd8804cf..b6cce6d59 100644 --- a/rust/noosphere-cli/src/native/render/job.rs +++ b/rust/noosphere-cli/src/native/render/job.rs @@ -2,11 +2,11 @@ use anyhow::{anyhow, Result}; use cid::Cid; -use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld}; -use noosphere_sphere::{ +use noosphere_core::context::{ HasSphereContext, SphereContentRead, SphereCursor, SpherePetnameRead, SphereReplicaRead, SphereWalker, }; +use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld}; use noosphere_storage::Storage; use std::{marker::PhantomData, sync::Arc}; use tokio::sync::mpsc::Sender; diff --git a/rust/noosphere-cli/src/native/render/renderer.rs b/rust/noosphere-cli/src/native/render/renderer.rs index a777a3394..244c92572 100644 --- a/rust/noosphere-cli/src/native/render/renderer.rs +++ b/rust/noosphere-cli/src/native/render/renderer.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use noosphere_sphere::HasSphereContext; +use noosphere_core::context::HasSphereContext; use noosphere_storage::Storage; use std::{collections::BTreeSet, marker::PhantomData, sync::Arc, thread::available_parallelism}; diff --git a/rust/noosphere-cli/src/native/render/writer.rs b/rust/noosphere-cli/src/native/render/writer.rs index 01179e0d7..25d4dd955 100644 --- a/rust/noosphere-cli/src/native/render/writer.rs +++ b/rust/noosphere-cli/src/native/render/writer.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use cid::Cid; +use noosphere_core::context::{AsyncFileBody, SphereFile}; use noosphere_core::data::Did; -use noosphere_sphere::{AsyncFileBody, SphereFile}; use pathdiff::diff_paths; use std::{ path::{Path, PathBuf}, diff --git a/rust/noosphere-cli/src/native/workspace.rs b/rust/noosphere-cli/src/native/workspace.rs index abd0a5ea2..0c4680b66 100644 --- a/rust/noosphere-cli/src/native/workspace.rs +++ b/rust/noosphere-cli/src/native/workspace.rs @@ -5,8 +5,10 @@ use cid::Cid; use directories::ProjectDirs; use noosphere::sphere::SphereContextBuilder; use noosphere_core::authority::Author; +use noosphere_core::context::{ + SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL, +}; use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld}; -use noosphere_sphere::{SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL}; use noosphere_storage::{KeyValueStore, NativeStorage, SphereDb}; use serde_json::Value; use std::path::{Path, PathBuf}; diff --git a/rust/noosphere-cli/src/web.rs b/rust/noosphere-cli/src/web.rs index ba5046fda..93210dfbb 100644 --- a/rust/noosphere-cli/src/web.rs +++ b/rust/noosphere-cli/src/web.rs @@ -1 +1,2 @@ +#[allow(missing_docs)] pub async fn main() {} diff --git a/rust/noosphere-common/Cargo.toml b/rust/noosphere-common/Cargo.toml new file mode 100644 index 000000000..43c752f95 --- /dev/null +++ b/rust/noosphere-common/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "noosphere-common" +version = "0.1.0" +edition = "2021" +description = "Common, generic utilities that are shared across other Noosphere packages" +keywords = ["noosphere"] +categories = [] +rust-version = "1.70.0" +license = "MIT OR Apache-2.0" +documentation = "https://docs.rs/noosphere-common" +repository = "https://github.com/subconsciousnetwork/noosphere" +homepage = "https://github.com/subconsciousnetwork/noosphere" +readme = "README.md" + +[features] +helpers = ["rand"] + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +rand = { workspace = true, optional = true } +futures-util = { workspace = true } + +[dev-dependencies] +rand = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { workspace = true, features = ["full"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +tokio = { workspace = true, features = ["sync", "macros"] } +futures = { workspace = true } +wasm-bindgen = { workspace = true } +wasm-bindgen-futures = { workspace = true } \ No newline at end of file diff --git a/rust/noosphere-common/README.md b/rust/noosphere-common/README.md new file mode 100644 index 000000000..2220602e9 --- /dev/null +++ b/rust/noosphere-common/README.md @@ -0,0 +1,5 @@ +![API Stability: Alpha](https://img.shields.io/badge/API%20Stability-Alpha-red) + +# Noosphere Common + +Common, generic utilities that are shared across other Noosphere packages. diff --git a/rust/noosphere-common/src/helpers/mod.rs b/rust/noosphere-common/src/helpers/mod.rs new file mode 100644 index 000000000..3542c5677 --- /dev/null +++ b/rust/noosphere-common/src/helpers/mod.rs @@ -0,0 +1,8 @@ +//! Generic helper utilities intended to be used exclusively in tests and hidden +//! setup for examples + +mod random; +mod wait; + +pub use random::*; +pub use wait::*; diff --git a/rust/noosphere-cli/tests/helpers/random.rs b/rust/noosphere-common/src/helpers/random.rs similarity index 51% rename from rust/noosphere-cli/tests/helpers/random.rs rename to rust/noosphere-common/src/helpers/random.rs index 54bf5cc77..23fd74339 100644 --- a/rust/noosphere-cli/tests/helpers/random.rs +++ b/rust/noosphere-common/src/helpers/random.rs @@ -6,6 +6,25 @@ use tokio::sync::Mutex; /// Its primary purpose is to retain and report the seed in use for a /// random number generator that can be shared across threads in tests. /// This is probably not suitable for use outside of tests. +/// +/// ```rust +/// # #![cfg(feature = "helpers")] +/// # use anyhow::Result; +/// # use rand::Rng; +/// # use noosphere_common::helpers::TestEntropy; +/// # +/// # #[tokio::main(flavor = "multi_thread")] +/// # async fn main() -> Result<()> { +/// # +/// let test_entropy = TestEntropy::default(); +/// let random_int = test_entropy.to_rng().lock().await.gen::(); +/// +/// let seeded_entropy = TestEntropy::from_seed(test_entropy.seed().clone()); +/// assert_eq!(random_int, seeded_entropy.to_rng().lock().await.gen::()); +/// # +/// # Ok(()) +/// # } +/// ``` pub struct TestEntropy { seed: [u8; 32], rng: Arc>, @@ -18,18 +37,22 @@ impl Default for TestEntropy { } impl TestEntropy { + /// Initialize the [TestEntropy] with an explicit seed pub fn from_seed(seed: [u8; 32]) -> Self { tracing::info!(?seed, "Initializing test entropy..."); - let rng = Arc::new(Mutex::new(SeedableRng::from_seed(seed.clone()))); + let rng = Arc::new(Mutex::new(SeedableRng::from_seed(seed))); Self { seed, rng } } + /// Get an owned instance of the internal [Rng] initialized by this + /// [TestEntropy] pub fn to_rng(&self) -> Arc> { self.rng.clone() } - pub fn seed(&self) -> &[u8] { + /// Get the seed used to initialize the internal [Rng] of this [TestEntropy] + pub fn seed(&self) -> &[u8; 32] { &self.seed } } diff --git a/rust/noosphere-common/src/helpers/wait.rs b/rust/noosphere-common/src/helpers/wait.rs new file mode 100644 index 000000000..2604fa849 --- /dev/null +++ b/rust/noosphere-common/src/helpers/wait.rs @@ -0,0 +1,8 @@ +use std::time::Duration; + +/// Wait for the specified number of seconds; uses [tokio::time::sleep], so this +/// will yield to the async runtime rather than block until the sleep time is +/// elapsed. +pub async fn wait(seconds: u64) { + tokio::time::sleep(Duration::from_secs(seconds)).await; +} diff --git a/rust/noosphere-common/src/lib.rs b/rust/noosphere-common/src/lib.rs new file mode 100644 index 000000000..51b7cebae --- /dev/null +++ b/rust/noosphere-common/src/lib.rs @@ -0,0 +1,16 @@ +//! Common, generic utilities that are shared across other Noosphere packages. +#![warn(missing_docs)] + +#[macro_use] +extern crate tracing; + +mod sync; +mod task; +mod unshared; + +pub use sync::*; +pub use task::*; +pub use unshared::*; + +#[cfg(any(test, feature = "helpers"))] +pub mod helpers; diff --git a/rust/noosphere-common/src/sync.rs b/rust/noosphere-common/src/sync.rs new file mode 100644 index 000000000..af6b65bc1 --- /dev/null +++ b/rust/noosphere-common/src/sync.rs @@ -0,0 +1,27 @@ +#[allow(missing_docs)] +#[cfg(not(target_arch = "wasm32"))] +pub trait ConditionalSend: Send {} + +#[cfg(not(target_arch = "wasm32"))] +impl ConditionalSend for S where S: Send {} + +#[allow(missing_docs)] +#[cfg(not(target_arch = "wasm32"))] +pub trait ConditionalSync: Send + Sync {} + +#[cfg(not(target_arch = "wasm32"))] +impl ConditionalSync for S where S: Send + Sync {} + +#[allow(missing_docs)] +#[cfg(target_arch = "wasm32")] +pub trait ConditionalSend {} + +#[cfg(target_arch = "wasm32")] +impl ConditionalSend for S {} + +#[allow(missing_docs)] +#[cfg(target_arch = "wasm32")] +pub trait ConditionalSync {} + +#[cfg(target_arch = "wasm32")] +impl ConditionalSync for S {} diff --git a/rust/noosphere-common/src/task.rs b/rust/noosphere-common/src/task.rs new file mode 100644 index 000000000..6fc481f4e --- /dev/null +++ b/rust/noosphere-common/src/task.rs @@ -0,0 +1,128 @@ +use anyhow::Result; +use std::future::Future; + +#[cfg(target_arch = "wasm32")] +use std::pin::Pin; + +#[cfg(target_arch = "wasm32")] +use tokio::sync::oneshot::channel; + +#[cfg(target_arch = "wasm32")] +use futures::future::join_all; + +#[cfg(not(target_arch = "wasm32"))] +use tokio::task::JoinSet; + +#[cfg(target_arch = "wasm32")] +/// Spawn a future by scheduling it with the local executor. The returned +/// future will be pending until the spawned future completes. +pub async fn spawn(future: F) -> Result +where + F: Future + 'static, + F::Output: Send + 'static, +{ + let (tx, rx) = channel(); + + wasm_bindgen_futures::spawn_local(async move { + if let Err(_) = tx.send(future.await) { + warn!("Receiver dropped before spawned task completed"); + } + }); + + Ok(rx.await?) +} + +#[cfg(not(target_arch = "wasm32"))] +/// Spawn a future by scheduling it with the local executor. The returned +/// future will be pending until the spawned future completes. +pub async fn spawn(future: F) -> Result +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + Ok(tokio::spawn(future).await?) +} + +/// An aggregator of async work that can be used to observe the moment when all +/// the aggregated work is completed. It is similar to tokio's [JoinSet], but is +/// relatively constrained and also works on `wasm32-unknown-unknown`. Unlike +/// [JoinSet], the results can not be observed individually. +/// +/// ```rust +/// # use anyhow::Result; +/// # use noosphere_common::TaskQueue; +/// # +/// # #[tokio::main(flavor = "multi_thread")] +/// # async fn main() -> Result<()> { +/// # +/// let mut task_queue = TaskQueue::default(); +/// for i in 0..10 { +/// task_queue.spawn(async move { +/// println!("{}", i); +/// Ok(()) +/// }); +/// } +/// task_queue.join().await?; +/// # +/// # Ok(()) +/// # } +/// ``` +#[derive(Default)] +pub struct TaskQueue { + #[cfg(not(target_arch = "wasm32"))] + tasks: JoinSet>, + + #[cfg(target_arch = "wasm32")] + tasks: Vec>>>, +} + +impl TaskQueue { + #[cfg(not(target_arch = "wasm32"))] + /// Queue a future to be spawned in the local executor. All queued futures will be polled + /// to completion before the [TaskQueue] can be joined. + pub fn spawn(&mut self, future: F) + where + F: Future> + Send + 'static, + { + self.tasks.spawn(future); + } + + #[cfg(not(target_arch = "wasm32"))] + /// Returns a future that finishes when all queued futures have finished. + pub async fn join(&mut self) -> Result<()> { + while let Some(result) = self.tasks.join_next().await { + trace!("Task completed, {} remaining in queue...", self.tasks.len()); + result??; + } + Ok(()) + } + + #[cfg(target_arch = "wasm32")] + /// Queue a future to be spawned in the local executor. All queued futures will be polled + /// to completion before the [TaskQueue] can be joined. + pub fn spawn(&mut self, future: F) + where + F: Future> + 'static, + { + let task_count = self.tasks.len(); + + self.tasks.push(Box::pin(async move { + if let Err(error) = spawn(future).await { + error!("Queued task failed: {:?}", error); + } + trace!("Task {} completed...", task_count + 1); + })); + } + + #[cfg(target_arch = "wasm32")] + /// Returns a future that finishes when all queued futures have finished. + pub async fn join(&mut self) -> Result<()> { + let tasks = std::mem::replace(&mut self.tasks, Vec::new()); + + debug!("Joining {} queued tasks...", tasks.len()); + + join_all(tasks).await; + + Ok(()) + } +} diff --git a/rust/noosphere-common/src/unshared.rs b/rust/noosphere-common/src/unshared.rs new file mode 100644 index 000000000..000b9a1c3 --- /dev/null +++ b/rust/noosphere-common/src/unshared.rs @@ -0,0 +1,83 @@ +use crate::ConditionalSend; +use futures_util::Stream; + +/// NOTE: This type was adapted from https://github.com/Nullus157/async-compression/blob/main/src/unshared.rs +/// Original implementation licensed MIT/Apache 2 +/// +/// Wraps a type and only allows unique borrowing, the main usecase is to wrap a `!Sync` type and +/// implement `Sync` for it as this type blocks having multiple shared references to the inner +/// value. +/// +/// # Safety +/// +/// We must be careful when accessing `inner`, there must be no way to create a shared reference to +/// it from a shared reference to an `Unshared`, as that would allow creating shared references on +/// multiple threads. +/// +/// As an example deriving or implementing `Clone` is impossible, two threads could attempt to +/// clone a shared `Unshared` reference which would result in accessing the same inner value +/// concurrently. +#[repr(transparent)] +pub struct Unshared(T); + +impl Unshared { + /// Initialize a new [Unshared], wrapping the provided inner value + pub fn new(inner: T) -> Self { + Unshared(inner) + } + + /// Get a mutable (unique) reference to the inner value + pub fn get_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +/// Safety: See comments on main docs for `Unshared` +unsafe impl Sync for Unshared {} + +impl std::fmt::Debug for Unshared { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct(core::any::type_name::()).finish() + } +} + +/// Wrapper that implements [Stream] for any [Unshared] that happens to wrap an +/// appropriately bounded [Stream]. This is useful for making a `!Sync` stream +/// into a `Sync` one in cases where we know it will not be shared by concurrent +/// actors. +/// +/// Implementation note: we do not implement [Stream] directly on [Unshared] as +/// an expression of hygiene; only mutable borrows of the inner value should be +/// possible in order to preserve the soundness of [Unshared]. +#[repr(transparent)] +pub struct UnsharedStream(Unshared) +where + T: Stream + Unpin, + T::Item: ConditionalSend + 'static; + +impl UnsharedStream +where + T: Stream + Unpin, + T::Item: ConditionalSend + 'static, +{ + /// Initialize a new [UnsharedStream] wrapping a provided (presumably `!Sync`) + /// [Stream] + pub fn new(inner: T) -> Self { + UnsharedStream(Unshared::new(inner)) + } +} + +impl Stream for UnsharedStream +where + T: Stream + Unpin, + T::Item: ConditionalSend + 'static, +{ + type Item = T::Item; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::pin!(self.get_mut().0.get_mut()).poll_next(cx) + } +} diff --git a/rust/noosphere-core/Cargo.toml b/rust/noosphere-core/Cargo.toml index e9f6c10ff..03a90473b 100644 --- a/rust/noosphere-core/Cargo.toml +++ b/rust/noosphere-core/Cargo.toml @@ -19,13 +19,14 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] +default = [] sentry = ["dep:sentry-tracing"] helpers = [] [dependencies] tracing = { workspace = true } cid = { workspace = true } -url = { workspace = true } +url = { workspace = true, features = ["serde"] } async-trait = "~0.1" async-recursion = "^1" async-stream = { workspace = true } @@ -33,23 +34,30 @@ async-stream = { workspace = true } # NOTE: async-once-cell 0.4.0 shipped unstable feature usage async-once-cell = "~0.4" anyhow = { workspace = true } +bytes = { workspace = true } +iroh-car = { workspace = true } thiserror = { workspace = true } fastcdc = { workspace = true } -futures = "~0.3" +futures = { workspace = true } +futures-util = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_urlencoded = { workspace = true } byteorder = "^1.4" base64 = "0.21" ed25519-zebra = "^3" -rand = "~0.8" +rand = { workspace = true } once_cell = "^1" tiny-bip39 = "^1" tokio-stream = { workspace = true } libipld-core = { workspace = true } libipld-cbor = { workspace = true } +reqwest = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } +tokio-util = { workspace = true, features = ["io"] } +noosphere-common = { version = "0.1.0", path = "../noosphere-common" } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } noosphere-collections = { version = "0.6.3", path = "../noosphere-collections" } @@ -60,14 +68,26 @@ sentry-tracing = { workspace = true, optional = true } [dev-dependencies] wasm-bindgen-test = { workspace = true } serde_bytes = "~0.11" +noosphere-common = { version = "0.1.0", path = "../noosphere-common", features = ["helpers"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { workspace = true, features = ["full"] } tracing-subscriber = { workspace = true } +axum = { workspace = true, features = ["headers", "macros"] } [target.'cfg(target_arch = "wasm32")'.dependencies] # NOTE: This is needed so that rand can be included in WASM builds getrandom = { version = "~0.2", features = ["js"] } +gloo-net = { workspace = true } +wasm-streams = { workspace = true } +wasm-bindgen = { workspace = true } +js-sys = { workspace = true } tokio = { workspace = true, features = ["sync", "macros"] } console_error_panic_hook = "0.1" tracing-wasm = "~0.2" + +[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys] +workspace = true +features = [ + "ReadableStream" +] \ No newline at end of file diff --git a/rust/noosphere-core/src/api/client.rs b/rust/noosphere-core/src/api/client.rs new file mode 100644 index 000000000..fe2f929af --- /dev/null +++ b/rust/noosphere-core/src/api/client.rs @@ -0,0 +1,498 @@ +use std::str::FromStr; + +use crate::{ + api::{route::RouteUrl, v0alpha1, v0alpha2}, + stream::{from_car_stream, memo_history_stream, put_block_stream, to_car_stream}, +}; + +use anyhow::{anyhow, Result}; +use async_stream::try_stream; +use bytes::Bytes; +use cid::Cid; +use iroh_car::CarReader; +use libipld_cbor::DagCborCodec; +use noosphere_common::{ConditionalSend, ConditionalSync, UnsharedStream}; + +use crate::{ + authority::{generate_capability, Author, SphereAbility, SphereReference}, + data::{Link, MemoIpld}, +}; +use noosphere_storage::{block_deserialize, block_serialize, BlockStore}; +use reqwest::{header::HeaderMap, StatusCode}; +use tokio_stream::{Stream, StreamExt}; +use tokio_util::io::StreamReader; +use ucan::{ + builder::UcanBuilder, + capability::CapabilityView, + crypto::{did::DidParser, KeyMaterial}, + store::{UcanJwtStore, UcanStore}, + ucan::Ucan, +}; +use url::Url; + +#[cfg(doc)] +use crate::data::Did; + +/// A [Client] is a simple, portable HTTP client for the Noosphere gateway REST +/// API. It embodies the intended usage of the REST API, which includes an +/// opening handshake (with associated key verification) and various +/// UCAN-authorized verbs over sphere data. +pub struct Client +where + K: KeyMaterial + Clone + 'static, + S: UcanStore + BlockStore + 'static, +{ + /// The [v0alpha1::IdentifyResponse] that was received from the gateway when + /// the [Client] was initialized + pub session: v0alpha1::IdentifyResponse, + + /// The [Did] of the sphere represented by this [Client] + pub sphere_identity: String, + + /// The [Url] for the gateway API being used by this [Client] + pub api_base: Url, + + /// The [Author] that is wielding this [Client] to interact with the gateway API + pub author: Author, + + /// The backing [BlockStore] (also used as a [UcanStore]) for this [Client] + pub store: S, + + client: reqwest::Client, +} + +impl Client +where + K: KeyMaterial + Clone + 'static, + S: UcanStore + BlockStore + 'static, +{ + /// Initialize the [Client] by perfoming an "identification" handshake with + /// a gateway whose API presumably lives at the specified URL. The request + /// is authorized (so the provided [Author] must have the appropriate + /// credentials), and the gateway responds with a + /// [v0alpha1::IdentifyResponse] to verify its own credentials for the + /// client. + pub async fn identify( + sphere_identity: &str, + api_base: &Url, + author: &Author, + did_parser: &mut DidParser, + store: S, + ) -> Result> { + debug!("Initializing Noosphere API client"); + debug!("Client represents sphere {}", sphere_identity); + debug!("Client targetting API at {}", api_base); + + let client = reqwest::Client::new(); + + let mut url = api_base.clone(); + url.set_path(&v0alpha1::Route::Did.to_string()); + + let did_response = client.get(url).send().await?; + + match did_response.status() { + StatusCode::OK => (), + _ => return Err(anyhow!("Unable to look up gateway identity")), + }; + + let gateway_identity = did_response.text().await?; + + let mut url = api_base.clone(); + url.set_path(&v0alpha1::Route::Identify.to_string()); + + let (jwt, ucan_headers) = Self::make_bearer_token( + &gateway_identity, + author, + &generate_capability(sphere_identity, SphereAbility::Fetch), + &store, + ) + .await?; + + let identify_response: v0alpha1::IdentifyResponse = client + .get(url) + .bearer_auth(jwt) + .headers(ucan_headers) + .send() + .await? + .json() + .await?; + + identify_response.verify(did_parser, &store).await?; + + debug!( + "Handshake succeeded with gateway {}", + identify_response.gateway_identity + ); + + Ok(Client { + session: identify_response, + sphere_identity: sphere_identity.into(), + api_base: api_base.clone(), + author: author.clone(), + store, + client, + }) + } + + async fn make_bearer_token( + gateway_identity: &str, + author: &Author, + capability: &CapabilityView, + store: &S, + ) -> Result<(String, HeaderMap)> { + let mut signable = UcanBuilder::default() + .issued_by(&author.key) + .for_audience(gateway_identity) + .with_lifetime(120) + .claiming_capability(capability) + .with_nonce() + .build()?; + + let mut ucan_headers = HeaderMap::new(); + + let authorization = author.require_authorization()?; + let authorization_cid = Cid::try_from(authorization)?; + + match authorization.as_ucan(store).await { + Ok(ucan) => { + if let Some(ucan_proofs) = ucan.proofs() { + // TODO(ucan-wg/rs-ucan#37): We should integrate a helper for this kind of stuff into rs-ucan + let mut proofs_to_search: Vec = ucan_proofs.clone(); + + debug!("Making bearer token... {:?}", proofs_to_search); + + while let Some(cid_string) = proofs_to_search.pop() { + let cid = Cid::from_str(cid_string.as_str())?; + let jwt = store.require_token(&cid).await?; + let ucan = Ucan::from_str(&jwt)?; + + debug!("Adding UCAN header for {}", cid); + + if let Some(ucan_proofs) = ucan.proofs() { + proofs_to_search.extend(ucan_proofs.clone().into_iter()); + } + + ucan_headers.append("ucan", format!("{cid} {jwt}").parse()?); + } + } + + ucan_headers.append( + "ucan", + format!("{} {}", authorization_cid, ucan.encode()?).parse()?, + ); + } + _ => { + debug!( + "Unable to resolve authorization to a UCAN; it will be used as a blind proof" + ) + } + }; + + // TODO(ucan-wg/rs-ucan#32): This is kind of a hack until we can add proofs by CID + signable + .proofs + .push(Cid::try_from(authorization)?.to_string()); + + let jwt = signable.sign().await?.encode()?; + + // TODO: It is inefficient to send the same UCANs with every request, + // we should probably establish a conventional flow for syncing UCANs + // this way only once when pairing a gateway. For now, this is about the + // same efficiency as what we had before when UCANs were all inlined to + // a single token. + Ok((jwt, ucan_headers)) + } + + /// Replicate content from Noosphere, streaming its blocks from the + /// configured gateway. If the gateway doesn't have the desired content, it + /// will look it up from other sources such as IPFS if they are available. + /// Note that this means this call can potentially block on upstream + /// access to an IPFS node (which, depending on the node's network + /// configuration and peering status, can be quite slow). + pub async fn replicate( + &self, + memo_version: &Cid, + params: Option<&v0alpha1::ReplicateParameters>, + ) -> Result)>>> { + let url = Url::try_from(RouteUrl( + &self.api_base, + v0alpha1::Route::Replicate(Some(*memo_version)), + params, + ))?; + + debug!("Client replicating {} from {}", memo_version, url); + + let capability = generate_capability(&self.sphere_identity, SphereAbility::Fetch); + + let (token, ucan_headers) = Self::make_bearer_token( + &self.session.gateway_identity, + &self.author, + &capability, + &self.store, + ) + .await?; + + let response = self + .client + .get(url) + .bearer_auth(token) + .headers(ucan_headers) + .send() + .await?; + + Ok( + CarReader::new(StreamReader::new(response.bytes_stream().map( + |item| match item { + Ok(item) => Ok(item), + Err(error) => { + error!("Failed to read CAR stream: {}", error); + Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) + } + }, + ))) + .await? + .stream() + .map(|block| match block { + Ok(block) => Ok(block), + Err(error) => Err(anyhow!(error)), + }), + ) + } + + /// Fetch the latest, canonical history of the client's sphere from the + /// gateway, which serves as the aggregation point for history across many + /// clients. + pub async fn fetch( + &self, + params: &v0alpha1::FetchParameters, + ) -> Result, impl Stream)>>)>> { + let url = Url::try_from(RouteUrl( + &self.api_base, + v0alpha1::Route::Fetch, + Some(params), + ))?; + + debug!("Client fetching blocks from {}", url); + + let capability = generate_capability(&self.sphere_identity, SphereAbility::Fetch); + let (token, ucan_headers) = Self::make_bearer_token( + &self.session.gateway_identity, + &self.author, + &capability, + &self.store, + ) + .await?; + + let response = self + .client + .get(url) + .bearer_auth(token) + .headers(ucan_headers) + .send() + .await?; + + let reader = CarReader::new(StreamReader::new(response.bytes_stream().map( + |item| match item { + Ok(item) => Ok(item), + Err(error) => { + error!("Failed to read CAR stream: {}", error); + Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) + } + }, + ))) + .await?; + + let tip = reader.header().roots().first().cloned(); + + if let Some(tip) = tip { + Ok(match tip.codec() { + // Identity codec = no changes + 0 => None, + _ => Some(( + tip.into(), + reader.stream().map(|block| match block { + Ok(block) => Ok(block), + Err(error) => Err(anyhow!(error)), + }), + )), + }) + } else { + Ok(None) + } + } + + fn make_push_request_stream( + store: S, + push_body: v0alpha2::PushBody, + ) -> impl Stream> + ConditionalSync + 'static { + let root = push_body.local_tip.clone().into(); + trace!("Creating push stream..."); + + let block_stream = try_stream! { + + let history_stream = memo_history_stream( + store, + &push_body.local_tip, + push_body.local_base.as_ref(), + true + ); + + yield block_serialize::(push_body)?; + + for await item in history_stream { + yield item?; + }; + }; + + // Safety: this stream is not shared by us, or by its consumer (reqwest + // on native targets, gloo-net on web) to others; the [Unshared] is required + // in order for the wrapped [Stream] to satisfy a `Sync` bound. + // See: https://github.com/seanmonstar/reqwest/issues/1969 + UnsharedStream::new(Box::pin(to_car_stream(vec![root], block_stream))) + } + + #[cfg(target_arch = "wasm32")] + async fn make_push_request( + &self, + url: Url, + ucan_headers: HeaderMap, + token: &str, + push_body: &v0alpha2::PushBody, + ) -> Result)>> + ConditionalSend, v0alpha2::PushError> + { + // Implementation note: currently reqwest does not support streaming + // request bodies under wasm32 targets even though it is technically + // feasiable via [ReadableStream]. So, we jury rig a one-off streaming + // request here using wasm-bindgen and wasm-streams: + + use gloo_net::http::Headers; + use gloo_net::http::Method; + use gloo_net::http::RequestBuilder; + use js_sys::{JsString, Uint8Array}; + use wasm_bindgen::JsValue; + use wasm_streams::ReadableStream; + + let headers = Headers::new(); + headers.append("Authorization", &format!("Bearer {}", token)); + + for (name, value) in ucan_headers { + if let (Some(name), Ok(value)) = (name, value.to_str()) { + headers.append(name.as_str(), value); + } + } + + let stream = Self::make_push_request_stream(self.store.clone(), push_body.clone()); + + let readable_stream = ReadableStream::from_stream(stream.map(|result| match result { + Ok(bytes) => Ok(JsValue::from(Uint8Array::from(bytes.as_ref()))), + Err(error) => Err(JsValue::from(JsString::from(error.to_string()))), + })); + + let request = RequestBuilder::new(url.as_str()) + .method(Method::PUT) + .headers(headers) + .body(JsValue::from(readable_stream.as_raw())) + .map_err(|error| v0alpha2::PushError::Internal(Some(error.to_string())))?; + + let response = request + .send() + .await + .map_err(|error| v0alpha2::PushError::Internal(Some(error.to_string())))?; + + let body_stream = response + .body() + .ok_or_else(|| v0alpha2::PushError::UnexpectedBody)?; + let body_stream = ReadableStream::from_raw(wasm_bindgen::JsCast::unchecked_into::< + wasm_streams::readable::sys::ReadableStream, + >(JsValue::from(body_stream))); + + let car_stream = body_stream.into_stream().map(|result| match result { + Ok(value) => match Uint8Array::try_from(value) { + Ok(array) => Ok(Bytes::from(array.to_vec())), + Err(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + v0alpha2::PushError::Internal(None), + )), + }, + Err(error) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + error.as_string().unwrap_or_default(), + )), + }); + + Ok(from_car_stream(car_stream)) + } + + #[cfg(not(target_arch = "wasm32"))] + async fn make_push_request( + &self, + url: Url, + ucan_headers: HeaderMap, + token: &str, + push_body: &v0alpha2::PushBody, + ) -> Result)>> + ConditionalSend, v0alpha2::PushError> + { + use reqwest::Body; + + let stream = Self::make_push_request_stream(self.store.clone(), push_body.clone()); + + let response = self + .client + .put(url) + .bearer_auth(token) + .headers(ucan_headers) + .header("Content-Type", "application/octet-stream") + .body(Body::wrap_stream(stream)) + .send() + .await + .map_err(|err| v0alpha2::PushError::from(anyhow!(err)))?; + + trace!("Checking response..."); + if response.status() == StatusCode::CONFLICT { + return Err(v0alpha2::PushError::Conflict); + } + + trace!("Fielding response..."); + + Ok(from_car_stream(response.bytes_stream())) + } + + /// Push the latest local history of this client to the gateway + pub async fn push( + &self, + push_body: &v0alpha2::PushBody, + ) -> Result { + let url = Url::try_from(RouteUrl::<_, ()>( + &self.api_base, + v0alpha2::Route::Push, + None, + ))?; + debug!( + "Client pushing changes for sphere {} to {}", + push_body.sphere, url + ); + let capability = generate_capability(&self.sphere_identity, SphereAbility::Push); + let (token, ucan_headers) = Self::make_bearer_token( + &self.session.gateway_identity, + &self.author, + &capability, + &self.store, + ) + .await?; + + let block_stream = self + .make_push_request(url, ucan_headers, &token, push_body) + .await?; + + tokio::pin!(block_stream); + + let push_response = match block_stream.try_next().await? { + Some((_, bytes)) => block_deserialize::(&bytes)?, + _ => return Err(v0alpha2::PushError::UnexpectedBody), + }; + + put_block_stream(self.store.clone(), block_stream).await?; + + Ok(push_response) + } +} diff --git a/rust/noosphere-core/src/api/data.rs b/rust/noosphere-core/src/api/data.rs new file mode 100644 index 000000000..0f845cbb8 --- /dev/null +++ b/rust/noosphere-core/src/api/data.rs @@ -0,0 +1,35 @@ +use std::str::FromStr; + +use anyhow::Result; + +use serde::{Deserialize, Deserializer}; + +/// A helper to express the deserialization of a query string to some +/// consistent result type +pub trait AsQuery { + /// Get the value of this trait implementor as a [Result>] + fn as_query(&self) -> Result>; +} + +impl AsQuery for () { + fn as_query(&self) -> Result> { + Ok(None) + } +} + +// NOTE: Adapted from https://github.com/tokio-rs/axum/blob/7caa4a3a47a31c211d301f3afbc518ea2c07b4de/examples/query-params-with-empty-strings/src/main.rs#L42-L54 +/// Serde deserialization decorator to map empty Strings to None, +pub(crate) fn empty_string_as_none<'de, D, T>(de: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: FromStr, + T::Err: std::fmt::Display, +{ + let opt = Option::::deserialize(de)?; + match opt.as_deref() { + None | Some("") => Ok(None), + Some(s) => FromStr::from_str(s) + .map_err(serde::de::Error::custom) + .map(Some), + } +} diff --git a/rust/noosphere-core/src/api/mod.rs b/rust/noosphere-core/src/api/mod.rs new file mode 100644 index 000000000..2973c1d19 --- /dev/null +++ b/rust/noosphere-core/src/api/mod.rs @@ -0,0 +1,13 @@ +//! This module contains data structures and client implementation for working +//! with the REST API exposed by Noosphere Gateways. + +mod client; +mod data; +mod route; + +pub use client::*; +pub use data::*; +pub use route::*; + +pub mod v0alpha1; +pub mod v0alpha2; diff --git a/rust/noosphere-core/src/api/route.rs b/rust/noosphere-core/src/api/route.rs new file mode 100644 index 000000000..9125b3113 --- /dev/null +++ b/rust/noosphere-core/src/api/route.rs @@ -0,0 +1,54 @@ +use anyhow::Result; +use std::fmt::Display; + +use url::Url; + +use crate::api::data::AsQuery; + +/// A helper macro to quickly implement a common [Display] format for +/// Noosphere Gateway REST API routes +#[macro_export] +macro_rules! route_display { + ($routes:ty) => { + impl std::fmt::Display for $routes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "/api/{}/{}", self.api_version(), self.to_fragment()) + } + } + }; +} + +/// A helper trait implemented by any route enum that enables it to be easily +/// serialized as part of a URL +pub trait RouteSignature: Display { + /// Produces the path fragment for a given route + fn to_fragment(&self) -> String; + /// Gets the API version for the given route + fn api_version(&self) -> &str; +} + +/// The [RouteUrl] is a helper to produce a [Url] from any implementor of [RouteSignature], but specifically +/// the enums found in the [crate::v0alpha1] and [crate::v0alpha2] modules. +pub struct RouteUrl<'a, 'b, Route: RouteSignature, Params: AsQuery = ()>( + pub &'a Url, + pub Route, + pub Option<&'b Params>, +); + +impl<'a, 'b, Route: RouteSignature, Params: AsQuery> TryFrom> + for Url +{ + type Error = anyhow::Error; + + fn try_from(value: RouteUrl<'a, 'b, Route, Params>) -> Result { + let RouteUrl(api_base, route, params) = value; + let mut url = api_base.clone(); + url.set_path(&route.to_string()); + if let Some(params) = params { + url.set_query(params.as_query()?.as_deref()); + } else { + url.set_query(None); + } + Ok(url) + } +} diff --git a/rust/noosphere-core/src/api/v0alpha1/data.rs b/rust/noosphere-core/src/api/v0alpha1/data.rs new file mode 100644 index 000000000..6ae446a2b --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha1/data.rs @@ -0,0 +1,243 @@ +use std::fmt::Display; + +use crate::api::data::{empty_string_as_none, AsQuery}; +use crate::{ + authority::{generate_capability, SphereAbility, SPHERE_SEMANTICS}, + data::{Bundle, Did, Jwt, Link, MemoIpld}, + error::NoosphereError, +}; +use anyhow::{anyhow, Result}; +use cid::Cid; +use noosphere_storage::{base64_decode, base64_encode}; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use ucan::{ + chain::ProofChain, + crypto::{did::DidParser, KeyMaterial}, + store::UcanStore, + Ucan, +}; + +/// The query parameters expected for the "replicate" API route +#[derive(Debug, Serialize, Deserialize)] +pub struct ReplicateParameters { + /// This is the last revision of the content that is being fetched that is + /// already fully available to the caller of the API + #[serde(default, deserialize_with = "empty_string_as_none")] + pub since: Option>, +} + +impl AsQuery for ReplicateParameters { + fn as_query(&self) -> Result> { + Ok(self.since.as_ref().map(|since| format!("since={since}"))) + } +} + +/// The query parameters expected for the "fetch" API route +#[derive(Debug, Serialize, Deserialize)] +pub struct FetchParameters { + /// This is the last revision of the "counterpart" sphere that is managed + /// by the API host that the client is fetching from + #[serde(default, deserialize_with = "empty_string_as_none")] + pub since: Option>, +} + +impl AsQuery for FetchParameters { + fn as_query(&self) -> Result> { + Ok(self.since.as_ref().map(|since| format!("since={since}"))) + } +} + +/// The possible responses from the "fetch" API route +#[derive(Debug, Serialize, Deserialize)] +pub enum FetchResponse { + /// There are new revisions to the local and "counterpart" spheres to sync + /// with local history + NewChanges { + /// The tip of the "counterpart" sphere that is managed by the API host + /// that the client is fetching from + tip: Cid, + }, + /// There are no new revisions since the revision specified in the initial + /// fetch request + UpToDate, +} + +/// The body payload expected by the "push" API route +#[derive(Debug, Serialize, Deserialize)] +pub struct PushBody { + /// The DID of the local sphere whose revisions are being pushed + pub sphere: Did, + /// The base revision represented by the payload being pushed; if the + /// entire history is being pushed, then this should be None + pub local_base: Option>, + /// The tip of the history represented by the payload being pushed + pub local_tip: Link, + /// The last received tip of the counterpart sphere + pub counterpart_tip: Option>, + /// A bundle of all the blocks needed to hydrate the revisions from the + /// base to the tip of history as represented by this payload + pub blocks: Bundle, + /// An optional name record to publish to the Noosphere Name System + pub name_record: Option, +} + +/// The possible responses from the "push" API route +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum PushResponse { + /// The new history was accepted + Accepted { + /// This is the new tip of the "counterpart" sphere after accepting + /// the latest history from the local sphere. This is guaranteed to be + /// at least one revision ahead of the latest revision being tracked + /// by the client (because it points to the newly received tip of the + /// local sphere's history) + new_tip: Link, + /// The blocks needed to hydrate the revisions of the "counterpart" + /// sphere history to the tip represented in this response + blocks: Bundle, + }, + /// The history was already known by the API host, so no changes were made + NoChange, +} + +/// Error types for typical "push" API failure conditions +#[derive(Error, Debug)] +pub enum PushError { + #[allow(missing_docs)] + #[error("Pushed history conflicts with canonical history")] + Conflict, + #[allow(missing_docs)] + #[error("Missing some implied history")] + MissingHistory, + #[allow(missing_docs)] + #[error("Replica is up to date")] + UpToDate, + #[allow(missing_docs)] + #[error("Internal error")] + Internal(anyhow::Error), +} + +impl From for PushError { + fn from(error: NoosphereError) -> Self { + error.into() + } +} + +impl From for PushError { + fn from(value: anyhow::Error) -> Self { + PushError::Internal(value) + } +} + +impl From for StatusCode { + fn from(error: PushError) -> Self { + match error { + PushError::Conflict => StatusCode::CONFLICT, + PushError::MissingHistory => StatusCode::UNPROCESSABLE_ENTITY, + PushError::UpToDate => StatusCode::BAD_REQUEST, + PushError::Internal(error) => { + error!("Internal: {:?}", error); + StatusCode::INTERNAL_SERVER_ERROR + } + } + } +} + +/// The response from the "identify" API route; this is a signed response that +/// allows the client to verify the authority of the API host +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IdentifyResponse { + /// The DID of the API host + pub gateway_identity: Did, + /// The DID of the "counterpart" sphere + pub sphere_identity: Did, + /// The signature of the API host over this payload, as base64-encoded bytes + pub signature: String, + /// The proof that the API host was authorized by the "counterpart" sphere + /// in the form of a UCAN JWT + pub proof: String, +} + +impl IdentifyResponse { + /// Create and sign an [IdentifyResponse] with the provided credential + pub async fn sign(sphere_identity: &str, key: &K, proof: &Ucan) -> Result + where + K: KeyMaterial, + { + let gateway_identity = Did(key.get_did().await?); + let signature = base64_encode( + &key.sign(&[gateway_identity.as_bytes(), sphere_identity.as_bytes()].concat()) + .await?, + )?; + Ok(IdentifyResponse { + gateway_identity, + sphere_identity: sphere_identity.into(), + signature, + proof: proof.encode()?, + }) + } + + /// Compare one [IdentifyResponse] with another to verify that they refer to + /// the same gateway + pub fn shares_identity_with(&self, other: &IdentifyResponse) -> bool { + self.gateway_identity == other.gateway_identity + && self.sphere_identity == other.sphere_identity + } + + /// Verifies that the signature scheme on the payload. The signature is made + /// by signing the bytes of the gateway's key DID plus the bytes of the + /// sphere DID that the gateway claims to manage. Remember: this sphere is + /// not the user's sphere, but rather the "counterpart" sphere created and + /// modified by the gateway. Additionally, a proof is given that the gateway + /// has been authorized to modify its own sphere. + /// + /// This verification is intended to check two things: + /// + /// 1. The gateway has control of the key that it represents with its DID + /// 2. The gateway is authorized to modify the sphere it claims to manage + pub async fn verify(&self, did_parser: &mut DidParser, store: &S) -> Result<()> { + let gateway_key = did_parser.parse(&self.gateway_identity)?; + let payload_bytes = [ + self.gateway_identity.as_bytes(), + self.sphere_identity.as_bytes(), + ] + .concat(); + let signature_bytes = base64_decode(&self.signature)?; + + // Verify that the signature is valid + gateway_key.verify(&payload_bytes, &signature_bytes).await?; + + let proof = ProofChain::try_from_token_string(&self.proof, None, did_parser, store).await?; + + if proof.ucan().audience() != self.gateway_identity.as_str() { + return Err(anyhow!("Wrong audience!")); + } + + let capability = generate_capability(&self.sphere_identity, SphereAbility::Push); + let capability_infos = proof.reduce_capabilities(&SPHERE_SEMANTICS); + + for capability_info in capability_infos { + if capability_info.capability.enables(&capability) + && capability_info + .originators + .contains(self.sphere_identity.as_str()) + { + return Ok(()); + } + } + + Err(anyhow!("Not authorized!")) + } +} + +impl Display for IdentifyResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "((Gateway {}), (Sphere {}))", + self.gateway_identity, self.sphere_identity + ) + } +} diff --git a/rust/noosphere-core/src/api/v0alpha1/mod.rs b/rust/noosphere-core/src/api/v0alpha1/mod.rs new file mode 100644 index 000000000..030aacfc0 --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha1/mod.rs @@ -0,0 +1,6 @@ +//! The original, alpha version of the Noosphere Gateway REST API +mod data; +mod route; + +pub use data::*; +pub use route::*; diff --git a/rust/noosphere-core/src/api/v0alpha1/route.rs b/rust/noosphere-core/src/api/v0alpha1/route.rs new file mode 100644 index 000000000..2513d4008 --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha1/route.rs @@ -0,0 +1,41 @@ +use crate::api::route::RouteSignature; +use crate::route_display; +use cid::Cid; + +/// The version of the API represented by this module +pub const API_VERSION: &str = "v0alpha1"; + +/// An enum whose variants represent all of the routes in this version of the API +pub enum Route { + /// Fetch the latest canonical history of a sphere from the gateway + Fetch, + /// Push the latest local history of a sphere from a client to the gateway + Push, + /// Get the DID of the gateway + Did, + /// Get a signed verification of the gateway's credentials + Identify, + /// Replicate content from the broader Noosphere network + Replicate(Option), +} + +route_display!(Route); + +impl RouteSignature for Route { + fn to_fragment(&self) -> String { + match self { + Route::Fetch => "fetch".into(), + Route::Push => "push".into(), + Route::Did => "did".into(), + Route::Identify => "identify".into(), + Route::Replicate(cid) => match cid { + Some(cid) => format!("replicate/{cid}"), + None => "replicate/:memo".into(), + }, + } + } + + fn api_version(&self) -> &str { + API_VERSION + } +} diff --git a/rust/noosphere-core/src/api/v0alpha2/data.rs b/rust/noosphere-core/src/api/v0alpha2/data.rs new file mode 100644 index 000000000..2d5aa0fd1 --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha2/data.rs @@ -0,0 +1,83 @@ +use crate::{ + data::{Did, Jwt, Link, MemoIpld}, + error::NoosphereError, +}; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// The body payload expected by the "push" API route +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PushBody { + /// The DID of the local sphere whose revisions are being pushed + pub sphere: Did, + /// The base revision represented by the payload being pushed; if the + /// entire history is being pushed, then this should be None + pub local_base: Option>, + /// The tip of the history represented by the payload being pushed + pub local_tip: Link, + /// The last received tip of the counterpart sphere + pub counterpart_tip: Option>, + /// An optional name record to publish to the Noosphere Name System + pub name_record: Option, +} + +/// The possible responses from the "push" API route +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum PushResponse { + /// The new history was accepted + Accepted { + /// This is the new tip of the "counterpart" sphere after accepting + /// the latest history from the local sphere. This is guaranteed to be + /// at least one revision ahead of the latest revision being tracked + /// by the client (because it points to the newly received tip of the + /// local sphere's history) + new_tip: Link, + }, + /// The history was already known by the API host, so no changes were made + NoChange, +} + +/// Error types for typical "push" API failure conditions +#[derive(Serialize, Deserialize, Error, Debug)] +pub enum PushError { + #[allow(missing_docs)] + #[error("First block in upstream was missing or unexpected type")] + UnexpectedBody, + #[allow(missing_docs)] + #[error("Pushed history conflicts with canonical history")] + Conflict, + #[allow(missing_docs)] + #[error("Missing some implied history")] + MissingHistory, + #[allow(missing_docs)] + #[error("Replica is up to date")] + UpToDate, + #[allow(missing_docs)] + #[error("Internal error: {0:?}")] + Internal(Option), +} + +impl From<&PushError> for StatusCode { + fn from(value: &PushError) -> Self { + match value { + PushError::UnexpectedBody => StatusCode::UNPROCESSABLE_ENTITY, + PushError::Conflict => StatusCode::CONFLICT, + PushError::MissingHistory => StatusCode::FAILED_DEPENDENCY, + PushError::UpToDate => StatusCode::NOT_MODIFIED, + PushError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl From for PushError { + fn from(error: NoosphereError) -> Self { + error.into() + } +} + +impl From for PushError { + fn from(value: anyhow::Error) -> Self { + PushError::Internal(Some(format!("{value}"))) + } +} diff --git a/rust/noosphere-core/src/api/v0alpha2/mod.rs b/rust/noosphere-core/src/api/v0alpha2/mod.rs new file mode 100644 index 000000000..dcd1ff84b --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha2/mod.rs @@ -0,0 +1,6 @@ +//! The first major revision of the alpha Noosphere Gateway REST API +mod data; +mod route; + +pub use data::*; +pub use route::*; diff --git a/rust/noosphere-core/src/api/v0alpha2/route.rs b/rust/noosphere-core/src/api/v0alpha2/route.rs new file mode 100644 index 000000000..43be7acbc --- /dev/null +++ b/rust/noosphere-core/src/api/v0alpha2/route.rs @@ -0,0 +1,25 @@ +use crate::api::route::RouteSignature; +use crate::route_display; + +/// The version of the API represented by this module +pub const API_VERSION: &str = "v0alpha2"; + +/// An enum whose variants represent all of the routes in this version of the API +pub enum Route { + /// Push the latest local history of a sphere from a client to the gateway + Push, +} + +route_display!(Route); + +impl RouteSignature for Route { + fn to_fragment(&self) -> String { + match self { + Route::Push => "push".to_owned(), + } + } + + fn api_version(&self) -> &str { + API_VERSION + } +} diff --git a/rust/noosphere-core/src/authority/author.rs b/rust/noosphere-core/src/authority/author.rs index e65ad9d8b..c307caa3c 100644 --- a/rust/noosphere-core/src/authority/author.rs +++ b/rust/noosphere-core/src/authority/author.rs @@ -20,7 +20,9 @@ use super::generate_capability; /// else read-only access (to all other spheres). #[derive(PartialEq, Eq, Debug, Clone)] pub enum Access { + /// Read/write access to a sphere ReadWrite, + /// Read-only access to a sphere ReadOnly, } @@ -33,7 +35,9 @@ pub struct Author where K: KeyMaterial + Clone + 'static, { + /// [KeyMaterial] that the [Author] has access to pub key: K, + /// Optional proof of [Authorization] for the associated key pub authorization: Option, } diff --git a/rust/noosphere-core/src/authority/authorization.rs b/rust/noosphere-core/src/authority/authorization.rs index 2cd6d4c2b..bd9d7198d 100644 --- a/rust/noosphere-core/src/authority/authorization.rs +++ b/rust/noosphere-core/src/authority/authorization.rs @@ -23,14 +23,16 @@ use super::SUPPORTED_KEYS; /// similar construct to land in rs-ucan #[derive(Clone, Debug)] pub enum Authorization { - /// A fully instantiated UCAN + /// A fully instantiated [Ucan] Ucan(Ucan), - /// A CID that refers to a UCAN that may be looked up in storage at the + /// A [Cid] that refers to a [Ucan] that may be looked up in storage at the /// point of invocation Cid(Cid), } impl Authorization { + /// Attempt to resolve the [Authorization] as a fully deserialized [Ucan] + /// (if it is not one already. pub async fn as_ucan(&self, store: &S) -> Result { match self { Authorization::Ucan(ucan) => Ok(ucan.clone()), @@ -38,6 +40,7 @@ impl Authorization { } } + /// Attempt to resolve the [Authorization] as a [ProofChain] (via its associated [Ucan]) pub async fn as_proof_chain(&self, store: &S) -> Result { let mut did_parser = DidParser::new(SUPPORTED_KEYS); Ok(match self { diff --git a/rust/noosphere-core/src/authority/capability.rs b/rust/noosphere-core/src/authority/capability.rs index 4db0516eb..71624d892 100644 --- a/rust/noosphere-core/src/authority/capability.rs +++ b/rust/noosphere-core/src/authority/capability.rs @@ -5,6 +5,7 @@ use ucan::capability::{ }; use url::Url; +/// The ordinal levels of authority allowed within Noosphere #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug)] pub enum SphereAbility { /// May read information about a sphere from a counterpart @@ -45,8 +46,13 @@ impl TryFrom for SphereAbility { } } +#[cfg(doc)] +use crate::data::Did; + +/// A reference to a sphere, by its [Did] #[derive(Clone, Debug, PartialEq, Eq)] pub struct SphereReference { + /// The [Did] of the sphere pub did: String, } @@ -78,10 +84,12 @@ impl TryFrom for SphereReference { } } +/// A struct that implements [CapabilitySemantics] for spheres pub struct SphereSemantics {} impl CapabilitySemantics for SphereSemantics {} +/// A shared instance of [SphereSemantics] pub const SPHERE_SEMANTICS: SphereSemantics = SphereSemantics {}; /// Generates a [Capability] struct representing permissions in a [LinkRecord]. diff --git a/rust/noosphere-core/src/authority/key_material.rs b/rust/noosphere-core/src/authority/key_material.rs index b25a4da6f..c28e2dc6d 100644 --- a/rust/noosphere-core/src/authority/key_material.rs +++ b/rust/noosphere-core/src/authority/key_material.rs @@ -8,18 +8,21 @@ use ucan_key_support::{ rsa::{bytes_to_rsa_key, RSA_MAGIC_BYTES}, }; +/// A common set of DID Key formats that are supported by this crate // TODO: Conditional web crypto support pub const SUPPORTED_KEYS: &KeyConstructorSlice = &[ (ED25519_MAGIC_BYTES, bytes_to_ed25519_key), (RSA_MAGIC_BYTES, bytes_to_rsa_key), ]; +/// Produce a unique [Ed25519KeyMaterial] for general purpose use cases pub fn generate_ed25519_key() -> Ed25519KeyMaterial { let private_key = Ed25519PrivateKey::new(rand::thread_rng()); let public_key = Ed25519PublicKey::from(&private_key); Ed25519KeyMaterial(public_key, Some(private_key)) } +/// Restore an [Ed25519KeyMaterial] from a [Mnemonic] pub fn restore_ed25519_key(mnemonic: &str) -> Result { let mnemonic = BipMnemonic::from_phrase(mnemonic, Language::English)?; let private_key = Ed25519PrivateKey::try_from(mnemonic.entropy())?; @@ -28,6 +31,9 @@ pub fn restore_ed25519_key(mnemonic: &str) -> Result { Ok(Ed25519KeyMaterial(public_key, Some(private_key))) } +/// Produce a [Mnemonic] for a given [Ed25519KeyMaterial]; note that the private +/// part of the key must be available in the [Ed25519KeyMaterial] in order to +/// produce the mnemonic. pub fn ed25519_key_to_mnemonic(key_material: &Ed25519KeyMaterial) -> Result { let private_key = &key_material.1.ok_or_else(|| { anyhow!( @@ -38,8 +44,11 @@ pub fn ed25519_key_to_mnemonic(key_material: &Ed25519KeyMaterial) -> Result Result<[u8; ED25519_KEYPAIR_LENGTH]> { diff --git a/rust/noosphere-core/src/authority/mod.rs b/rust/noosphere-core/src/authority/mod.rs index 7647bf004..b6ed6f474 100644 --- a/rust/noosphere-core/src/authority/mod.rs +++ b/rust/noosphere-core/src/authority/mod.rs @@ -1,3 +1,9 @@ +//! Data types and helper routines related to general Noosphere authority +//! concepts. +//! +//! This includes key material generation, expressing capabilities and passing +//! around proof of authorization within the other corners of the API. + mod author; mod authorization; mod capability; diff --git a/rust/noosphere-core/src/context/authority/mod.rs b/rust/noosphere-core/src/context/authority/mod.rs new file mode 100644 index 000000000..4c768903e --- /dev/null +++ b/rust/noosphere-core/src/context/authority/mod.rs @@ -0,0 +1,5 @@ +mod read; +pub use read::*; + +mod write; +pub use write::*; diff --git a/rust/noosphere-core/src/context/authority/read.rs b/rust/noosphere-core/src/context/authority/read.rs new file mode 100644 index 000000000..01cd92f4a --- /dev/null +++ b/rust/noosphere-core/src/context/authority/read.rs @@ -0,0 +1,192 @@ +use std::sync::Arc; + +use crate::{ + authority::{Author, Authorization}, + context::{HasSphereContext, SphereContextKey}, + data::{Did, Link, Mnemonic}, + error::NoosphereError, +}; +use anyhow::{anyhow, Result}; +use noosphere_storage::Storage; + +use tokio_stream::StreamExt; + +use async_trait::async_trait; + +/// Anything that can read the authority section from a sphere should implement +/// [SphereAuthorityRead]. A blanket implementation is provided for anything +/// that implements [HasSphereContext]. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereAuthorityRead +where + S: Storage + 'static, + Self: Sized, +{ + /// For a given [Authorization], checks that the authorization and all of its + /// ancester proofs are valid and have not been revoked + async fn verify_authorization( + &self, + authorization: &Authorization, + ) -> Result<(), NoosphereError>; + + /// Look up an [Authorization] by a [Did]. + async fn get_authorization(&self, did: &Did) -> Result>; + + /// Look up all [Authorization]s with the specified name + async fn get_authorizations_by_name(&self, name: &str) -> Result>; + + /// Derive a root sphere key from a mnemonic and return a version of this + /// [SphereAuthorityRead] whose inner [SphereContext]'s [Author] is using + /// that root sphere key. + async fn escalate_authority(&self, mnemonic: &Mnemonic) -> Result; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereAuthorityRead for C +where + C: HasSphereContext, + S: Storage + 'static, +{ + async fn verify_authorization( + &self, + authorization: &Authorization, + ) -> Result<(), NoosphereError> { + self.to_sphere() + .await? + .verify_authorization(authorization) + .await + } + + async fn get_authorization(&self, did: &Did) -> Result> { + let sphere = self.to_sphere().await?; + let authority = sphere.get_authority().await?; + let delegations = authority.get_delegations().await?; + let delegations_stream = delegations.into_stream().await?; + + tokio::pin!(delegations_stream); + + while let Some((Link { cid, .. }, delegation)) = delegations_stream.try_next().await? { + let ucan = delegation.resolve_ucan(sphere.store()).await?; + let authorized_did = ucan.audience(); + + if authorized_did == did { + return Ok(Some(Authorization::Cid(cid))); + } + } + + Ok(None) + } + + async fn get_authorizations_by_name(&self, name: &str) -> Result> { + let sphere = self.to_sphere().await?; + let authority = sphere.get_authority().await?; + let delegations = authority.get_delegations().await?; + let delegations_stream = delegations.into_stream().await?; + let mut authorizations = Vec::new(); + + tokio::pin!(delegations_stream); + + while let Some((link, delegation)) = delegations_stream.try_next().await? { + if delegation.name == name { + authorizations.push(Authorization::Cid(link.into())); + } + } + + Ok(authorizations) + } + + async fn escalate_authority(&self, mnemonic: &Mnemonic) -> Result { + let root_key: SphereContextKey = Arc::new(Box::new(mnemonic.to_credential()?)); + let root_author = Author { + key: root_key, + authorization: None, + }; + + let root_identity = root_author.did().await?; + let sphere_identity = self.identity().await?; + + if sphere_identity != root_identity { + return Err(anyhow!( + "Provided mnemonic did not produce the expected credential" + )); + } + + Ok(Self::wrap( + self.sphere_context() + .await? + .with_author(&root_author) + .await?, + ) + .await) + } +} + +#[cfg(test)] +mod tests { + use crate::data::Did; + use anyhow::Result; + + use ucan::crypto::KeyMaterial; + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + use crate::{ + context::{HasSphereContext, SphereAuthorityRead}, + helpers::{simulated_sphere_context, SimulationAccess}, + }; + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_get_an_authorization_by_did() -> Result<()> { + let (sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let author_did = Did(sphere_context + .sphere_context() + .await? + .author() + .key + .get_did() + .await?); + + let authorization = sphere_context + .get_authorization(&author_did) + .await? + .unwrap(); + + let _ucan = authorization + .as_ucan(sphere_context.sphere_context().await?.db()) + .await?; + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_verify_an_authorization_to_write_to_a_sphere() -> Result<()> { + let (sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let author_did = Did(sphere_context + .sphere_context() + .await? + .author() + .key + .get_did() + .await?); + + let authorization = sphere_context + .get_authorization(&author_did) + .await? + .unwrap(); + + sphere_context.verify_authorization(&authorization).await?; + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/context/authority/write.rs b/rust/noosphere-core/src/context/authority/write.rs new file mode 100644 index 000000000..db62725a7 --- /dev/null +++ b/rust/noosphere-core/src/context/authority/write.rs @@ -0,0 +1,387 @@ +use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, HasSphereContext}; + +use super::SphereAuthorityRead; +use crate::{ + authority::{generate_capability, Authorization, SphereAbility}, + data::{DelegationIpld, Did, Jwt, Link, RevocationIpld}, + view::SPHERE_LIFETIME, +}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use cid::Cid; +use noosphere_storage::{Storage, UcanStore}; +use tokio_stream::StreamExt; +use ucan::{builder::UcanBuilder, crypto::KeyMaterial}; + +/// Any type which implements [SphereAuthorityWrite] is able to manipulate the +/// [AuthorityIpld] section of a sphere. This includes authorizing other keys +/// and revoking prior authorizations. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereAuthorityWrite: SphereAuthorityRead +where + S: Storage + 'static, +{ + /// Authorize another key by its [Did], associating the authorization with a + /// provided display name + async fn authorize(&mut self, name: &str, identity: &Did) -> Result; + + /// Revoke a previously granted authorization. + /// + /// Note that correctly revoking an authorization requires signing with a credential + /// that is in the chain of authority that ultimately granted the authorization being + /// revoked. Attempting to revoke a credential with any credential that isn't in that + /// chain of authority will fail. + async fn revoke_authorization(&mut self, authorization: &Authorization) -> Result<()>; + + /// Recover authority by revoking all previously delegated authorizations + /// and creating a new one that delegates authority to the specified key + /// (given by its [Did]). + /// + /// Note that correctly recovering authority requires signing with the root + /// sphere credential, so generally can only be performed on a type that + /// implements [SphereAuthorityEscalate] + async fn recover_authority(&mut self, new_owner: &Did) -> Result; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereAuthorityWrite for C +where + C: HasSphereContext + HasMutableSphereContext, + S: Storage + 'static, +{ + // TODO(#423): We allow optional human-readable names for authorizations, + // but this will bear the consequence of leaking personal information about + // the user (e.g., a list of their authorized devices). We should encrypt + // these names so that they are only readable by the user themselves. + // TODO(#560): We should probably enforce that each [Did] only gets one + // authorization, from a hygeine perspective; elsewhere we need to assume + // multiple authorizations for the same [Did] are possible. + async fn authorize(&mut self, name: &str, identity: &Did) -> Result { + self.assert_write_access().await?; + + let author = self.sphere_context().await?.author().clone(); + let mut sphere = self.to_sphere().await?; + let authorization = author.require_authorization()?; + + self.verify_authorization(authorization).await?; + + let authorization_expiry: Option = { + let ucan = authorization + .as_ucan(&UcanStore(sphere.store().clone())) + .await?; + *ucan.expires_at() + }; + + let mut builder = UcanBuilder::default() + .issued_by(&author.key) + .for_audience(identity) + .claiming_capability(&generate_capability( + &sphere.get_identity().await?, + SphereAbility::Authorize, + )) + .with_nonce(); + + // TODO(ucan-wg/rs-ucan#114): Clean this up when + // `UcanBuilder::with_expiration` accepts `Option` + if let Some(expiry) = authorization_expiry { + builder = builder.with_expiration(expiry); + } + + // TODO(ucan-wg/rs-ucan#32): Clean this up when we can use a CID as an authorization + let mut signable = builder.build()?; + + signable + .proofs + .push(Cid::try_from(authorization)?.to_string()); + + let jwt = signable.sign().await?.encode()?; + + let delegation = DelegationIpld::register(name, &jwt, sphere.store_mut()).await?; + + self.sphere_context_mut() + .await? + .mutation_mut() + .delegations_mut() + .set(&Link::new(delegation.jwt), &delegation); + + Ok(Authorization::Cid(delegation.jwt)) + } + + async fn revoke_authorization(&mut self, authorization: &Authorization) -> Result<()> { + self.assert_write_access().await?; + + let mut sphere_context = self.sphere_context_mut().await?; + + if !sphere_context + .author() + .is_authorizer_of(authorization, sphere_context.db()) + .await? + { + let author_did = sphere_context.author().did().await?; + + return Err(anyhow!( + "{} cannot revoke authorization {} (not a delegating authority)", + author_did, + authorization + )); + } + + let authorization_cid = Link::::from(Cid::try_from(authorization)?); + let delegations = sphere_context + .sphere() + .await? + .get_authority() + .await? + .get_delegations() + .await?; + + if delegations.get(&authorization_cid).await?.is_none() { + return Err(anyhow!( + "No authority has been delegated to the authorization being revoked" + )); + } + + let revocation = + RevocationIpld::revoke(&authorization_cid, &sphere_context.author().key).await?; + + sphere_context + .mutation_mut() + .delegations_mut() + .remove(&authorization_cid); + + sphere_context + .mutation_mut() + .revocations_mut() + .set(&authorization_cid, &revocation); + + // TODO(#424): Recursively remove any sub-delegations here (and revoke them?) + + Ok(()) + } + + async fn recover_authority(&mut self, new_owner: &Did) -> Result { + self.assert_write_access().await?; + + let mut sphere_context = self.sphere_context_mut().await?; + let author_did = Did(sphere_context.author().key.get_did().await?); + let sphere_identity = sphere_context.identity().clone(); + + if author_did != sphere_identity { + return Err(anyhow!( + "Only the root sphere credential can be used to recover authority" + )); + } + + let sphere = sphere_context.sphere().await?; + let authority = sphere.get_authority().await?; + let delegations = authority.get_delegations().await?; + let delegation_stream = delegations.into_stream().await?; + + tokio::pin!(delegation_stream); + + // First: revoke all current authority + while let Some((link, _)) = delegation_stream.try_next().await? { + let revocation = RevocationIpld::revoke(&link, &sphere_context.author().key).await?; + + sphere_context + .mutation_mut() + .delegations_mut() + .remove(&link); + sphere_context + .mutation_mut() + .revocations_mut() + .set(&link, &revocation); + } + + // Then: bless a new owner + let ucan = UcanBuilder::default() + .issued_by(&sphere_context.author().key) + .for_audience(new_owner) + .with_lifetime(SPHERE_LIFETIME) + .with_nonce() + .claiming_capability(&generate_capability( + &sphere_identity, + SphereAbility::Authorize, + )) + .build()? + .sign() + .await?; + + let jwt = ucan.encode()?; + let delegation = DelegationIpld::register("(OWNER)", &jwt, sphere_context.db()).await?; + let link = Link::new(delegation.jwt); + + sphere_context + .mutation_mut() + .delegations_mut() + .set(&link, &delegation); + + Ok(Authorization::Cid(link.into())) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::authority::{generate_ed25519_key, Author}; + use crate::data::Did; + use anyhow::Result; + + use tokio::sync::Mutex; + use ucan::crypto::KeyMaterial; + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + use crate::{ + context::{ + HasMutableSphereContext, HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, + SphereContextKey, + }, + helpers::{simulated_sphere_context, SimulationAccess}, + }; + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_allows_an_authorized_key_to_authorize_other_keys() -> Result<()> { + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let other_key = generate_ed25519_key(); + let other_did = Did(other_key.get_did().await?); + + let other_authorization = sphere_context.authorize("other", &other_did).await?; + sphere_context.save(None).await?; + + assert!(sphere_context + .verify_authorization(&other_authorization) + .await + .is_ok()); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_implicitly_revokes_transitive_authorizations() -> Result<()> { + let (mut sphere_context, mnemonic) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let other_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key())); + let other_did = Did(other_key.get_did().await?); + + let other_authorization = sphere_context.authorize("other", &other_did).await?; + sphere_context.save(None).await?; + + let mut sphere_context_with_other_credential = Arc::new(Mutex::new( + sphere_context + .sphere_context() + .await? + .with_author(&Author { + key: other_key.clone(), + authorization: Some(other_authorization.clone()), + }) + .await?, + )); + + let third_key = generate_ed25519_key(); + let third_did = Did(third_key.get_did().await?); + + let third_authorization = sphere_context_with_other_credential + .authorize("third", &third_did) + .await?; + sphere_context_with_other_credential.save(None).await?; + + let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?; + + root_sphere_context + .revoke_authorization(&other_authorization) + .await?; + root_sphere_context.save(None).await?; + + assert!(sphere_context + .verify_authorization(&third_authorization) + .await + .is_err()); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_catches_revoked_authorizations_when_verifying() -> Result<()> { + let (mut sphere_context, mnemonic) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let other_key = generate_ed25519_key(); + let other_did = Did(other_key.get_did().await?); + + let other_authorization = sphere_context.authorize("other", &other_did).await?; + sphere_context.save(None).await?; + + let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?; + + root_sphere_context + .revoke_authorization(&other_authorization) + .await?; + root_sphere_context.save(None).await?; + + assert!(sphere_context + .verify_authorization(&other_authorization) + .await + .is_err()); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_perform_access_recovery_given_a_mnemonic() -> Result<()> { + let (mut sphere_context, mnemonic) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let owner = sphere_context.sphere_context().await?.author().clone(); + + let other_key = generate_ed25519_key(); + let other_did = Did(other_key.get_did().await?); + + let other_authorization = sphere_context.authorize("other", &other_did).await?; + sphere_context.save(None).await?; + + let next_owner_key = generate_ed25519_key(); + let next_owner_did = Did(next_owner_key.get_did().await?); + + let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?; + + root_sphere_context + .recover_authority(&next_owner_did) + .await?; + root_sphere_context.save(None).await?; + + assert!(sphere_context + .verify_authorization(&other_authorization) + .await + .is_err()); + + assert!(sphere_context + .verify_authorization(&owner.authorization.unwrap()) + .await + .is_err()); + + sphere_context + .verify_authorization( + &sphere_context + .get_authorization(&next_owner_did) + .await? + .unwrap(), + ) + .await?; + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/context/content/decoder.rs b/rust/noosphere-core/src/context/content/decoder.rs new file mode 100644 index 000000000..cfc13029c --- /dev/null +++ b/rust/noosphere-core/src/context/content/decoder.rs @@ -0,0 +1,29 @@ +use crate::data::BodyChunkIpld; +use async_stream::try_stream; +use bytes::Bytes; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use noosphere_storage::BlockStore; +use tokio_stream::Stream; + +/// Helper to easily decode a linked list of `BodyChunkIpld` as a byte stream +pub struct BodyChunkDecoder<'a, 'b, S: BlockStore>(pub &'a Cid, pub &'b S); + +impl<'a, 'b, S: BlockStore> BodyChunkDecoder<'a, 'b, S> { + /// Consume the [BodyChunkDecoder] and return an async [Stream] of bytes + /// representing the raw body contents + pub fn stream(self) -> impl Stream> + Unpin { + let mut next = Some(*self.0); + let store = self.1.clone(); + Box::pin(try_stream! { + while let Some(cid) = next { + debug!("Unpacking block {}...", cid); + let chunk = store.load::(&cid).await.map_err(|error| { + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, error.to_string()) + })?; + yield Bytes::from(chunk.bytes); + next = chunk.next; + } + }) + } +} diff --git a/rust/noosphere-core/src/context/content/file.rs b/rust/noosphere-core/src/context/content/file.rs new file mode 100644 index 000000000..780796e52 --- /dev/null +++ b/rust/noosphere-core/src/context/content/file.rs @@ -0,0 +1,49 @@ +use std::pin::Pin; + +use crate::data::{Did, Link, MemoIpld}; +use tokio::io::AsyncRead; + +/// A type that may be used as the contents field in a [SphereFile] +#[cfg(not(target_arch = "wasm32"))] +pub trait AsyncFileBody: AsyncRead + Unpin + Send {} + +#[cfg(not(target_arch = "wasm32"))] +impl AsyncFileBody for S where S: AsyncRead + Unpin + Send {} + +#[cfg(target_arch = "wasm32")] +/// A type that may be used as the contents field in a [SphereFile] +pub trait AsyncFileBody: AsyncRead + Unpin {} + +#[cfg(target_arch = "wasm32")] +impl AsyncFileBody for S where S: AsyncRead + Unpin {} + +/// A descriptor for contents that is stored in a sphere. +pub struct SphereFile { + /// The identity of the associated sphere from which the file was read + pub sphere_identity: Did, + /// The version of the associated sphere from which the file was read + pub sphere_version: Link, + /// The version of the memo that wraps the file's body contents + pub memo_version: Link, + /// The memo that wraps the file's body contents + pub memo: MemoIpld, + /// The body contents of the file + pub contents: C, +} + +impl SphereFile +where + C: AsyncFileBody + 'static, +{ + /// Consume the file and return a version of it where its body contents have + /// been boxed and pinned + pub fn boxed(self) -> SphereFile>> { + SphereFile { + sphere_identity: self.sphere_identity, + sphere_version: self.sphere_version, + memo_version: self.memo_version, + memo: self.memo, + contents: Box::pin(self.contents), + } + } +} diff --git a/rust/noosphere-core/src/context/content/mod.rs b/rust/noosphere-core/src/context/content/mod.rs new file mode 100644 index 000000000..ec6444013 --- /dev/null +++ b/rust/noosphere-core/src/context/content/mod.rs @@ -0,0 +1,13 @@ +//! Sphere content is a storage space for any files that the sphere owner wishes to associate +//! with a public "slug", that is addressable by them or others who have replicated the sphere +//! data. + +mod decoder; +mod file; +mod read; +mod write; + +pub use decoder::*; +pub use file::*; +pub use read::*; +pub use write::*; diff --git a/rust/noosphere-core/src/context/content/read.rs b/rust/noosphere-core/src/context/content/read.rs new file mode 100644 index 000000000..3a4b72026 --- /dev/null +++ b/rust/noosphere-core/src/context/content/read.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use noosphere_storage::Storage; + +use crate::context::HasSphereContext; +use async_trait::async_trait; + +use crate::context::{internal::SphereContextInternal, AsyncFileBody, SphereFile}; + +/// Anything that can read content from a sphere should implement [SphereContentRead]. +/// A blanket implementation is provided for anything that implements [HasSphereContext]. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereContentRead +where + S: Storage + 'static, +{ + /// Read a file that is associated with a given slug at the revision of the + /// sphere that this view is pointing to. + /// Note that "contents" are `AsyncRead`, and content bytes won't be read + /// until contents is polled. + async fn read(&self, slug: &str) -> Result>>>; + + /// Returns true if the content identitifed by slug exists in the sphere at + /// the current revision. + async fn exists(&self, slug: &str) -> Result { + Ok(self.read(slug).await?.is_some()) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereContentRead for C +where + C: HasSphereContext, + S: Storage + 'static, +{ + async fn read(&self, slug: &str) -> Result>>> { + let revision = self.version().await?; + let sphere = self.to_sphere().await?; + + let links = sphere.get_content().await?; + let hamt = links.get_hamt().await?; + + Ok(match hamt.get(&slug.to_string()).await? { + Some(memo) => Some(self.get_file(&revision, memo.clone()).await?), + None => None, + }) + } +} diff --git a/rust/noosphere-core/src/context/content/write.rs b/rust/noosphere-core/src/context/content/write.rs new file mode 100644 index 000000000..e13bf387b --- /dev/null +++ b/rust/noosphere-core/src/context/content/write.rs @@ -0,0 +1,183 @@ +use crate::data::{BodyChunkIpld, Header, Link, MemoIpld}; +use anyhow::{anyhow, Result}; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use noosphere_storage::{BlockStore, Storage}; + +use tokio::io::AsyncReadExt; + +use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, HasSphereContext}; +use async_trait::async_trait; + +use crate::context::{AsyncFileBody, SphereContentRead}; + +fn validate_slug(slug: &str) -> Result<()> { + if slug.is_empty() { + Err(anyhow!("Slug must not be empty.")) + } else { + Ok(()) + } +} + +/// Anything that can write content to a sphere should implement +/// [SphereContentWrite]. A blanket implementation is provided for anything that +/// implements [HasMutableSphereContext]. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereContentWrite: SphereContentRead +where + S: Storage + 'static, +{ + /// Like link, this takes a [Link] that should be associated + /// directly with a slug, but in this case the [Link] is assumed + /// to refer to a memo, so no wrapping memo is created. + async fn link_raw(&mut self, slug: &str, cid: &Link) -> Result<()>; + + /// Similar to write, but instead of generating blocks from some provided + /// bytes, the caller provides a CID of an existing DAG in storage. That + /// CID is used as the body of a Memo that is written to the specified + /// slug, and the CID of the memo is returned. + async fn link( + &mut self, + slug: &str, + content_type: &str, + body_cid: &Cid, + additional_headers: Option>, + ) -> Result>; + + /// Write to a slug in the sphere. In order to commit the change to the + /// sphere, you must call save. You can buffer multiple writes before + /// saving. + /// + /// The returned CID is a link to the memo for the newly added content. + async fn write( + &mut self, + slug: &str, + content_type: &str, + mut value: F, + additional_headers: Option>, + ) -> Result>; + + /// Unlinks a slug from the content space. Note that this does not remove + /// the blocks that were previously associated with the content found at the + /// given slug, because they will still be available at an earlier revision + /// of the sphere. In order to commit the change, you must save. Note that + /// this call is a no-op if there is no matching slug linked in the sphere. + /// + /// The returned value is the CID previously associated with the slug, if + /// any. + async fn remove(&mut self, slug: &str) -> Result>>; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereContentWrite for C +where + C: HasSphereContext + HasMutableSphereContext, + S: Storage + 'static, +{ + async fn link_raw(&mut self, slug: &str, cid: &Link) -> Result<()> { + self.assert_write_access().await?; + validate_slug(slug)?; + + self.sphere_context_mut() + .await? + .mutation_mut() + .content_mut() + .set(&slug.into(), cid); + + Ok(()) + } + + async fn link( + &mut self, + slug: &str, + content_type: &str, + body_cid: &Cid, + additional_headers: Option>, + ) -> Result> { + self.assert_write_access().await?; + validate_slug(slug)?; + + let memo_cid = { + let current_file = self.read(slug).await?; + let previous_memo_cid = current_file.map(|file| file.memo_version); + + let mut sphere_context = self.sphere_context_mut().await?; + + let mut new_memo = match previous_memo_cid { + Some(cid) => { + let mut memo = MemoIpld::branch_from(&cid, sphere_context.db()).await?; + memo.body = *body_cid; + memo + } + None => MemoIpld { + parent: None, + headers: Vec::new(), + body: *body_cid, + }, + }; + + if let Some(headers) = additional_headers { + new_memo.replace_headers(headers) + } + + new_memo.replace_first_header(&Header::ContentType, content_type); + + // TODO(#43): Configure default/implicit headers here + sphere_context + .db_mut() + .save::(new_memo) + .await? + .into() + }; + + self.link_raw(slug, &memo_cid).await?; + + Ok(memo_cid) + } + + async fn write( + &mut self, + slug: &str, + content_type: &str, + mut value: F, + additional_headers: Option>, + ) -> Result> { + debug!("Writing {}...", slug); + + self.assert_write_access().await?; + validate_slug(slug)?; + + let mut bytes = Vec::new(); + value.read_to_end(&mut bytes).await?; + + // TODO(#38): We imply here that the only content types we care about + // amount to byte streams, but in point of fact we can support anything + // that may be referenced by CID including arbitrary IPLD structures + let body_cid = + BodyChunkIpld::store_bytes(&bytes, self.sphere_context_mut().await?.db_mut()).await?; + + self.link(slug, content_type, &body_cid, additional_headers) + .await + } + + async fn remove(&mut self, slug: &str) -> Result>> { + self.assert_write_access().await?; + + let current_file = self.read(slug).await?; + + Ok(match current_file { + Some(file) => { + self.sphere_context_mut() + .await? + .mutation_mut() + .content_mut() + .remove(&String::from(slug)); + + Some(file.memo_version) + } + None => None, + }) + } +} diff --git a/rust/noosphere-core/src/context/context.rs b/rust/noosphere-core/src/context/context.rs new file mode 100644 index 000000000..53ade732a --- /dev/null +++ b/rust/noosphere-core/src/context/context.rs @@ -0,0 +1,445 @@ +use std::sync::Arc; + +use anyhow::Result; + +use crate::{ + api::Client, + authority::{Access, Author, SUPPORTED_KEYS}, + context::metadata::GATEWAY_URL, + data::{Did, Link, MemoIpld}, + view::{Sphere, SphereMutation}, +}; +use noosphere_storage::{KeyValueStore, SphereDb, Storage}; +use tokio::sync::OnceCell; +use ucan::crypto::{did::DidParser, KeyMaterial}; +use url::Url; + +#[cfg(doc)] +use crate::context::has::HasSphereContext; + +/// The type of any [KeyMaterial] that is used within a [SphereContext] +pub type SphereContextKey = Arc>; + +/// A [SphereContext] is an accessor construct over locally replicated sphere +/// data. It embodies both the storage layer that contains the sphere's data +/// as the information needed to verify a user's intended level of access to +/// it (e.g., local key material and [ucan::Ucan]-based authorization). +/// Additionally, the [SphereContext] maintains a reference to an API [Client] +/// that may be initialized as the network becomes available. +/// +/// All interactions that pertain to a sphere, including reading or writing +/// its contents and syncing with a gateway, flow through the [SphereContext]. +pub struct SphereContext +where + S: Storage + 'static, +{ + sphere_identity: Did, + origin_sphere_identity: Did, + author: Author, + access: OnceCell, + db: SphereDb, + did_parser: DidParser, + client: OnceCell>>>, + mutation: SphereMutation, +} + +impl Clone for SphereContext +where + S: Storage + 'static, +{ + fn clone(&self) -> Self { + Self { + sphere_identity: self.sphere_identity.clone(), + origin_sphere_identity: self.origin_sphere_identity.clone(), + author: self.author.clone(), + access: OnceCell::new(), + db: self.db.clone(), + did_parser: DidParser::new(SUPPORTED_KEYS), + client: self.client.clone(), + mutation: SphereMutation::new(self.mutation.author()), + } + } +} + +impl SphereContext +where + S: Storage, +{ + /// Instantiate a new [SphereContext] given a sphere [Did], an [Author], a + /// [SphereDb] and an optional origin sphere [Did]. The origin sphere [Did] + /// is intended to signify whether the [SphereContext] is a local sphere, or + /// a global sphere that is being visited by a local author. In most cases, + /// a [SphereContext] with _some_ value set as the origin sphere [Did] will + /// be read-only. + pub async fn new( + sphere_identity: Did, + author: Author, + db: SphereDb, + origin_sphere_identity: Option, + ) -> Result { + let author_did = author.identity().await?; + let origin_sphere_identity = + origin_sphere_identity.unwrap_or_else(|| sphere_identity.clone()); + + Ok(SphereContext { + sphere_identity, + origin_sphere_identity, + access: OnceCell::new(), + author, + db, + did_parser: DidParser::new(SUPPORTED_KEYS), + client: OnceCell::new(), + mutation: SphereMutation::new(&author_did), + }) + } + + /// Clone this [SphereContext], setting the sphere identity to a peer's [Did] + pub async fn to_visitor(&self, peer_identity: &Did) -> Result { + self.db().require_version(peer_identity).await?; + + SphereContext::new( + peer_identity.clone(), + self.author.clone(), + self.db.clone(), + Some(self.origin_sphere_identity.clone()), + ) + .await + } + + /// Clone this [SphereContext], replacing the [Author] with the provided one + pub async fn with_author(&self, author: &Author) -> Result> { + SphereContext::new( + self.sphere_identity.clone(), + author.clone(), + self.db.clone(), + Some(self.origin_sphere_identity.clone()), + ) + .await + } + + /// Given a [Did] of a sphere, produce a [SphereContext] backed by the same credentials and + /// storage primitives as this one, but that accesses the sphere referred to by the provided + /// [Did]. + pub async fn traverse_by_identity(&self, _sphere_identity: &Did) -> Result> { + unimplemented!() + } + + /// The identity of the sphere + pub fn identity(&self) -> &Did { + &self.sphere_identity + } + + /// The identity of the gateway sphere in use during this session, if + /// any; note that this will cause a request to be made to a gateway if no + /// handshake has yet occurred. + pub async fn gateway_identity(&self) -> Result { + Ok(self.client().await?.session.gateway_identity.clone()) + } + + /// The CID of the most recent local version of this sphere + pub async fn version(&self) -> Result> { + Ok(self.db().require_version(self.identity()).await?.into()) + } + + /// The [Author] who is currently accessing the sphere + pub fn author(&self) -> &Author { + &self.author + } + + /// The [Access] level that the configured [Author] has relative to the + /// sphere that this [SphereContext] refers to. + pub async fn access(&self) -> Result { + let access = self + .access + .get_or_try_init(|| async { + self.author.access_to(&self.sphere_identity, &self.db).await + }) + .await?; + Ok(access.clone()) + } + + /// Get a mutable reference to the [DidParser] used in this [SphereContext] + pub fn did_parser_mut(&mut self) -> &mut DidParser { + &mut self.did_parser + } + + /// Sets or unsets the gateway URL that points to the gateway API that the + /// sphere will use when it is syncing. + pub async fn configure_gateway_url(&mut self, url: Option<&Url>) -> Result<()> { + self.client = OnceCell::new(); + + match url { + Some(url) => { + self.db.set_key(GATEWAY_URL, url.to_string()).await?; + } + None => { + self.db.unset_key(GATEWAY_URL).await?; + } + } + + Ok(()) + } + + /// Get the [SphereDb] instance that manages the current sphere's block + /// space and persisted configuration. + pub fn db(&self) -> &SphereDb { + &self.db + } + + /// Get a mutable reference to the [SphereDb] instance that manages the + /// current sphere's block space and persisted configuration. + pub fn db_mut(&mut self) -> &mut SphereDb { + &mut self.db + } + + /// Get a read-only reference to the underlying [SphereMutation] that this + /// [SphereContext] is tracking + pub fn mutation(&self) -> &SphereMutation { + &self.mutation + } + + /// Get a mutable reference to the underlying [SphereMutation] that this + /// [SphereContext] is tracking + pub fn mutation_mut(&mut self) -> &mut SphereMutation { + &mut self.mutation + } + + /// Get a [Sphere] view over the current sphere's latest revision. This view + /// offers lower-level access than [HasSphereContext], but includes affordances to + /// help tranversing and manipulating IPLD structures that are more + /// convenient than working directly with raw data. + pub async fn sphere(&self) -> Result>> { + Ok(Sphere::at( + &self.db.require_version(self.identity()).await?.into(), + self.db(), + )) + } + + /// Get a [Client] that will interact with a configured gateway (if a URL + /// for one has been configured). This will initialize a [Client] if one is + /// not already intialized, and will fail if the [Client] is unable to + /// verify the identity of the gateway or otherwise connect to it. + pub async fn client(&self) -> Result>>> { + let client = self + .client + .get_or_try_init::(|| async { + let gateway_url: Url = self.db.require_key(GATEWAY_URL).await?; + + Ok(Arc::new( + Client::identify( + &self.origin_sphere_identity, + &gateway_url, + &self.author, + // TODO: Kill `DidParser` with fire + &mut DidParser::new(SUPPORTED_KEYS), + self.db.clone(), + ) + .await?, + )) + }) + .await?; + + Ok(client.clone()) + } + + // Reset access so that it is re-evaluated the next time it is measured + // self.access.take(); + pub(crate) fn reset_access(&mut self) { + self.access.take(); + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + + use crate::{ + authority::{generate_capability, generate_ed25519_key, SphereAbility}, + context::{ + HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite, + }, + data::{ContentType, LinkRecord, LINK_RECORD_FACT_NAME}, + helpers::{make_valid_link_record, simulated_sphere_context, SimulationAccess}, + tracing::initialize_tracing, + view::Sphere, + }; + + use noosphere_storage::{MemoryStorage, SphereDb}; + use ucan::{builder::UcanBuilder, crypto::KeyMaterial, store::UcanJwtStore}; + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_validates_slug_names_when_writing() -> Result<()> { + initialize_tracing(None); + let valid_names: &[&str] = &["j@__/_大", "/"]; + let invalid_names: &[&str] = &[""]; + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + for invalid_name in invalid_names { + assert!(sphere_context + .write(invalid_name, &ContentType::Text, "hello".as_ref(), None,) + .await + .is_err()); + } + + for valid_name in valid_names { + assert!(sphere_context + .write(valid_name, &ContentType::Text, "hello".as_ref(), None,) + .await + .is_ok()); + } + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_validates_petnames_when_setting() -> Result<()> { + initialize_tracing(None); + let valid_names: &[&str] = &["j@__/_大"]; + let invalid_names: &[&str] = &["", "did:key:foo"]; + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let mut db = sphere_context.sphere_context().await?.db().clone(); + let (other_identity, link_record, _) = make_valid_link_record(&mut db).await?; + + for invalid_name in invalid_names { + assert!(sphere_context + .set_petname_record(invalid_name, &link_record) + .await + .is_err()); + assert!(sphere_context + .set_petname(invalid_name, Some(other_identity.clone())) + .await + .is_err()); + } + + for valid_name in valid_names { + assert!(sphere_context + .set_petname(valid_name, Some(other_identity.clone())) + .await + .is_ok()); + sphere_context.save(None).await?; + assert!(sphere_context + .set_petname_record(valid_name, &link_record) + .await + .is_ok()); + } + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_disallows_adding_self_as_petname() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let db = sphere_context.sphere_context().await?.db().clone(); + let sphere_identity = sphere_context.identity().await?; + + let link_record = { + let version = sphere_context.version().await?; + let author = sphere_context.sphere_context().await?.author().clone(); + LinkRecord::from( + UcanBuilder::default() + .issued_by(&author.key) + .for_audience(&sphere_identity) + .witnessed_by( + &author.authorization.as_ref().unwrap().as_ucan(&db).await?, + None, + ) + .claiming_capability(&generate_capability( + &sphere_identity, + SphereAbility::Publish, + )) + .with_lifetime(120) + .with_fact(LINK_RECORD_FACT_NAME, version.to_string()) + .build()? + .sign() + .await?, + ) + }; + + assert!(sphere_context + .set_petname_record("myself", &link_record) + .await + .is_err()); + assert!(sphere_context + .set_petname("myself", Some(sphere_identity.clone())) + .await + .is_err()); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_disallows_adding_outdated_records() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let mut store = sphere_context.sphere_context().await?.db().clone(); + + // Generate two LinkRecords, the first one having a later expiry + // than the second. + let (records, foo_identity) = { + let mut records: Vec = vec![]; + let owner_key = generate_ed25519_key(); + let owner_did = owner_key.get_did().await?; + let mut db = SphereDb::new(&MemoryStorage::default()).await?; + let (sphere, proof, _) = Sphere::generate(&owner_did, &mut db).await?; + let ucan_proof = proof.as_ucan(&db).await?; + let sphere_identity = sphere.get_identity().await?; + store.write_token(&ucan_proof.encode()?).await?; + + for lifetime in [500, 100] { + let link_record = LinkRecord::from( + UcanBuilder::default() + .issued_by(&owner_key) + .for_audience(&sphere_identity) + .witnessed_by(&ucan_proof, None) + .claiming_capability(&generate_capability( + &sphere_identity, + SphereAbility::Publish, + )) + .with_lifetime(lifetime) + .with_fact(LINK_RECORD_FACT_NAME, sphere.cid().to_string()) + .build()? + .sign() + .await?, + ); + + store.write_token(&link_record.encode()?).await?; + records.push(link_record); + } + (records, sphere_identity) + }; + + sphere_context + .set_petname("foo", Some(foo_identity)) + .await?; + sphere_context.save(None).await?; + + assert!(sphere_context + .set_petname_record("foo", records.get(0).unwrap()) + .await + .is_ok()); + sphere_context.save(None).await?; + assert!(sphere_context + .set_petname_record("foo", records.get(1).unwrap()) + .await + .is_err()); + Ok(()) + } +} diff --git a/rust/noosphere-core/src/context/cursor.rs b/rust/noosphere-core/src/context/cursor.rs new file mode 100644 index 000000000..4e1aa6528 --- /dev/null +++ b/rust/noosphere-core/src/context/cursor.rs @@ -0,0 +1,745 @@ +use crate::api::v0alpha1::ReplicateParameters; +use crate::stream::put_block_stream; +use crate::{ + data::{Link, MemoIpld}, + view::{Sphere, Timeline}, +}; +use anyhow::Result; +use async_trait::async_trait; +use noosphere_storage::Storage; + +use crate::context::{HasMutableSphereContext, HasSphereContext, SphereContext, SphereReplicaRead}; +use std::marker::PhantomData; + +/// A [SphereCursor] is a structure that enables reading from and writing to a +/// [SphereContext] at specific versions of the associated sphere's history. +/// There are times when you may wish to be able to use the convenience +/// implementation of traits built on [HasSphereContext], but to always be sure +/// of what version you are using them on (such as when traversing sphere +/// history). That is when you would use a [SphereCursor], which can wrap any +/// implementor of [HasSphereContext] and mount it to a specific version of the +/// sphere. +#[derive(Clone)] +pub struct SphereCursor +where + C: HasSphereContext, + S: Storage + 'static, +{ + has_sphere_context: C, + storage: PhantomData, + sphere_version: Option>, +} + +impl SphereCursor +where + C: HasSphereContext, + S: Storage + 'static, +{ + /// Consume the [SphereCursor] and return its wrapped [HasSphereContext] + pub fn to_inner(self) -> C { + self.has_sphere_context + } + + /// Same as [SphereCursor::mount], but mounts the [SphereCursor] to a known + /// version of the history of the sphere. + pub fn mounted_at(has_sphere_context: C, sphere_version: &Link) -> Self { + SphereCursor { + has_sphere_context, + storage: PhantomData, + sphere_version: Some(sphere_version.clone()), + } + } + + /// Create the [SphereCursor] at the latest local version of the associated + /// sphere, mounted to that version. If the latest version changes due to + /// effects in the distance, the cursor will still point to the same version + /// it referred to when it was created. + pub async fn mounted(has_sphere_context: C) -> Result { + let mut cursor = Self::latest(has_sphere_context); + cursor.mount().await?; + Ok(cursor) + } + + /// "Mount" the [SphereCursor] to the given version of the sphere it refers + /// to. If the [SphereCursor] is already mounted, the version it is mounted + /// to will be overwritten. A mounted [SphereCursor] will remain at the + /// version it is mounted to even when the latest version of the sphere + /// changes. + pub async fn mount_at(&mut self, sphere_version: &Link) -> Result<&Self> { + self.sphere_version = Some(sphere_version.clone()); + + Ok(self) + } + + /// Same as [SphereCursor::mount_at] except that it mounts to the latest + /// local version of the sphere. + pub async fn mount(&mut self) -> Result<&Self> { + let sphere_version = self + .has_sphere_context + .sphere_context() + .await? + .version() + .await?; + + self.mount_at(&sphere_version).await + } + + /// "Unmount" the [SphereCursor] so that it always uses the latest local + /// version of the sphere that it refers to. + pub fn unmount(mut self) -> Result { + self.sphere_version = None; + Ok(self) + } + + /// Create this [SphereCursor] at the latest local version of the associated + /// sphere. The [SphereCursor] will always point to the latest local + /// version, unless subsequently mounted. + pub fn latest(has_sphere_context: C) -> Self { + SphereCursor { + has_sphere_context, + storage: PhantomData, + sphere_version: None, + } + } + + /// Rewind the [SphereCursor] to point to the version of the sphere just + /// prior to this one in the edit chronology. If there was a previous + /// version to rewind to then the returned `Option` has the [Cid] of the + /// revision, otherwise if the current version is the oldest one it is + /// `None`. + pub async fn rewind(&mut self) -> Result>> { + let sphere = self.to_sphere().await?; + + match sphere.get_parent().await? { + Some(parent) => { + self.sphere_version = Some(parent.cid().clone()); + Ok(self.sphere_version.as_ref()) + } + None => Ok(None), + } + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasMutableSphereContext for SphereCursor +where + C: HasMutableSphereContext, + S: Storage, +{ + type MutableSphereContext = C::MutableSphereContext; + + async fn sphere_context_mut(&mut self) -> Result { + self.has_sphere_context.sphere_context_mut().await + } + + async fn save( + &mut self, + additional_headers: Option>, + ) -> Result> { + let new_version = self.has_sphere_context.save(additional_headers).await?; + + if self.sphere_version.is_some() { + self.sphere_version = Some(new_version.clone()); + } + + Ok(new_version) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasSphereContext for SphereCursor +where + C: HasSphereContext, + S: Storage + 'static, +{ + type SphereContext = C::SphereContext; + + async fn sphere_context(&self) -> Result { + self.has_sphere_context.sphere_context().await + } + + async fn version(&self) -> Result> { + match &self.sphere_version { + Some(sphere_version) => Ok(sphere_version.clone()), + None => self.has_sphere_context.version().await, + } + } + + async fn wrap(sphere_context: SphereContext) -> Self { + SphereCursor::latest(C::wrap(sphere_context).await) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereReplicaRead for SphereCursor +where + C: HasSphereContext, + S: Storage + 'static, +{ + #[instrument(level = "debug", skip(self))] + async fn traverse_by_petnames(&self, petname_path: &[String]) -> Result> { + debug!("Traversing by petname..."); + + let replicate = { + let cursor = self.clone(); + + move |version: Link, since: Option>| { + let cursor = cursor.clone(); + + async move { + let replicate_parameters = since.as_ref().map(|since| ReplicateParameters { + since: Some(since.clone()), + }); + let (db, client) = { + let sphere_context = cursor.sphere_context().await?; + (sphere_context.db().clone(), sphere_context.client().await?) + }; + let stream = client + .replicate(&version, replicate_parameters.as_ref()) + .await?; + put_block_stream(db.clone(), stream).await?; + + // If this was incremental replication, we have to hydrate... + if let Some(since) = since { + let since_memo = since.load_from(&db).await?; + let latest_memo = version.load_from(&db).await?; + + // Only hydrate if since is a causal antecedent + if since_memo.lamport_order() < latest_memo.lamport_order() { + let timeline = Timeline::new(&db); + + Sphere::hydrate_timeslice( + &timeline.slice(&version, Some(&since)).exclude_past(), + ) + .await?; + } + } + + Ok(()) as Result<(), anyhow::Error> + } + } + }; + + let sphere = self.to_sphere().await?; + + let peer_sphere = match sphere + .traverse_by_petnames(petname_path, &replicate) + .await? + { + Some(sphere) => sphere, + None => return Ok(None), + }; + + let mut db = sphere.store().clone(); + let peer_identity = peer_sphere.get_identity().await?; + let local_version = db.get_version(&peer_identity).await?.map(|cid| cid.into()); + + let should_update_version = if let Some(since) = local_version { + let since_memo = Sphere::at(&since, &db).to_memo().await?; + let latest_memo = peer_sphere.to_memo().await?; + + since_memo.lamport_order() < latest_memo.lamport_order() + } else { + true + }; + + if should_update_version { + debug!( + "Updating local version of {} to more recent revision {}", + peer_identity, + peer_sphere.cid() + ); + + db.set_version(&peer_identity, peer_sphere.cid()).await?; + } + + let peer_sphere_context = self + .sphere_context() + .await? + .to_visitor(&peer_identity) + .await?; + + Ok(Some(SphereCursor::mounted_at( + C::wrap(peer_sphere_context).await, + peer_sphere.cid(), + ))) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use anyhow::Result; + use noosphere_storage::UcanStore; + use tokio::io::AsyncReadExt; + + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + use crate::{ + context::{ + HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, + SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, + }, + data::{ContentType, Header}, + helpers::{ + make_sphere_context_with_peer_chain, make_valid_link_record, simulated_sphere_context, + SimulationAccess, + }, + tracing::initialize_tracing, + }; + + use super::SphereCursor; + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_unlink_slugs_from_the_content_space() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are great".as_ref(), + None, + ) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + + assert!(cursor.read("cats").await.unwrap().is_some()); + + cursor.remove("cats").await.unwrap(); + cursor.save(None).await.unwrap(); + + assert!(cursor.read("cats").await.unwrap().is_none()); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_flushes_on_every_save() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let initial_stats = { + sphere_context + .lock() + .await + .db() + .to_block_store() + .to_stats() + .await + }; + let mut cursor = SphereCursor::latest(sphere_context.clone()); + + cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are great".as_ref(), + None, + ) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + + let first_save_stats = { + sphere_context + .lock() + .await + .db() + .to_block_store() + .to_stats() + .await + }; + + assert_eq!(first_save_stats.flushes, initial_stats.flushes + 1); + + cursor.remove("cats").await.unwrap(); + cursor.save(None).await.unwrap(); + + let second_save_stats = { + sphere_context + .lock() + .await + .db() + .to_block_store() + .to_stats() + .await + }; + + assert_eq!(second_save_stats.flushes, first_save_stats.flushes + 1); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_does_not_allow_writes_when_an_author_has_read_only_access() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::Readonly, None) + .await + .unwrap(); + + let mut cursor = SphereCursor::latest(sphere_context); + + let write_result = cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are great".as_ref(), + None, + ) + .await; + + assert!(write_result.is_err()); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_write_a_file_and_read_it_back() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are great".as_ref(), + None, + ) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + + let mut file = cursor.read("cats").await.unwrap().unwrap(); + + file.memo + .expect_header(&Header::ContentType, &ContentType::Subtext) + .unwrap(); + + let mut value = String::new(); + file.contents.read_to_string(&mut value).await.unwrap(); + + assert_eq!("Cats are great", value.as_str()); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_overwrite_a_file_with_new_contents_and_preserve_history() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are great".as_ref(), + None, + ) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + + cursor + .write( + "cats", + &ContentType::Subtext, + b"Cats are better than dogs".as_ref(), + None, + ) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + + let mut file = cursor.read("cats").await.unwrap().unwrap(); + + file.memo + .expect_header(&Header::ContentType, &ContentType::Subtext) + .unwrap(); + + let mut value = String::new(); + file.contents.read_to_string(&mut value).await.unwrap(); + + assert_eq!("Cats are better than dogs", value.as_str()); + + assert!(cursor.rewind().await.unwrap().is_some()); + + file = cursor.read("cats").await.unwrap().unwrap(); + + file.memo + .expect_header(&Header::ContentType, &ContentType::Subtext) + .unwrap(); + + value.clear(); + file.contents.read_to_string(&mut value).await.unwrap(); + + assert_eq!("Cats are great", value.as_str()); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_throws_an_error_when_saving_without_changes() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + let result = cursor.save(None).await; + + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "No changes to save"); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_throws_an_error_when_saving_with_empty_mutation_and_empty_headers() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + let result = cursor.save(Some(vec![])).await; + + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "No changes to save"); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_get_all_petnames_assigned_to_an_identity() -> Result<()> { + let (sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let mut db = UcanStore(sphere_context.sphere_context().await?.db().clone()); + + let (peer_1, link_record_1, _) = make_valid_link_record(&mut db).await?; + let (peer_2, link_record_2, _) = make_valid_link_record(&mut db).await?; + let (peer_3, link_record_3, _) = make_valid_link_record(&mut db).await?; + + let mut cursor = SphereCursor::latest(sphere_context); + + cursor + .set_petname("foo1", Some(link_record_1.to_sphere_identity())) + .await?; + cursor + .set_petname("bar1", Some(link_record_1.to_sphere_identity())) + .await?; + cursor + .set_petname("baz1", Some(link_record_1.to_sphere_identity())) + .await?; + + cursor + .set_petname("foo2", Some(link_record_2.to_sphere_identity())) + .await?; + cursor.save(None).await?; + + cursor.set_petname_record("foo1", &link_record_1).await?; + cursor.set_petname_record("bar1", &link_record_1).await?; + cursor.set_petname_record("baz1", &link_record_1).await?; + + cursor.set_petname_record("foo2", &link_record_2).await?; + + cursor.save(None).await?; + + assert_eq!( + cursor.get_assigned_petnames(&peer_1).await?, + vec![ + String::from("foo1"), + String::from("bar1"), + String::from("baz1") + ] + ); + + assert_eq!( + cursor.get_assigned_petnames(&peer_2).await?, + vec![String::from("foo2")] + ); + + assert_eq!( + cursor.get_assigned_petnames(&peer_3).await?, + Vec::::new() + ); + + // Check one more time for good measure, since results are cached internally + assert_eq!( + cursor.get_assigned_petnames(&peer_1).await?, + vec![ + String::from("foo1"), + String::from("bar1"), + String::from("baz1") + ] + ); + + cursor + .set_petname("bar2", Some(link_record_2.to_sphere_identity())) + .await?; + cursor + .set_petname("foo3", Some(link_record_3.to_sphere_identity())) + .await?; + cursor.save(None).await?; + + cursor.set_petname_record("bar2", &link_record_2).await?; + cursor.set_petname_record("foo3", &link_record_3).await?; + cursor.save(None).await?; + + assert_eq!( + cursor.get_assigned_petnames(&peer_1).await?, + vec![ + String::from("foo1"), + String::from("bar1"), + String::from("baz1") + ] + ); + + assert_eq!( + cursor.get_assigned_petnames(&peer_2).await?, + vec![String::from("bar2"), String::from("foo2")] + ); + + assert_eq!( + cursor.get_assigned_petnames(&peer_3).await?, + vec![String::from("foo3")] + ); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_traverse_a_sequence_of_petnames() -> Result<()> { + initialize_tracing(None); + + let name_seqeuence: Vec = vec!["a".into(), "b".into(), "c".into()]; + let (origin_sphere_context, _) = + make_sphere_context_with_peer_chain(&name_seqeuence).await?; + + let cursor = SphereCursor::latest(Arc::new( + origin_sphere_context.sphere_context().await?.clone(), + )); + + let target_sphere_context = cursor + .traverse_by_petnames(&name_seqeuence.into_iter().rev().collect::>()) + .await? + .unwrap(); + + let mut name = String::new(); + let mut file = target_sphere_context.read("my-name").await?.unwrap(); + file.contents.read_to_string(&mut name).await?; + + assert_eq!(name.as_str(), "c"); + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_resolves_none_when_a_petname_is_missing_from_the_sequence() -> Result<()> { + initialize_tracing(None); + + let name_sequence: Vec = vec!["b".into(), "c".into()]; + let (origin_sphere_context, _) = + make_sphere_context_with_peer_chain(&name_sequence).await?; + + let cursor = SphereCursor::latest(Arc::new( + origin_sphere_context.sphere_context().await?.clone(), + )); + + let traversed_sequence: Vec = vec!["a".into(), "b".into(), "c".into()]; + + let target_sphere_context = cursor + .traverse_by_petnames( + &traversed_sequence + .into_iter() + .rev() + .collect::>(), + ) + .await + .unwrap(); + + assert!(target_sphere_context.is_none()); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_correctly_identifies_a_visited_perer() -> Result<()> { + initialize_tracing(None); + + let name_sequence: Vec = vec!["a".into(), "b".into(), "c".into()]; + + let (origin_sphere_context, dids) = + make_sphere_context_with_peer_chain(&name_sequence).await?; + + let cursor = SphereCursor::latest(Arc::new( + origin_sphere_context.sphere_context().await?.clone(), + )); + + let mut target_sphere_context = cursor; + let mut identities = vec![target_sphere_context.identity().await?]; + + for name in name_sequence.iter() { + target_sphere_context = target_sphere_context + .traverse_by_petnames(&[name.clone()]) + .await? + .unwrap(); + identities.push(target_sphere_context.identity().await?); + } + + assert_eq!(identities.into_iter().rev().collect::>(), dids); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_traverse_a_sequence_of_petnames_one_at_a_time() -> Result<()> { + initialize_tracing(None); + + let name_sequence: Vec = vec!["a".into(), "b".into(), "c".into()]; + + let (origin_sphere_context, _) = + make_sphere_context_with_peer_chain(&name_sequence).await?; + + let cursor = SphereCursor::latest(Arc::new( + origin_sphere_context.sphere_context().await?.clone(), + )); + + let mut target_sphere_context = cursor; + + for name in name_sequence.iter() { + target_sphere_context = target_sphere_context + .traverse_by_petnames(&[name.clone()]) + .await? + .unwrap(); + } + + let mut name = String::new(); + let mut file = target_sphere_context + .read("my-name") + .await + .unwrap() + .unwrap(); + file.contents.read_to_string(&mut name).await.unwrap(); + + assert_eq!(name.as_str(), "c"); + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/context/has.rs b/rust/noosphere-core/src/context/has.rs new file mode 100644 index 000000000..b9fca8f5c --- /dev/null +++ b/rust/noosphere-core/src/context/has.rs @@ -0,0 +1,223 @@ +use crate::{ + authority::Author, + data::{Did, Link, MemoIpld}, + view::Sphere, +}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use noosphere_storage::{SphereDb, Storage}; +use std::{ + ops::{Deref, DerefMut}, + sync::Arc, +}; +use tokio::sync::{Mutex, OwnedMutexGuard}; + +use crate::context::SphereContextKey; + +use super::SphereContext; + +#[allow(missing_docs)] +#[cfg(not(target_arch = "wasm32"))] +pub trait HasConditionalSendSync: Send + Sync {} + +#[cfg(not(target_arch = "wasm32"))] +impl HasConditionalSendSync for S where S: Send + Sync {} + +#[allow(missing_docs)] +#[cfg(target_arch = "wasm32")] +pub trait HasConditionalSendSync {} + +#[cfg(target_arch = "wasm32")] +impl HasConditionalSendSync for S {} + +/// Any container that can provide non-mutable access to a [SphereContext] +/// should implement [HasSphereContext]. The most common example of something +/// that may implement this trait is an `Arc>`. Implementors +/// of this trait will automatically implement other traits that provide +/// convience methods for accessing different parts of the sphere, such as +/// content and petnames. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait HasSphereContext: Clone + HasConditionalSendSync +where + S: Storage + 'static, +{ + /// The type of the internal read-only [SphereContext] + type SphereContext: Deref> + HasConditionalSendSync; + + /// Get the [SphereContext] that is made available by this container. + async fn sphere_context(&self) -> Result; + + /// Get the DID identity of the sphere that this FS view is reading from and + /// writing to + async fn identity(&self) -> Result { + let sphere_context = self.sphere_context().await?; + + Ok(sphere_context.identity().clone()) + } + + /// The CID of the most recent local version of this sphere + async fn version(&self) -> Result> { + self.sphere_context().await?.version().await + } + + /// Get a data view into the sphere at the current revision + async fn to_sphere(&self) -> Result>> { + let version = self.version().await?; + Ok(Sphere::at(&version, self.sphere_context().await?.db())) + } + + /// Create a new [SphereContext] via [SphereContext::with_author] and wrap it in the same + /// [HasSphereContext] implementation, returning the result + async fn with_author(&self, author: &Author) -> Result { + Ok(Self::wrap(self.sphere_context().await?.with_author(author).await?).await) + } + + /// Wrap a given [SphereContext] in this [HasSphereContext] + async fn wrap(sphere_context: SphereContext) -> Self; +} + +/// Any container that can provide mutable access to a [SphereContext] should +/// implement [HasMutableSphereContext]. The most common example of something +/// that may implement this trait is `Arc>>`. +/// Implementors of this trait will automatically implement other traits that +/// provide convenience methods for modifying the contents, petnames and other +/// aspects of a sphere. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait HasMutableSphereContext: HasSphereContext + HasConditionalSendSync +where + S: Storage + 'static, +{ + /// The type of the internal mutable [SphereContext] + type MutableSphereContext: Deref> + + DerefMut> + + HasConditionalSendSync; + + /// Get a mutable reference to the [SphereContext] that is wrapped by this + /// container. + async fn sphere_context_mut(&mut self) -> Result; + + /// Returns true if any changes have been made to the underlying + /// [SphereContext] that have not been committed to the associated sphere + /// yet (according to local history). + async fn has_unsaved_changes(&self) -> Result { + let context = self.sphere_context().await?; + Ok(!context.mutation().is_empty()) + } + + /// Commits a series of writes to the sphere and signs the new version. The + /// new version [Link] of the sphere is returned. This method must + /// be invoked in order to update the local history of the sphere with any + /// changes that have been made. + async fn save( + &mut self, + additional_headers: Option>, + ) -> Result> { + let sphere = self.to_sphere().await?; + let mut sphere_context = self.sphere_context_mut().await?; + let sphere_identity = sphere_context.identity().clone(); + let mut revision = sphere.apply_mutation(sphere_context.mutation()).await?; + + match additional_headers { + Some(headers) if !headers.is_empty() => revision.memo.replace_headers(headers), + _ if sphere_context.mutation().is_empty() => return Err(anyhow!("No changes to save")), + _ => (), + } + + let new_sphere_version = revision + .sign( + &sphere_context.author().key, + sphere_context.author().authorization.as_ref(), + ) + .await?; + + sphere_context + .db_mut() + .set_version(&sphere_identity, &new_sphere_version) + .await?; + sphere_context.db_mut().flush().await?; + sphere_context.mutation_mut().reset(); + + Ok(new_sphere_version) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasSphereContext for Arc>> +where + S: Storage + 'static, +{ + type SphereContext = OwnedMutexGuard>; + + async fn sphere_context(&self) -> Result { + Ok(self.clone().lock_owned().await) + } + + async fn wrap(sphere_context: SphereContext) -> Self { + Arc::new(Mutex::new(sphere_context)) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasSphereContext for Box +where + T: HasSphereContext, + S: Storage + 'static, +{ + type SphereContext = T::SphereContext; + + async fn sphere_context(&self) -> Result { + T::sphere_context(self).await + } + + async fn wrap(sphere_context: SphereContext) -> Self { + Box::new(T::wrap(sphere_context).await) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasSphereContext for Arc> +where + S: Storage, +{ + type SphereContext = Arc>; + + async fn sphere_context(&self) -> Result { + Ok(self.clone()) + } + + async fn wrap(sphere_context: SphereContext) -> Self { + Arc::new(sphere_context) + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasMutableSphereContext for Arc>> +where + S: Storage + 'static, +{ + type MutableSphereContext = OwnedMutexGuard>; + + async fn sphere_context_mut(&mut self) -> Result { + self.sphere_context().await + } +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl HasMutableSphereContext for Box +where + T: HasMutableSphereContext, + S: Storage + 'static, +{ + type MutableSphereContext = T::MutableSphereContext; + + async fn sphere_context_mut(&mut self) -> Result { + T::sphere_context_mut(self).await + } +} diff --git a/rust/noosphere-core/src/context/internal.rs b/rust/noosphere-core/src/context/internal.rs new file mode 100644 index 000000000..af05fb5c6 --- /dev/null +++ b/rust/noosphere-core/src/context/internal.rs @@ -0,0 +1,105 @@ +use super::{BodyChunkDecoder, SphereFile}; +use crate::{ + context::{AsyncFileBody, HasSphereContext}, + stream::put_block_stream, +}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use noosphere_storage::{BlockStore, Storage}; +use std::str::FromStr; +use tokio_util::io::StreamReader; + +use crate::{ + authority::Access, + data::{ContentType, Header, Link, MemoIpld}, +}; +use cid::Cid; + +/// A module-private trait for internal trait methods; this is a workaround for +/// the fact that all trait methods are implicitly public implementation +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub(crate) trait SphereContextInternal +where + S: Storage + 'static, +{ + /// Returns an error result if the configured author of the [SphereContext] + /// does not have write access to it (as a matter of cryptographic + /// authorization). + async fn assert_write_access(&self) -> Result<()>; + + async fn get_file( + &self, + sphere_revision: &Cid, + memo_link: Link, + ) -> Result>>; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereContextInternal for C +where + C: HasSphereContext, + S: Storage + 'static, +{ + async fn assert_write_access(&self) -> Result<()> { + let sphere_context = self.sphere_context().await?; + match sphere_context.access().await? { + Access::ReadOnly => Err(anyhow!( + "Cannot mutate sphere; author only has read access to its contents" + )), + _ => Ok(()), + } + } + + async fn get_file( + &self, + sphere_revision: &Cid, + memo_link: Link, + ) -> Result>> { + let db = self.sphere_context().await?.db().clone(); + let memo = memo_link.load_from(&db).await?; + + // If we have a memo, but not the content it refers to, we should try to + // replicate from the gateway + if db.get_block(&memo.body).await?.is_none() { + let client = self + .sphere_context() + .await? + .client() + .await + .map_err(|error| { + warn!("Unable to initialize API client for replicating missing content"); + error + })?; + + // NOTE: This is kind of a hack, since we may be accessing a + // "read-only" context. Technically this should be acceptable + // because our mutation here is propagating immutable blocks + // into the local DB + let stream = client.replicate(&memo_link, None).await?; + + put_block_stream(db.clone(), stream).await?; + } + + let content_type = match memo.get_first_header(&Header::ContentType) { + Some(content_type) => Some(ContentType::from_str(content_type.as_str())?), + None => None, + }; + + let stream = match content_type { + // TODO(#86): Content-type aware decoding of body bytes + Some(_) => BodyChunkDecoder(&memo.body, &db).stream(), + None => return Err(anyhow!("No content type specified")), + }; + + Ok(SphereFile { + sphere_identity: self.sphere_context().await?.identity().clone(), + sphere_version: sphere_revision.into(), + memo_version: memo_link, + memo, + // NOTE: we have to box here because traits don't support `impl` types in return values + contents: Box::new(StreamReader::new(stream)), + }) + } +} diff --git a/rust/noosphere-core/src/context/metadata.rs b/rust/noosphere-core/src/context/metadata.rs new file mode 100644 index 000000000..ce30a5a7f --- /dev/null +++ b/rust/noosphere-core/src/context/metadata.rs @@ -0,0 +1,36 @@ +//! These constants represent the metadata keys used when a [SphereContext] is +//! is initialized. Since these represent somewhat free-form key/values in the +//! storage layer, we are make a best effort to document them here. + +#[cfg(doc)] +use crate::context::SphereContext; + +#[cfg(doc)] +use crate::data::Did; + +#[cfg(doc)] +use cid::Cid; + +#[cfg(doc)] +use url::Url; + +/// A key that corresponds to the sphere's identity, which is represented by a +/// [Did] when it is set. +pub const IDENTITY: &str = "identity"; + +/// A name that corresponds to the locally available key. This name is +/// represented as a string, and should match a credential ID for a key in +/// whatever the supported platform key storage is. +pub const USER_KEY_NAME: &str = "user_key_name"; + +/// The [Cid] of a UCAN JWT that authorizes the configured user key to access +/// the sphere. +pub const AUTHORIZATION: &str = "authorization"; + +/// The base [Url] of a Noosphere Gateway API that will allow this sphere to +/// sync with it. +pub const GATEWAY_URL: &str = "gateway_url"; + +/// The counterpart sphere [Did] that either tracks or is tracked by this +/// sphere. +pub const COUNTERPART: &str = "counterpart"; diff --git a/rust/noosphere-core/src/context/mod.rs b/rust/noosphere-core/src/context/mod.rs new file mode 100644 index 000000000..c95cd0ab9 --- /dev/null +++ b/rust/noosphere-core/src/context/mod.rs @@ -0,0 +1,94 @@ +//! This module implements content, petname and other forms of acccess to +//! spheres. If you have storage and network primitives on your platform, you +//! can initialize a [SphereContext] and use it to work with and synchronize +//! spheres, as well as traverse the broader Noosphere data graph. +//! +//! In order to initialize a [SphereContext], you need a [Did] (like an ID) for +//! a Sphere, a [Storage] primitive and an [Author] (which represents whoever or +//! whatever is trying to access the Sphere inquestion). +//! +//! Once you have a [SphereContext], you can begin reading from, writing to and +//! traversing the Noosphere content graph. +//! +//! ```rust +//! # use anyhow::Result; +//! # use noosphere_core::context::{SphereCursor, HasMutableSphereContext, SphereContentWrite}; +//! # +//! # #[cfg(feature = "helpers")] +//! # use noosphere_core::helpers::{simulated_sphere_context,SimulationAccess}; +//! # +//! # #[cfg(feature = "helpers")] +//! # #[tokio::main(flavor = "multi_thread")] +//! # async fn main() -> Result<()> { +//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # +//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?; +//! sphere_context.save(None).await?; +//! # +//! # Ok(()) +//! # } +//! # +//! # #[cfg(not(feature = "helpers"))] +//! # fn main() {} +//! ``` +//! +//! You can also use a [SphereContext] to access petnames in the sphere: +//! +//! ```rust +//! # use anyhow::Result; +//! # #[cfg(feature = "helpers")] +//! # use noosphere_core::{ +//! # helpers::{simulated_sphere_context,SimulationAccess}, +//! # data::Did, +//! # context::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite} +//! # }; +//! # +//! # #[cfg(feature = "helpers")] +//! # #[tokio::main(flavor = "multi_thread")] +//! # async fn main() -> Result<()> { +//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # +//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?; +//! sphere_context.save(None).await?; +//! # +//! # Ok(()) +//! # } +//! # +//! # #[cfg(not(feature = "helpers"))] +//! # fn main() {} +//! ``` +//! +//! + +#![warn(missing_docs)] + +#[cfg(doc)] +use crate::{authority::Author, data::Did}; + +#[cfg(doc)] +use noosphere_storage::Storage; + +mod authority; +mod content; +#[allow(clippy::module_inception)] +mod context; +mod cursor; +mod has; +mod replication; +mod walker; + +mod internal; +pub mod metadata; +mod petname; +mod sync; + +pub use authority::*; +pub use content::*; +pub use context::*; +pub use cursor::*; +pub use has::*; +pub use metadata::*; +pub use petname::*; +pub use replication::*; +pub use sync::*; +pub use walker::*; diff --git a/rust/noosphere-core/src/context/petname/mod.rs b/rust/noosphere-core/src/context/petname/mod.rs new file mode 100644 index 000000000..b1ad86483 --- /dev/null +++ b/rust/noosphere-core/src/context/petname/mod.rs @@ -0,0 +1,10 @@ +//! Sphere petnames are shorthand names that are associated with DIDs. A petname +//! can be any string, and always refers to a DID. The DID, in turn, may be +//! resolved to a CID that represents the tip of history for the sphere that is +//! implicitly identified by the provided DID. + +mod read; +mod write; + +pub use read::*; +pub use write::*; diff --git a/rust/noosphere-core/src/context/petname/read.rs b/rust/noosphere-core/src/context/petname/read.rs new file mode 100644 index 000000000..baa29e803 --- /dev/null +++ b/rust/noosphere-core/src/context/petname/read.rs @@ -0,0 +1,138 @@ +use std::collections::BTreeMap; + +use crate::data::{Did, Link, LinkRecord, MemoIpld}; +use anyhow::Result; +use async_trait::async_trait; +use cid::Cid; +use futures_util::TryStreamExt; +use noosphere_storage::{KeyValueStore, Storage}; + +use crate::context::{HasSphereContext, SphereWalker}; + +/// Anything that provides read access to petnames in a sphere should implement +/// [SpherePetnameRead]. A blanket implementation is provided for any container +/// that implements [HasSphereContext]. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SpherePetnameRead +where + S: Storage + 'static, +{ + /// Get the [Did] that is assigned to a petname, if any + async fn get_petname(&self, name: &str) -> Result>; + + /// Resolve the petname via its assigned [Did] to a [Cid] that refers to a + /// point in history of a sphere + async fn resolve_petname(&self, name: &str) -> Result>>; + + /// Given a [Did], get all the petnames that have been assigned to it + /// in this sphere + async fn get_assigned_petnames(&self, did: &Did) -> Result>; + + /// Given a petname, get the raw last known [LinkRecord] for that peer + async fn get_petname_record(&self, name: &str) -> Result>; +} + +fn assigned_petnames_cache_key(origin: &Did, peer: &Did, origin_version: &Cid) -> String { + format!( + "noosphere:cache:petname:assigned:{}:{}:{}", + origin, peer, origin_version + ) +} + +fn sphere_checkpoint_cache_key(origin: &Did, origin_version: &Cid) -> String { + format!( + "noosphere:cache:petname:checkpoint:{}:{}", + origin, origin_version + ) +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SpherePetnameRead for C +where + C: HasSphereContext, + S: Storage + 'static, +{ + #[instrument(skip(self))] + async fn get_assigned_petnames(&self, peer: &Did) -> Result> { + let version = self.version().await?; + let origin = self.identity().await?; + + debug!("Getting petnames assigned in {origin} at version {version}"); + + let mut db = self.sphere_context().await?.db().clone(); + let key = assigned_petnames_cache_key(&origin, peer, &version); + + if let Some(names) = db.get_key::<_, Vec>(key).await? { + return Ok(names); + } + + let checkpoint_key = sphere_checkpoint_cache_key(&origin, &version); + + if db.get_key::<_, u8>(&checkpoint_key).await?.is_some() { + warn!("No names were assigned to {peer}",); + return Ok(vec![]); + } + + let walker = SphereWalker::from(self); + let petname_stream = walker.petname_stream(); + let mut did_petnames: BTreeMap> = BTreeMap::new(); + + tokio::pin!(petname_stream); + + while let Some((petname, identity)) = petname_stream.try_next().await? { + match did_petnames.get_mut(&identity.did) { + Some(petnames) => { + petnames.push(petname); + } + None => { + did_petnames.insert(identity.did, vec![petname]); + } + }; + } + + let mut assigned_petnames = None; + + for (did, petnames) in did_petnames { + if &did == peer { + assigned_petnames = Some(petnames.clone()); + } + + let key = assigned_petnames_cache_key(&origin, &did, &version); + db.set_key(key, petnames).await?; + } + + db.set_key(checkpoint_key, 1u8).await?; + + Ok(assigned_petnames.unwrap_or_default()) + } + + async fn get_petname(&self, name: &str) -> Result> { + let sphere = self.to_sphere().await?; + let identities = sphere.get_address_book().await?.get_identities().await?; + let address_ipld = identities.get(&name.to_string()).await?; + + Ok(address_ipld.map(|ipld| ipld.did.clone())) + } + + async fn resolve_petname(&self, name: &str) -> Result>> { + Ok(match self.get_petname_record(name).await? { + Some(link_record) => link_record.get_link(), + None => None, + }) + } + + async fn get_petname_record(&self, name: &str) -> Result> { + let sphere = self.to_sphere().await?; + let identities = sphere.get_address_book().await?.get_identities().await?; + let address_ipld = identities.get(&name.to_string()).await?; + + trace!("Recorded address for {name}: {:?}", address_ipld); + + Ok(match address_ipld { + Some(identity) => identity.link_record(sphere.store()).await, + None => None, + }) + } +} diff --git a/rust/noosphere-core/src/context/petname/write.rs b/rust/noosphere-core/src/context/petname/write.rs new file mode 100644 index 000000000..a782a936c --- /dev/null +++ b/rust/noosphere-core/src/context/petname/write.rs @@ -0,0 +1,192 @@ +use crate::data::{Did, IdentityIpld, LinkRecord}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use noosphere_storage::Storage; +use ucan::store::UcanJwtStore; + +use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, SpherePetnameRead}; + +fn validate_petname(petname: &str) -> Result<()> { + if petname.is_empty() { + Err(anyhow!("Petname must not be empty.")) + } else if petname.len() >= 4 && petname.starts_with("did:") { + Err(anyhow!("Petname must not be a DID.")) + } else { + Ok(()) + } +} + +/// Anything that can write petnames to a sphere should implement +/// [SpherePetnameWrite]. A blanket implementation is provided for anything that +/// implements [HasMutableSphereContext] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SpherePetnameWrite: SpherePetnameRead +where + S: Storage + 'static, +{ + /// Configure a petname, by assigning some [Did] to it or none. By assigning + /// none, the petname is implicitly removed from the address space (note: + /// this does not erase the name from historical versions of the sphere). If + /// a name is set that already exists, the previous name shall be + /// overwritten by the new one, and any associated [Jwt] shall be unset. + async fn set_petname(&mut self, name: &str, identity: Option) -> Result<()>; + + /// Set the [LinkRecord] associated with a petname. The [LinkRecord] must + /// resolve a valid UCAN that authorizes the corresponding sphere to be + /// published and grants sufficient authority from the configured [Did] to + /// the publisher. The audience of the UCAN must match the [Did] that was + /// most recently assigned the associated petname. Note that a petname + /// _must_ be assigned to the audience [Did] in order for the record to be + /// set. + async fn set_petname_record(&mut self, name: &str, record: &LinkRecord) -> Result>; + + /// Deprecated; use [SpherePetnameWrite::set_petname_record] instead + #[deprecated(note = "Use set_petname_record instead")] + async fn adopt_petname(&mut self, name: &str, record: &LinkRecord) -> Result>; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SpherePetnameWrite for C +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + async fn set_petname(&mut self, name: &str, identity: Option) -> Result<()> { + self.assert_write_access().await?; + validate_petname(name)?; + + if identity.is_some() + && self.sphere_context().await?.identity() == identity.as_ref().unwrap() + { + return Err(anyhow!("Sphere cannot assign itself to a petname.")); + } + + let current_address = self.get_petname(name).await?; + + if identity != current_address { + let mut context = self.sphere_context_mut().await?; + match identity { + Some(identity) => { + context.mutation_mut().identities_mut().set( + &name.to_string(), + &IdentityIpld { + did: identity, + // TODO: We should backfill this if we have already resolved + // this address by another name + link_record: None, + }, + ); + } + None => context + .mutation_mut() + .identities_mut() + .remove(&name.to_string()), + }; + } + + Ok(()) + } + + async fn adopt_petname(&mut self, name: &str, record: &LinkRecord) -> Result> { + self.set_petname_record(name, record).await + } + + async fn set_petname_record(&mut self, name: &str, record: &LinkRecord) -> Result> { + // NOTE: it is not safe for us to blindly adopt link records that don't + // match up with the petname we are adopting them against. For example, + // consider the following sequence of events: + // + // 1. A petname is assigned to a DID + // 2. During sync, the gateway kicks off a parallel job to resolve the + // petname + // 3. Meanwhile, we unassign the petname + // 4. We sync, the gateway takes no action (no new names to resolve) + // 5. Then, the original resolve job finishes and comes back with a + // record + // + // Record adoption is not able to disambiguate between between a new + // record being added and a race condition like the one described above. + self.assert_write_access().await?; + validate_petname(name)?; + + let identity = record.to_sphere_identity(); + let expected_identity = self.get_petname(name).await?; + + match expected_identity { + Some(expected_identity) => { + if expected_identity != identity { + return Err(anyhow!( + "Cannot adopt petname record for '{}'; expected record for {} but got record for {}", + name, + expected_identity, + identity + )); + } + } + None => { + return Err(anyhow!( + "Cannot adopt petname record for '{}' (not assigned to a sphere identity)", + name + )); + } + }; + + if self.sphere_context().await?.identity() == &identity { + return Err(anyhow!("Sphere cannot assign itself to a petname.")); + } + + if let Some(existing_record) = self.get_petname_record(name).await? { + if !existing_record.superceded_by(record) { + return Err(anyhow!( + "Previously stored record supercedes provided record." + )); + } + } + + let cid = self + .sphere_context_mut() + .await? + .db_mut() + .write_token(&record.encode()?) + .await?; + + // TODO: Validate the record as a UCAN + + debug!( + "Adopting '{}' ({}), resolving to {}...", + name, identity, record + ); + + let new_address = IdentityIpld { + did: identity.clone(), + link_record: Some(cid.into()), + }; + + let identities = self + .sphere_context() + .await? + .sphere() + .await? + .get_address_book() + .await? + .get_identities() + .await?; + let previous_identity = identities.get(&name.into()).await?; + + self.sphere_context_mut() + .await? + .mutation_mut() + .identities_mut() + .set(&name.into(), &new_address); + + if let Some(previous_identity) = previous_identity { + if identity != previous_identity.did { + return Ok(Some(previous_identity.did.to_owned())); + } + } + + Ok(None) + } +} diff --git a/rust/noosphere-core/src/context/replication/mod.rs b/rust/noosphere-core/src/context/replication/mod.rs new file mode 100644 index 000000000..cd2e912fe --- /dev/null +++ b/rust/noosphere-core/src/context/replication/mod.rs @@ -0,0 +1,3 @@ +mod read; + +pub use read::*; diff --git a/rust/noosphere-core/src/context/replication/read.rs b/rust/noosphere-core/src/context/replication/read.rs new file mode 100644 index 000000000..3318b9d3c --- /dev/null +++ b/rust/noosphere-core/src/context/replication/read.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use async_trait::async_trait; +use noosphere_storage::Storage; + +/// Implementors are able to traverse from one sphere to the next via +/// the address book entries found in those spheres +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereReplicaRead: Sized +where + S: Storage + 'static, +{ + /// Accepts a linear sequence of petnames and attempts to recursively + /// traverse through spheres using that sequence. The sequence is traversed + /// from back to front. So, if the sequence is "gold", "cat", "bob", it will + /// traverse to bob, then to bob's cat, then to bob's cat's gold. + async fn traverse_by_petnames(&self, petnames: &[String]) -> Result>; +} diff --git a/rust/noosphere-core/src/context/sync/error.rs b/rust/noosphere-core/src/context/sync/error.rs new file mode 100644 index 000000000..5295155cc --- /dev/null +++ b/rust/noosphere-core/src/context/sync/error.rs @@ -0,0 +1,29 @@ +use crate::api::v0alpha2::PushError; +use thiserror::Error; + +/// Different classes of error that may occur during synchronization with a +/// gateway +#[derive(Error, Debug)] +pub enum SyncError { + /// The error was a conflict; this is possibly recoverable + #[error("There was a conflict during sync")] + Conflict, + /// The error was some other, non-specific error + #[error("{0}")] + Other(anyhow::Error), +} + +impl From for SyncError { + fn from(value: anyhow::Error) -> Self { + SyncError::Other(value) + } +} + +impl From for SyncError { + fn from(value: PushError) -> Self { + match value { + PushError::Conflict => SyncError::Conflict, + any => SyncError::Other(any.into()), + } + } +} diff --git a/rust/noosphere-core/src/context/sync/mod.rs b/rust/noosphere-core/src/context/sync/mod.rs new file mode 100644 index 000000000..1ce2773e9 --- /dev/null +++ b/rust/noosphere-core/src/context/sync/mod.rs @@ -0,0 +1,9 @@ +mod error; +mod recovery; +mod strategy; +mod write; + +pub use error::*; +pub use recovery::*; +pub use strategy::*; +pub use write::*; diff --git a/rust/noosphere-core/src/context/sync/recovery.rs b/rust/noosphere-core/src/context/sync/recovery.rs new file mode 100644 index 000000000..28bcdfb0d --- /dev/null +++ b/rust/noosphere-core/src/context/sync/recovery.rs @@ -0,0 +1,9 @@ +/// Recovery strategies for cases when gateway synchronization fails but may be +/// able to recover gracefully (e.g., when the gateway reports a conflict). +#[derive(Debug)] +pub enum SyncRecovery { + /// Do not attempt to recover + None, + /// Automatically retry the synchronization up to a certain number of times + Retry(u32), +} diff --git a/rust/noosphere-core/src/context/sync/strategy.rs b/rust/noosphere-core/src/context/sync/strategy.rs new file mode 100644 index 000000000..8587f5ac9 --- /dev/null +++ b/rust/noosphere-core/src/context/sync/strategy.rs @@ -0,0 +1,467 @@ +use std::{collections::BTreeMap, marker::PhantomData}; + +use crate::{ + api::{ + v0alpha1::FetchParameters, + v0alpha2::{PushBody, PushResponse}, + }, + stream::put_block_stream, +}; +use crate::{ + authority::{generate_capability, SphereAbility}, + data::{Did, IdentityIpld, Jwt, Link, MemoIpld, LINK_RECORD_FACT_NAME}, + view::{Sphere, Timeline}, +}; +use anyhow::{anyhow, Result}; +use noosphere_storage::{KeyValueStore, SphereDb, Storage}; +use tokio_stream::StreamExt; +use ucan::builder::UcanBuilder; + +use crate::context::{ + metadata::COUNTERPART, HasMutableSphereContext, SpherePetnameRead, SpherePetnameWrite, + SyncError, +}; + +type HandshakeResults = (Option>, Did, Option>); +type FetchResults = ( + Link, + Link, + BTreeMap, +); +type CounterpartHistory = Vec, Sphere>)>>; + +/// The default synchronization strategy is a git-like fetch->rebase->push flow. +/// It depends on the corresponding history of a "counterpart" sphere that is +/// owned by a gateway server. As revisions are pushed to the gateway server, it +/// updates its own sphere to point to the tip of the latest lineage of the +/// user's. When a new change needs to be synchronized, the latest history of +/// the counterpart sphere is first fetched, and the local changes are rebased +/// on the counterpart sphere's reckoning of the authoritative lineage of the +/// user's sphere. Finally, after the rebase, the reconciled local lineage is +/// pushed to the gateway. +pub struct GatewaySyncStrategy +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + has_context_type: PhantomData, + store_type: PhantomData, +} + +impl Default for GatewaySyncStrategy +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + fn default() -> Self { + Self { + has_context_type: Default::default(), + store_type: Default::default(), + } + } +} + +impl GatewaySyncStrategy +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + /// Synchronize a local sphere's data with the data in a gateway, and rollback + /// if there is an error. The returned [Link] is the latest version of the local + /// sphere lineage after the sync has completed. + pub async fn sync(&self, context: &mut C) -> Result, SyncError> + where + C: HasMutableSphereContext, + { + let (local_sphere_version, counterpart_sphere_identity, counterpart_sphere_version) = + self.handshake(context).await?; + + let result: Result, anyhow::Error> = { + let (mut local_sphere_version, counterpart_sphere_version, updated_names) = self + .fetch_remote_changes( + context, + local_sphere_version.as_ref(), + &counterpart_sphere_identity, + counterpart_sphere_version.as_ref(), + ) + .await?; + + if let Some(version) = self.adopt_names(context, updated_names).await? { + local_sphere_version = version; + } + + self.push_local_changes( + context, + &local_sphere_version, + &counterpart_sphere_identity, + &counterpart_sphere_version, + ) + .await?; + + Ok(local_sphere_version) + }; + + // Rollback if there is an error while syncing + if result.is_err() { + self.rollback( + context, + local_sphere_version.as_ref(), + &counterpart_sphere_identity, + counterpart_sphere_version.as_ref(), + ) + .await? + } + + Ok(result?) + } + + #[instrument(level = "debug", skip(self, context))] + async fn handshake(&self, context: &mut C) -> Result { + let mut context = context.sphere_context_mut().await?; + let client = context.client().await?; + let counterpart_sphere_identity = client.session.sphere_identity.clone(); + + // TODO(#561): Some kind of due diligence to notify the caller when this + // value changes + context + .db_mut() + .set_key(COUNTERPART, &counterpart_sphere_identity) + .await?; + + let local_sphere_identity = context.identity().clone(); + + let local_sphere_version = context.db().get_version(&local_sphere_identity).await?; + let counterpart_sphere_version = context + .db() + .get_version(&counterpart_sphere_identity) + .await?; + + Ok(( + local_sphere_version.map(|cid| cid.into()), + counterpart_sphere_identity, + counterpart_sphere_version.map(|cid| cid.into()), + )) + } + + /// Fetches the latest changes from a gateway and updates the local lineage + /// using a conflict-free rebase strategy + #[instrument(level = "debug", skip(self, context))] + async fn fetch_remote_changes( + &self, + context: &mut C, + local_sphere_tip: Option<&Link>, + counterpart_sphere_identity: &Did, + counterpart_sphere_base: Option<&Link>, + ) -> Result { + let mut context = context.sphere_context_mut().await?; + let local_sphere_identity = context.identity().clone(); + let client = context.client().await?; + + let fetch_response = client + .fetch(&FetchParameters { + since: counterpart_sphere_base.cloned(), + }) + .await?; + + let mut updated_names = BTreeMap::new(); + + let (counterpart_sphere_tip, block_stream) = match fetch_response { + Some((tip, stream)) => (tip, stream), + None => { + info!("Local history is already up to date..."); + let local_sphere_tip = context + .db() + .require_version(&local_sphere_identity) + .await? + .into(); + return Ok(( + local_sphere_tip, + counterpart_sphere_base + .ok_or_else(|| anyhow!("Counterpart sphere history is missing!"))? + .clone(), + updated_names, + )); + } + }; + + put_block_stream(context.db_mut().clone(), block_stream).await?; + + trace!("Finished putting block stream"); + + let counterpart_history: CounterpartHistory = + Sphere::at(&counterpart_sphere_tip, context.db_mut()) + .into_history_stream(counterpart_sphere_base) + .collect() + .await; + + trace!("Iterating over counterpart history"); + + for item in counterpart_history.into_iter().rev() { + let (_, sphere) = item?; + sphere.hydrate().await?; + updated_names.append( + &mut sphere + .get_address_book() + .await? + .get_identities() + .await? + .get_added() + .await?, + ); + } + + let local_sphere_old_base = match counterpart_sphere_base { + Some(counterpart_sphere_base) => Sphere::at(counterpart_sphere_base, context.db()) + .get_content() + .await? + .get(&local_sphere_identity) + .await? + .cloned(), + None => None, + }; + let local_sphere_new_base = Sphere::at(&counterpart_sphere_tip, context.db()) + .get_content() + .await? + .get(&local_sphere_identity) + .await? + .cloned(); + + let local_sphere_tip = match ( + local_sphere_tip, + local_sphere_old_base, + local_sphere_new_base, + ) { + // History diverged, so rebase our local changes on the newly received branch + (Some(current_tip), Some(old_base), Some(new_base)) if old_base != new_base => { + info!( + ?current_tip, + ?old_base, + ?new_base, + "Syncing received local sphere revisions..." + ); + Sphere::at(current_tip, context.db()) + .rebase( + &old_base, + &new_base, + &context.author().key, + context.author().authorization.as_ref(), + ) + .await? + } + // No diverged history, just new linear history based on our local tip + (None, old_base, Some(new_base)) => { + info!("Hydrating received local sphere revisions..."); + let timeline = Timeline::new(context.db_mut()); + Sphere::hydrate_timeslice( + &timeline.slice(&new_base, old_base.as_ref()).exclude_past(), + ) + .await?; + + new_base.clone() + } + // No new history at all + (Some(current_tip), _, _) => { + info!("Nothing to sync!"); + current_tip.clone() + } + // We should have local history but we don't! + _ => { + return Err(anyhow!("Missing local history for sphere after sync!")); + } + }; + + context + .db_mut() + .set_version(&local_sphere_identity, &local_sphere_tip) + .await?; + + debug!("Setting counterpart sphere version to {counterpart_sphere_tip}"); + + context + .db_mut() + .set_version(counterpart_sphere_identity, &counterpart_sphere_tip) + .await?; + + Ok((local_sphere_tip, counterpart_sphere_tip, updated_names)) + } + + #[instrument(level = "debug", skip(self, context))] + async fn adopt_names( + &self, + context: &mut C, + updated_names: BTreeMap, + ) -> Result>> { + if updated_names.is_empty() { + return Ok(None); + } + info!( + "Considering {} updated link records for adoption...", + updated_names.len() + ); + + let db = context.sphere_context().await?.db().clone(); + + for (name, address) in updated_names.into_iter() { + if let Some(link_record) = address.link_record(&db).await { + if let Some(identity) = context.get_petname(&name).await? { + if identity != address.did { + warn!("Updated link record for {name} referred to unexpected sphere; expected {identity}, but record referred to {}; ignoring...", address.did); + continue; + } + + if context.resolve_petname(&name).await? == link_record.get_link() { + // TODO(#562): Should probably also verify record expiry + // in case we are dealing with a renewed record to the + // same link + debug!("Resolved got new link record for {name} but the link has not changed; skipping..."); + continue; + } + + if let Err(e) = context.set_petname_record(&name, &link_record).await { + warn!("Could not set petname record: {}", e); + continue; + } + } else { + debug!("Not adopting link record for {name}, which is no longer present in the address book") + } + } + } + + Ok(if context.has_unsaved_changes().await? { + Some(context.save(None).await?) + } else { + None + }) + } + + /// Attempts to push the latest local lineage to the gateway, causing the + /// gateway to update its own pointer to the tip of the local sphere's history + #[instrument(level = "debug", skip(self, context))] + async fn push_local_changes( + &self, + context: &mut C, + local_sphere_tip: &Link, + counterpart_sphere_identity: &Did, + counterpart_sphere_tip: &Link, + ) -> Result<(), SyncError> { + let mut context = context.sphere_context_mut().await?; + + let local_sphere_base = Sphere::at(counterpart_sphere_tip, context.db()) + .get_content() + .await? + .get(context.identity()) + .await? + .cloned(); + + if local_sphere_base.as_ref() == Some(local_sphere_tip) { + info!("Gateway is already up to date!"); + return Ok(()); + } + + info!("Collecting blocks from new local history..."); + debug!("Bundling until {:?}", local_sphere_base); + + let client = context.client().await?; + + let local_sphere_identity = context.identity(); + let authorization = context + .author() + .require_authorization()? + .as_ucan(context.db()) + .await?; + + let name_record = Jwt(UcanBuilder::default() + .issued_by(&context.author().key) + .for_audience(local_sphere_identity) + .witnessed_by(&authorization, None) + .claiming_capability(&generate_capability( + local_sphere_identity, + SphereAbility::Publish, + )) + .with_lifetime(120) + .with_fact(LINK_RECORD_FACT_NAME, local_sphere_tip.to_string()) + .build()? + .sign() + .await? + .encode()?); + + info!( + "Pushing new local history to gateway {}...", + client.session.gateway_identity + ); + + let result = client + .push(&PushBody { + sphere: local_sphere_identity.clone(), + local_base: local_sphere_base, + local_tip: local_sphere_tip.clone(), + counterpart_tip: Some(counterpart_sphere_tip.clone()), + name_record: Some(name_record), + }) + .await?; + + let counterpart_sphere_updated_tip = match result { + PushResponse::Accepted { new_tip } => new_tip, + PushResponse::NoChange => { + return Err(SyncError::Other(anyhow!("Gateway already up to date!"))); + } + }; + + info!("Saving updated counterpart sphere history..."); + + // TODO: Do this inside the client `push` method + //new_blocks.load_into(context.db_mut()).await?; + + debug!( + "Hydrating updated counterpart sphere history (from {} back to {})...", + counterpart_sphere_tip, counterpart_sphere_updated_tip + ); + + let timeline = Timeline::new(context.db_mut()); + Sphere::hydrate_timeslice( + &timeline + .slice( + &counterpart_sphere_updated_tip, + Some(counterpart_sphere_tip), + ) + .exclude_past(), + ) + .await?; + + context + .db_mut() + .set_version(counterpart_sphere_identity, &counterpart_sphere_updated_tip) + .await?; + + Ok(()) + } + + #[instrument(level = "debug", skip(self, context))] + async fn rollback( + &self, + context: &mut C, + original_sphere_version: Option<&Link>, + counterpart_identity: &Did, + original_counterpart_version: Option<&Link>, + ) -> Result<()> { + debug!("Rolling back!"); + let sphere_identity = context.identity().await?; + let mut context = context.sphere_context_mut().await?; + + if let Some(version) = original_sphere_version { + context + .db_mut() + .set_version(&sphere_identity, version) + .await?; + } + + if let Some(version) = original_counterpart_version { + context + .db_mut() + .set_version(counterpart_identity, version) + .await?; + } + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/context/sync/write.rs b/rust/noosphere-core/src/context/sync/write.rs new file mode 100644 index 000000000..1d25829c8 --- /dev/null +++ b/rust/noosphere-core/src/context/sync/write.rs @@ -0,0 +1,80 @@ +use crate::data::{Link, MemoIpld}; +use anyhow::Result; +use async_trait::async_trait; +use noosphere_storage::Storage; + +use crate::context::{HasMutableSphereContext, SyncError, SyncRecovery}; + +use crate::context::GatewaySyncStrategy; + +/// Implementors of [SphereSync] are able to sychronize with a Noosphere gateway +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait SphereSync +where + S: Storage + 'static, +{ + /// If a gateway URL has been configured, attempt to synchronize local + /// sphere data with the gateway. Changes on the gateway will first be + /// fetched to local storage. Then, the local changes will be replayed on + /// top of those changes. Finally, the synchronized local history will be + /// pushed up to the gateway. + /// + /// The returned [Link] is the latest version of the local + /// sphere lineage after the sync has completed. + async fn sync(&mut self, recovery: SyncRecovery) -> Result, SyncError>; +} + +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl SphereSync for C +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + #[instrument(level = "debug", skip(self))] + async fn sync(&mut self, recovery: SyncRecovery) -> Result, SyncError> { + debug!("Attempting to sync..."); + + let sync_strategy = GatewaySyncStrategy::default(); + + let version = match recovery { + SyncRecovery::None => sync_strategy.sync(self).await?, + SyncRecovery::Retry(max_retries) => { + let mut retries = 0; + let version; + + loop { + match sync_strategy.sync(self).await { + Ok(result) => { + debug!("Sync success with {retries} retries"); + version = result; + break; + } + Err(SyncError::Conflict) => { + if retries < max_retries { + warn!( + "Sync conflict; {} retries remaining...", + max_retries - retries + ); + retries += 1; + } else { + warn!("Sync conflict; no retries remaining!"); + return Err(SyncError::Conflict); + } + } + Err(other) => { + return Err(other); + } + } + } + + version + } + }; + + self.sphere_context_mut().await?.reset_access(); + + Ok(version) + } +} diff --git a/rust/noosphere-core/src/context/walker.rs b/rust/noosphere-core/src/context/walker.rs new file mode 100644 index 000000000..ef5e1a898 --- /dev/null +++ b/rust/noosphere-core/src/context/walker.rs @@ -0,0 +1,603 @@ +use crate::data::{Did, IdentityIpld, Jwt, Link, MapOperation, MemoIpld}; +use anyhow::Result; +use std::{collections::BTreeSet, marker::PhantomData}; + +use async_stream::try_stream; +use noosphere_storage::Storage; +use tokio::io::AsyncRead; +use tokio_stream::{Stream, StreamExt}; + +use crate::context::{ + content::{SphereContentRead, SphereFile}, + internal::SphereContextInternal, + HasSphereContext, SphereAuthorityRead, SpherePetnameRead, +}; + +/// A [SphereWalker] makes it possible to convert anything that implements +/// [HasSphereContext] into an async [Stream] over sphere content, allowing +/// incremental iteration over both the breadth of content at any version, or +/// the depth of changes over a range of history. +pub struct SphereWalker<'a, C, S> +where + C: HasSphereContext, + S: Storage + 'static, +{ + has_sphere_context: &'a C, + storage: PhantomData, +} + +impl<'a, C, S> From<&'a C> for SphereWalker<'a, C, S> +where + C: HasSphereContext, + S: Storage + 'static, +{ + fn from(has_sphere_context: &'a C) -> Self { + SphereWalker { + has_sphere_context, + storage: Default::default(), + } + } +} + +impl<'a, C, S> SphereWalker<'a, C, S> +where + C: SphereAuthorityRead + HasSphereContext, + S: Storage + 'static, +{ + /// Get a stream that yields a link to every authorization to access the + /// sphere along with its corresponding [DelegationIpld]. Note that since a + /// revocation may be issued without necessarily removing its revoked + /// delegation, this will yield all authorizations regardless of revocation + /// status. + pub fn authorization_stream( + &self, + ) -> impl Stream)>> + '_ { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let delegations = sphere.get_authority().await?.get_delegations().await?; + let stream = delegations.into_stream().await?; + + for await entry in stream { + let (link, delegation) = entry?; + let ucan = delegation.resolve_ucan(sphere.store()).await?; + yield (delegation.name, Did(ucan.audience().to_string()), link); + } + } + } + + /// Get a [BTreeSet] whose members are all the [Link]s to + /// authorizations that enable sphere access as of this version of the + /// sphere. Note that the full space of authoriztions may be very large; for + /// a more space-efficient approach, use + /// [SphereWalker::authorization_stream] to incrementally access all + /// authorizations in the sphere. + /// + /// This method is forgiving of missing or corrupted data, and will yield an + /// incomplete set of authorizations in the case that some or all names are + /// not able to be accessed. + pub async fn list_authorizations(&self) -> Result>> { + let sphere_identity = self.has_sphere_context.identity().await?; + let authorization_stream = self.authorization_stream(); + + tokio::pin!(authorization_stream); + + Ok(authorization_stream + .fold(BTreeSet::new(), |mut delegations, another_delegation| { + match another_delegation { + Ok((_, _, delegation)) => { + delegations.insert(delegation); + } + Err(error) => { + warn!( + "Could not read a petname from {}: {}", + sphere_identity, error + ) + } + }; + delegations + }) + .await) + } +} + +impl<'a, C, S> SphereWalker<'a, C, S> +where + C: SpherePetnameRead + HasSphereContext, + S: Storage + 'static, +{ + /// Same as [SphereWalker::petname_stream], but consumes the [SphereWalker]. + /// This is useful in cases where it would otherwise be necessary to borrow + /// a reference to [SphereWalker] for a static lifetime. + pub fn into_petname_stream(self) -> impl Stream> + 'a { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let petnames = sphere.get_address_book().await?.get_identities().await?; + let stream = petnames.into_stream().await?; + + for await entry in stream { + let (petname, address) = entry?; + yield (petname, address); + } + } + } + + /// Get a stream that yields every petname in the namespace along with its + /// corresponding [AddressIpld]. This is useful for iterating over sphere + /// petnames incrementally without having to load the entire index into + /// memory at once. + pub fn petname_stream(&self) -> impl Stream> + '_ { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let petnames = sphere.get_address_book().await?.get_identities().await?; + let stream = petnames.into_stream().await?; + + for await entry in stream { + let (petname, address) = entry?; + yield (petname, address); + } + } + } + + /// Get a stream that yields the set of petnames that changed at each + /// revision of the backing sphere, up to but excluding an optional `since` + /// CID parameter. To stream the entire history, pass `None` as the + /// parameter. + pub fn petname_change_stream<'b>( + &'b self, + since: Option<&'a Link>, + ) -> impl Stream, BTreeSet)>> + 'b { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let since = since.cloned(); + let stream = sphere.into_identities_changelog_stream(since.as_ref()); + + for await change in stream { + let (cid, changelog) = change?; + let mut changed_petnames = BTreeSet::new(); + + for operation in changelog.changes { + let petname = match operation { + MapOperation::Add { key, .. } => key, + MapOperation::Remove { key } => key, + }; + changed_petnames.insert(petname); + } + + yield (cid, changed_petnames); + } + } + } + + /// Get a stream that yields the set of petnames that changed at each + /// revision of the backing sphere, up to but excluding an optional `since` + /// CID parameter. To stream the entire history, pass `None` as the + /// parameter. + pub fn into_petname_change_stream( + self, + since: Option<&'a Link>, + ) -> impl Stream, BTreeSet)>> + '_ { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let since = since.cloned(); + let stream = sphere.into_identities_changelog_stream(since.as_ref()); + + for await change in stream { + let (cid, changelog) = change?; + let mut changed_petnames = BTreeSet::new(); + + for operation in changelog.changes { + let petname = match operation { + MapOperation::Add { key, .. } => key, + MapOperation::Remove { key } => key, + }; + changed_petnames.insert(petname); + } + + yield (cid, changed_petnames); + } + } + } + + /// Get a [BTreeSet] whose members are all the petnames that have addresses + /// as of this version of the sphere. Note that the full space of names may + /// be very large; for a more space-efficient approach, use + /// [SphereWalker::petname_stream] to incrementally access all petnames in + /// the sphere. + /// + /// This method is forgiving of missing or corrupted data, and will yield an + /// incomplete set of names in the case that some or all names are not able + /// to be accessed. + pub async fn list_petnames(&self) -> Result> { + let sphere_identity = self.has_sphere_context.identity().await?; + let petname_stream = self.petname_stream(); + + tokio::pin!(petname_stream); + + Ok(petname_stream + .fold(BTreeSet::new(), |mut petnames, another_petname| { + match another_petname { + Ok((petname, _)) => { + petnames.insert(petname); + } + Err(error) => { + warn!( + "Could not read a petname from {}: {}", + sphere_identity, error + ) + } + }; + petnames + }) + .await) + } + + /// Get a [BTreeSet] whose members are all the petnames whose values have + /// changed at least once since the provided version of the sphere + /// (exclusive of the provided version; use `None` to get all petnames + /// changed since the beginning of the sphere's history). + /// + /// This method is forgiving of missing or corrupted history, and will yield + /// an incomplete set of changes in the case that some or all changes are + /// not able to be accessed. + /// + /// Note that this operation will scale in memory consumption and duration + /// proportionally to the size of the sphere and the length of its history. + /// For a more efficient method of accessing changes, consider using + /// [SphereWalker::petname_change_stream] instead. + pub async fn petname_changes( + &self, + since: Option<&Link>, + ) -> Result> { + let sphere_identity = self.has_sphere_context.identity().await?; + let change_stream = self.petname_change_stream(since); + + tokio::pin!(change_stream); + + Ok(change_stream + .fold(BTreeSet::new(), |mut all, some| { + match some { + Ok((_, mut changes)) => all.append(&mut changes), + Err(error) => warn!( + "Could not read some changes from {}: {}", + sphere_identity, error + ), + }; + all + }) + .await) + } +} + +impl<'a, C, S> SphereWalker<'a, C, S> +where + C: SphereContentRead + HasSphereContext, + S: Storage + 'static, +{ + /// Same as [SphereWalker::content_stream], but consumes the [SphereWalker]. + /// This is useful in cases where it would otherwise be necessary to borrow + /// a reference to [SphereWalker] for a static lifetime. + pub fn into_content_stream( + self, + ) -> impl Stream)>> + 'a { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let content = sphere.get_content().await?; + let stream = content.into_stream().await?; + + for await entry in stream { + let (key, memo_link) = entry?; + let file = self.has_sphere_context.get_file(sphere.cid(), memo_link).await?; + + yield (key.clone(), file); + } + } + } + + /// Get a stream that yields every slug in the namespace along with its + /// corresponding [SphereFile]. This is useful for iterating over sphere + /// content incrementally without having to load the entire index into + /// memory at once. + pub fn content_stream( + &self, + ) -> impl Stream)>> + '_ { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let links = sphere.get_content().await?; + let stream = links.into_stream().await?; + + for await entry in stream { + let (key, memo) = entry?; + let file = self.has_sphere_context.get_file(sphere.cid(), memo).await?; + + yield (key.clone(), file); + } + } + } + + /// Get a stream that yields the set of slugs that changed at each revision + /// of the backing sphere, up to but excluding an optional CID. To stream + /// the entire history, pass `None` as the parameter. + pub fn into_content_change_stream( + self, + since: Option<&'a Link>, + ) -> impl Stream, BTreeSet)>> + '_ { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let since = since.cloned(); + let stream = sphere.into_content_changelog_stream(since.as_ref()); + + for await change in stream { + let (cid, changelog) = change?; + let mut changed_slugs = BTreeSet::new(); + + for operation in changelog.changes { + let slug = match operation { + MapOperation::Add { key, .. } => key, + MapOperation::Remove { key } => key, + }; + changed_slugs.insert(slug); + } + + yield (cid, changed_slugs); + } + } + } + + /// Get a stream that yields the set of slugs that changed at each revision + /// of the backing sphere, up to but excluding an optional CID. To stream + /// the entire history, pass `None` as the parameter. + pub fn content_change_stream<'b>( + &'b self, + since: Option<&'b Link>, + ) -> impl Stream, BTreeSet)>> + 'b { + try_stream! { + let sphere = self.has_sphere_context.to_sphere().await?; + let since = since.cloned(); + let stream = sphere.into_content_changelog_stream(since.as_ref()); + + for await change in stream { + let (cid, changelog) = change?; + let mut changed_slugs = BTreeSet::new(); + + for operation in changelog.changes { + let slug = match operation { + MapOperation::Add { key, .. } => key, + MapOperation::Remove { key } => key, + }; + changed_slugs.insert(slug); + } + + yield (cid, changed_slugs); + } + } + } + + /// Get a [BTreeSet] whose members are all the slugs that have values as of + /// this version of the sphere. Note that the full space of slugs may be + /// very large; for a more space-efficient approach, use + /// [SphereWalker::content_stream] or [SphereWalker::into_content_stream] to + /// incrementally access all slugs in the sphere. + /// + /// This method is forgiving of missing or corrupted data, and will yield an + /// incomplete set of links in the case that some or all links are not able + /// to be accessed. + pub async fn list_slugs(&self) -> Result> { + let sphere_identity = self.has_sphere_context.identity().await?; + let link_stream = self.content_stream(); + + tokio::pin!(link_stream); + + Ok(link_stream + .fold(BTreeSet::new(), |mut links, another_link| { + match another_link { + Ok((slug, _)) => { + links.insert(slug); + } + Err(error) => { + warn!("Could not read a link from {}: {}", sphere_identity, error) + } + }; + links + }) + .await) + } + + /// Get a [BTreeSet] whose members are all the slugs whose values have + /// changed at least once since the provided version of the sphere + /// (exclusive of the provided version; use `None` to get all slugs changed + /// since the beginning of the sphere's history). + /// + /// This method is forgiving of missing or corrupted history, and will yield + /// an incomplete set of changes in the case that some or all changes are + /// not able to be accessed. + /// + /// Note that this operation will scale in memory consumption and duration + /// proportionally to the size of the sphere and the length of its history. + /// For a more efficient method of accessing changes, consider using + /// [SphereWalker::content_change_stream] instead. + pub async fn content_changes( + &self, + since: Option<&Link>, + ) -> Result> { + let sphere_identity = self.has_sphere_context.identity().await?; + let change_stream = self.content_change_stream(since); + + tokio::pin!(change_stream); + + Ok(change_stream + .fold(BTreeSet::new(), |mut all, some| { + match some { + Ok((_, mut changes)) => all.append(&mut changes), + Err(error) => warn!( + "Could not read some changes from {}: {}", + sphere_identity, error + ), + }; + all + }) + .await) + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use std::collections::BTreeSet; + + use tokio::io::AsyncReadExt; + use tokio_stream::StreamExt; + + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + use crate::{ + context::{ + HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SphereCursor, + SphereWalker, + }, + data::{ContentType, Did}, + helpers::{simulated_sphere_context, SimulationAccess}, + }; + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_be_initialized_with_a_context_or_a_cursor() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context.clone()); + + let changes = vec![ + vec!["dogs", "birds"], + vec!["cats", "dogs"], + vec!["birds"], + vec!["cows", "beetles"], + ]; + + for change in changes { + for slug in change { + cursor + .write(slug, &ContentType::Subtext, b"are cool".as_ref(), None) + .await + .unwrap(); + } + + cursor.save(None).await.unwrap(); + } + + let walker_cursor = SphereWalker::from(&cursor); + let walker_context = SphereWalker::from(&sphere_context); + + let slugs_cursor = walker_cursor.list_slugs().await.unwrap(); + let slugs_context = walker_context.list_slugs().await.unwrap(); + + assert_eq!(slugs_cursor.len(), 5); + assert_eq!(slugs_cursor, slugs_context); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_list_all_slugs_currently_in_a_sphere() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + let changes = vec![ + vec!["dogs", "birds"], + vec!["cats", "dogs"], + vec!["birds"], + vec!["cows", "beetles"], + ]; + + for change in changes { + for slug in change { + cursor + .write(slug, &ContentType::Subtext, b"are cool".as_ref(), None) + .await + .unwrap(); + } + + cursor.save(None).await.unwrap(); + } + + let walker_cursor = cursor.clone(); + let walker = SphereWalker::from(&walker_cursor); + let slugs = walker.list_slugs().await.unwrap(); + + assert_eq!(slugs.len(), 5); + + cursor.remove("dogs").await.unwrap(); + cursor.save(None).await.unwrap(); + + let slugs = walker.list_slugs().await.unwrap(); + + assert_eq!(slugs.len(), 4); + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_list_all_authorizations_currently_in_a_sphere() -> Result<()> { + let (sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let mut cursor = SphereCursor::latest(sphere_context); + let authorizations_to_add = 10; + + for i in 0..authorizations_to_add { + cursor + .authorize(&format!("foo{}", i), &Did(format!("did:key:foo{}", i))) + .await?; + } + + cursor.save(None).await?; + + let authorizations = SphereWalker::from(&cursor).list_authorizations().await?; + + assert_eq!(authorizations.len(), authorizations_to_add + 1); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_stream_the_whole_index() { + let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + let mut cursor = SphereCursor::latest(sphere_context); + + let expected = BTreeSet::<(String, String)>::from([ + ("cats".into(), "Cats are awesome".into()), + ("dogs".into(), "Dogs are pretty cool".into()), + ("birds".into(), "Birds rights".into()), + ("mice".into(), "Mice like cookies".into()), + ]); + + for (slug, content) in &expected { + cursor + .write(slug.as_str(), &ContentType::Subtext, content.as_ref(), None) + .await + .unwrap(); + + cursor.save(None).await.unwrap(); + } + + let mut actual = BTreeSet::new(); + let walker = SphereWalker::from(&cursor); + let stream = walker.content_stream(); + + tokio::pin!(stream); + + while let Some(Ok((slug, mut file))) = stream.next().await { + let mut contents = String::new(); + file.contents.read_to_string(&mut contents).await.unwrap(); + actual.insert((slug, contents)); + } + + assert_eq!(expected, actual); + } +} diff --git a/rust/noosphere-core/src/data/address.rs b/rust/noosphere-core/src/data/address.rs index ab1c7fadd..87edf8374 100644 --- a/rust/noosphere-core/src/data/address.rs +++ b/rust/noosphere-core/src/data/address.rs @@ -15,12 +15,15 @@ use super::{Did, IdentitiesIpld, Jwt, Link, MemoIpld}; #[cfg(docs)] use crate::data::SphereIpld; +/// The name of the fact (as defined for a [Ucan]) that contains the link for a +/// [LinkRecord] pub const LINK_RECORD_FACT_NAME: &str = "link"; /// A subdomain of a [SphereIpld] that pertains to the management and recording of /// the petnames associated with the sphere. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Hash)] pub struct AddressBookIpld { + /// A pointer to the [IdentitiesIpld] associated with this address book pub identities: Link, } @@ -42,7 +45,9 @@ impl AddressBookIpld { /// value if one has ever been resolved. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Hash)] pub struct IdentityIpld { + /// The [Did] of a peer pub did: Did, + /// An optional pointer to a known [LinkRecord] for the peer pub link_record: Option>, } diff --git a/rust/noosphere-core/src/data/authority.rs b/rust/noosphere-core/src/data/authority.rs index 78274acb4..1a911f292 100644 --- a/rust/noosphere-core/src/data/authority.rs +++ b/rust/noosphere-core/src/data/authority.rs @@ -16,7 +16,12 @@ use crate::data::SphereIpld; /// access a sphere, as well as the revocations of that authority. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct AuthorityIpld { + /// A pointer to the [DelegationsIpld] for this [AuthorityIpld], embodying + /// all authorizations for keys that operate on this sphere pub delegations: Link, + /// A pointer to the [RevocationsIpld] for this [AuthorityIpld], embodying + /// revocations for any otherwise valid authorizations for keys issued by this + /// sphere in the past. pub revocations: Link, } @@ -43,17 +48,24 @@ impl AuthorityIpld { } } +#[cfg(doc)] +use crate::data::Jwt; + /// This delegation represents the sharing of access to resources within a /// sphere. The name of the delegation is for display purposes only, and helps /// the user identify the client device or application that the delegation is /// intended for. #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)] pub struct DelegationIpld { + /// The human-readable name of the delegation pub name: String, + /// A pointer to the [Jwt] created for this [DelegationIpld] pub jwt: Cid, } impl DelegationIpld { + /// Stores a [Ucan] that delegates authority to a key, and initializes a + /// [DelegationIpld] for it that is appropriate for storing in a sphere pub async fn register(name: &str, jwt: &str, store: &S) -> Result { let mut store = UcanStore(store.clone()); let cid = store.write_token(jwt).await?; @@ -64,6 +76,8 @@ impl DelegationIpld { }) } + /// Resolve a [Ucan] from storage via the pointer to a [Jwt] in this + /// [DelegationIpld] pub async fn resolve_ucan(&self, store: &S) -> Result { let store = UcanStore(store.clone()); let jwt = store.require_token(&self.jwt).await?; @@ -87,6 +101,8 @@ pub struct RevocationIpld { } impl RevocationIpld { + /// Revoke a delegation by the [Cid] of its associated [Jwt], using a key credential + /// of an authorizing ancestor of the original delegation pub async fn revoke(cid: &Cid, issuer: &K) -> Result { Ok(RevocationIpld { iss: issuer.get_did().await?, @@ -95,6 +111,8 @@ impl RevocationIpld { }) } + /// Verify that the [RevocationIpld] is valid compared to the public key of + /// the issuer pub async fn verify(&self, claimed_issuer: &K) -> Result<()> { let cid = Cid::try_from(self.revoke.as_str())?; let challenge_payload = Self::make_challenge_payload(&cid); diff --git a/rust/noosphere-core/src/data/body_chunk.rs b/rust/noosphere-core/src/data/body_chunk.rs index 66d935164..74eb98352 100644 --- a/rust/noosphere-core/src/data/body_chunk.rs +++ b/rust/noosphere-core/src/data/body_chunk.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use noosphere_storage::BlockStore; +/// The maximum size of a body chunk as produced by [BodyChunkIpld] pub const BODY_CHUNK_MAX_SIZE: u32 = 1024 * 1024; // ~1mb/chunk worst case, ~.5mb/chunk average case /// A body chunk is a simplified flexible byte layout used for linking @@ -21,6 +22,8 @@ pub struct BodyChunkIpld { } impl BodyChunkIpld { + /// Chunk and encode a slice of bytes as linked [BodyChunkIpld], storing the chunks in storage + /// and returning the [Cid] of the head of the list. // TODO(#498): Re-write to address potentially unbounded memory overhead pub async fn store_bytes(bytes: &[u8], store: &mut S) -> Result { let chunks = FastCDC::new( @@ -56,6 +59,7 @@ impl BodyChunkIpld { next_chunk_cid.ok_or_else(|| anyhow!("No CID; did you try to store zero bytes?")) } + /// Fold all bytes in the [BodyChunkIpld] chain into a single buffer and return it // TODO(#498): Re-write to address potentially unbounded memory overhead pub async fn load_all_bytes(&self, store: &S) -> Result> { let mut all_bytes = self.bytes.clone(); diff --git a/rust/noosphere-core/src/data/bundle.rs b/rust/noosphere-core/src/data/bundle.rs index fc36fe272..7d35963d4 100644 --- a/rust/noosphere-core/src/data/bundle.rs +++ b/rust/noosphere-core/src/data/bundle.rs @@ -1,3 +1,6 @@ +// We are removing this module, so not gonna bother documenting... +#![allow(missing_docs)] + use std::{collections::BTreeMap, str::FromStr}; use anyhow::{anyhow, Result}; diff --git a/rust/noosphere-core/src/data/changelog.rs b/rust/noosphere-core/src/data/changelog.rs index ed8d279ca..491e5f4e3 100644 --- a/rust/noosphere-core/src/data/changelog.rs +++ b/rust/noosphere-core/src/data/changelog.rs @@ -2,22 +2,32 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; use std::default::Default; +#[cfg(doc)] +use crate::data::{Did, VersionedMapIpld}; + +/// A [ChangelogIpld] records a series of changes that represent the delta of a +/// given [VersionedMapIpld] from its immediate ancestor #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct ChangelogIpld { + /// The [Did] of the author of the change pub did: Option, + /// The changes that were made to the associated [VersionedMapIpld] pub changes: Vec, } impl ChangelogIpld { + /// Returns true if the [ChangelogIpld] represents zero changes pub fn is_empty(&self) -> bool { self.changes.len() == 0 } + /// Adds a single change to the [ChangelogIpld] pub fn push(&mut self, op: Op) -> Result<()> { self.changes.push(op); Ok(()) } + /// Initializes a [ChangelogIpld] for the author with the given [Did] pub fn mark(&self, did: &str) -> Self { ChangelogIpld { did: Some(did.to_string()), diff --git a/rust/noosphere-core/src/data/headers/content_type.rs b/rust/noosphere-core/src/data/headers/content_type.rs index 8f98bef27..d57c97acc 100644 --- a/rust/noosphere-core/src/data/headers/content_type.rs +++ b/rust/noosphere-core/src/data/headers/content_type.rs @@ -1,13 +1,21 @@ use std::{convert::Infallible, fmt::Display, ops::Deref, str::FromStr}; +/// Various well-known mimes in Noosphere #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Debug)] pub enum ContentType { + /// Plain text Text, + /// Subtext Subtext, + /// A sphere Sphere, + /// Raw bytes Bytes, + /// CBOR bytes Cbor, + /// JSON Json, + /// All others Unknown(String), } diff --git a/rust/noosphere-core/src/data/headers/header.rs b/rust/noosphere-core/src/data/headers/header.rs index 9a9ce3547..aa00f1121 100644 --- a/rust/noosphere-core/src/data/headers/header.rs +++ b/rust/noosphere-core/src/data/headers/header.rs @@ -1,14 +1,25 @@ use std::{convert::Infallible, fmt::Display, ops::Deref, str::FromStr}; +/// Well-known headers in the Noosphere pub enum Header { + /// Content-type, for mimes ContentType, + /// A proof, typically a UCAN JWT Proof, + /// The author's DID Author, + /// A title for the associated content body Title, + /// A signature by the author's key Signature, + /// The Noosphere protocol version Version, + /// A file extension to use when rendering the content to + /// the file system FileExtension, + /// The logical order relative to any ancestors LamportOrder, + /// All others Unknown(String), } diff --git a/rust/noosphere-core/src/data/headers/version.rs b/rust/noosphere-core/src/data/headers/version.rs index 2113b09b2..2c22af954 100644 --- a/rust/noosphere-core/src/data/headers/version.rs +++ b/rust/noosphere-core/src/data/headers/version.rs @@ -1,8 +1,11 @@ use anyhow::anyhow; use std::{convert::Infallible, fmt::Display, str::FromStr}; +/// The Noosphere protocol version pub enum Version { + #[allow(missing_docs)] V0, + /// All others Unknown(String), } diff --git a/rust/noosphere-core/src/data/link.rs b/rust/noosphere-core/src/data/link.rs index bb3f38345..94c2c13c3 100644 --- a/rust/noosphere-core/src/data/link.rs +++ b/rust/noosphere-core/src/data/link.rs @@ -4,6 +4,7 @@ use libipld_core::{ codec::{Codec, Decode, Encode}, raw::RawCodec, }; +use noosphere_common::ConditionalSend; use std::fmt::Debug; use std::{ fmt::{Display, Formatter}, @@ -18,18 +19,6 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use noosphere_collections::hamt::Hash as HamtHash; -#[cfg(not(target_arch = "wasm32"))] -pub trait LinkSend: Send {} - -#[cfg(not(target_arch = "wasm32"))] -impl LinkSend for T where T: Send {} - -#[cfg(target_arch = "wasm32")] -pub trait LinkSend {} - -#[cfg(target_arch = "wasm32")] -impl LinkSend for T {} - /// A [Link] is a [Cid] with a type attached. The type represents the data that /// the [Cid] refers to. This is a helpful construct to use to ensure that data /// structures whose fields or elements may be [Cid]s can still retain strong @@ -45,6 +34,7 @@ pub struct Link where T: Clone, { + /// The wrapped [Cid] of this [Link] pub cid: Cid, linked_type: PhantomData, } @@ -105,6 +95,7 @@ impl Link where T: Clone, { + /// Wrap a given [Cid] in a typed [Link] pub fn new(cid: Cid) -> Self { Link { cid, @@ -200,7 +191,7 @@ where impl Link where - T: Serialize + DeserializeOwned + Clone + LinkSend, + T: Serialize + DeserializeOwned + Clone + ConditionalSend, { /// Given a [BlockStore], attempt to load a value for the [Cid] of this /// [Link]. The loaded block will be interpretted as the type that is diff --git a/rust/noosphere-core/src/data/memo.rs b/rust/noosphere-core/src/data/memo.rs index b1cf82e86..975021799 100644 --- a/rust/noosphere-core/src/data/memo.rs +++ b/rust/noosphere-core/src/data/memo.rs @@ -6,15 +6,17 @@ use std::{ use anyhow::{anyhow, Result}; use cid::Cid; use libipld_cbor::DagCborCodec; +use noosphere_common::ConditionalSend; use serde::{Deserialize, Serialize}; use ucan::{crypto::KeyMaterial, Ucan}; use crate::data::Header; -use noosphere_storage::{base64_encode, BlockStore, BlockStoreSend}; +use noosphere_storage::{base64_encode, BlockStore}; use super::{ContentType, Link}; +/// A designated int for lamport order values in [MemoIpld] headers pub type LamportOrder = u32; /// A basic Memo. A Memo is a history-retaining structure that pairs @@ -99,7 +101,7 @@ impl MemoIpld { /// Initializes a memo for the provided body, persisting the body to storage /// and returning the memo. Note that only the body is persisted, not the /// memo that wraps it. - pub async fn for_body( + pub async fn for_body( store: &mut S, body: Body, ) -> Result { diff --git a/rust/noosphere-core/src/data/mod.rs b/rust/noosphere-core/src/data/mod.rs index 552de5bd6..3631f7f4b 100644 --- a/rust/noosphere-core/src/data/mod.rs +++ b/rust/noosphere-core/src/data/mod.rs @@ -1,3 +1,7 @@ +//! Core data types in use by the Noosphere protocol. Data types in here +//! represent the canonical structure of Noosphere data as expressed by +//! block-encoded IPLD. + mod address; mod authority; mod body_chunk; diff --git a/rust/noosphere-core/src/data/sphere.rs b/rust/noosphere-core/src/data/sphere.rs index 25ec203c9..df89c0f6d 100644 --- a/rust/noosphere-core/src/data/sphere.rs +++ b/rust/noosphere-core/src/data/sphere.rs @@ -29,6 +29,7 @@ pub struct SphereIpld { } impl SphereIpld { + /// Initialize a new, empty [SphereIpld] with a given [Did] sphere identity. pub async fn new(identity: &Did, store: &mut S) -> Result where S: BlockStore, @@ -58,31 +59,23 @@ impl SphereIpld { #[cfg(test)] mod tests { use anyhow::Result; - use ed25519_zebra::{SigningKey as Ed25519PrivateKey, VerificationKey as Ed25519PublicKey}; use libipld_cbor::DagCborCodec; use ucan::{builder::UcanBuilder, crypto::KeyMaterial, store::UcanJwtStore}; - use ucan_key_support::ed25519::Ed25519KeyMaterial; #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::wasm_bindgen_test; use crate::{ - authority::{generate_capability, SphereAbility}, + authority::{generate_capability, generate_ed25519_key, SphereAbility}, data::{ContentType, Did, Header, MemoIpld, SphereIpld}, view::Sphere, }; use noosphere_storage::{BlockStore, MemoryStorage, SphereDb}; - fn generate_credential() -> Ed25519KeyMaterial { - let private_key = Ed25519PrivateKey::new(rand::thread_rng()); - let public_key = Ed25519PublicKey::from(&private_key); - Ed25519KeyMaterial(public_key, Some(private_key)) - } - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_be_signed_by_identity_key_and_verified() -> Result<()> { - let identity_credential = generate_credential(); + let identity_credential = generate_ed25519_key(); let identity = Did(identity_credential.get_did().await?); let mut store = SphereDb::new(&MemoryStorage::default()).await?; @@ -128,8 +121,8 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_be_signed_by_an_authorized_key_and_verified() -> Result<()> { - let identity_credential = generate_credential(); - let authorized_credential = generate_credential(); + let identity_credential = generate_ed25519_key(); + let authorized_credential = generate_ed25519_key(); let identity = Did(identity_credential.get_did().await?); let authorized = Did(authorized_credential.get_did().await?); diff --git a/rust/noosphere-core/src/data/versioned_map.rs b/rust/noosphere-core/src/data/versioned_map.rs index 9e1a5ced5..fccf5b873 100644 --- a/rust/noosphere-core/src/data/versioned_map.rs +++ b/rust/noosphere-core/src/data/versioned_map.rs @@ -7,66 +7,79 @@ use std::{fmt::Display, hash::Hash, marker::PhantomData}; use noosphere_collections::hamt::{Hamt, Hash as HamtHash, Sha256}; use noosphere_storage::BlockStore; +use noosphere_common::ConditionalSync; + use super::{ChangelogIpld, DelegationIpld, IdentityIpld, Jwt, Link, MemoIpld, RevocationIpld}; -pub type IdentitiesIpld = VersionedMapIpld; +/// A [VersionedMapIpld] that represents the content space of a sphere pub type ContentIpld = VersionedMapIpld>; +/// A [VersionedMapIpld] that represents the petname space of a sphere +pub type IdentitiesIpld = VersionedMapIpld; +/// A [VersionedMapIpld] that represents the key authorizations in a sphere pub type DelegationsIpld = VersionedMapIpld, DelegationIpld>; +/// A [VersionedMapIpld] that represents the authority revocations in a sphere pub type RevocationsIpld = VersionedMapIpld, RevocationIpld>; -#[cfg(not(target_arch = "wasm32"))] -pub trait VersionedMapSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl VersionedMapSendSync for T where T: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait VersionedMapSendSync {} - -#[cfg(target_arch = "wasm32")] -impl VersionedMapSendSync for T {} - +/// A helper trait to simplify expressing the bounds of a valid [VersionedMapIpld] key pub trait VersionedMapKey: - Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + VersionedMapSendSync + Display + Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + ConditionalSync + Display { } impl VersionedMapKey for T where - T: Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + VersionedMapSendSync + Display + T: Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + ConditionalSync + Display { } +/// A helper trait to simplify expressing the bounds of a valid [VersionedMapIpld] value pub trait VersionedMapValue: - Serialize + DeserializeOwned + Clone + Eq + Hash + VersionedMapSendSync + Serialize + DeserializeOwned + Clone + Eq + Hash + ConditionalSync { } impl VersionedMapValue for T where - T: Serialize + DeserializeOwned + Clone + Eq + Hash + VersionedMapSendSync + T: Serialize + DeserializeOwned + Clone + Eq + Hash + ConditionalSync { } +/// A [MapOperation] represents a single change to a [VersionedMapIpld] as it may +/// be recorded in a [ChangelogIpld]. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub enum MapOperation { - Add { key: Key, value: Value }, - Remove { key: Key }, + /// A [MapOperation] that represents an update or insert to a [VersionedMapIpld] + Add { + /// The key that was updated or inserted + key: Key, + /// The new value associated with the key + value: Value, + }, + /// A [MapOperation] that represents a removal of a key from a [VersionedMapIpld] + Remove { + /// The key that was removed + key: Key, + }, } +/// A [VersionedMapIpld] pairs a [Hamt] and a [ChangelogIpld] to enable a data structure +/// that contains its difference from a historical ancestor without requiring that a diff +/// be performed. #[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct VersionedMapIpld where Key: VersionedMapKey, Value: VersionedMapValue, { - /// A pointer to a HAMT + /// A pointer to a [Hamt] root pub hamt: Cid, // TODO(#262): The size of this vec is implicitly limited by the IPLD block // size limit. This is probably fine most of the time; the vec only holds // the delta changes, and N<10 probably holds in the majority of cases. But, // it will be necessary to gracefully survive the outlier cases where // N>~1000. + /// A pointer to a [ChangelogIpld] pub changelog: Cid, + #[allow(missing_docs)] #[serde(skip)] pub signature: PhantomData<(Key, Value)>, } @@ -76,10 +89,12 @@ where Key: VersionedMapKey, Value: VersionedMapValue, { + /// Load the [Hamt] root associated with this [VersionedMapIpld] pub async fn load_hamt(&self, store: &S) -> Result> { Hamt::load(&self.hamt, store.clone()).await } + /// Load the [ChangelogIpld] associated with this [VersionedMapIpld] pub async fn load_changelog( &self, store: &S, @@ -87,6 +102,8 @@ where store.load::(&self.changelog).await } + /// Initialize an empty [VersionedMapIpld], creating an empty [Hamt] root + /// and [ChangelogIpld] as well // NOTE: We currently don't have a mechanism to prepuplate the store with // "empty" DAGs like a HAMT. So, we do it lazily by requiring async // initialization of this struct even when it is empty. diff --git a/rust/noosphere-core/src/error.rs b/rust/noosphere-core/src/error.rs index efc0f86ab..e75799bd1 100644 --- a/rust/noosphere-core/src/error.rs +++ b/rust/noosphere-core/src/error.rs @@ -1,20 +1,28 @@ +//! Noosphere errors + use crate::authority::Authorization; use thiserror::Error; +/// High-level error types relevant to the Noosphere protocol #[derive(Error, Debug)] pub enum NoosphereError { + /// Any error not covered by the other errors #[error("{0}")] Other(anyhow::Error), + #[allow(missing_docs)] #[error("Network access required but network is currently offline")] NetworkOffline, + #[allow(missing_docs)] #[error("No credentials configured")] NoCredentials, + #[allow(missing_docs)] #[error("Missing configuration: {0}")] MissingConfiguration(&'static str), + #[allow(missing_docs)] #[error("The provided authorization {0} is invalid: {1}")] InvalidAuthorization(Authorization, String), } diff --git a/rust/noosphere-core/src/helpers/context.rs b/rust/noosphere-core/src/helpers/context.rs new file mode 100644 index 000000000..05e06dbf2 --- /dev/null +++ b/rust/noosphere-core/src/helpers/context.rs @@ -0,0 +1,229 @@ +//! These helpers are intended for use in documentation examples and tests only. +//! They are useful for quickly scaffolding common scenarios that would +//! otherwise be verbosely rubber-stamped in a bunch of places. +use std::sync::Arc; + +use crate::{ + authority::{generate_capability, generate_ed25519_key, Author, SphereAbility}, + data::{ContentType, Did, LinkRecord, Mnemonic, LINK_RECORD_FACT_NAME}, + view::Sphere, +}; +use anyhow::Result; +use noosphere_storage::{BlockStore, MemoryStorage, SphereDb, TrackingStorage, UcanStore}; +use tokio::{io::AsyncReadExt, sync::Mutex}; +use ucan::{builder::UcanBuilder, crypto::KeyMaterial}; + +use crate::{ + context::{ + HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, + SphereContext, SphereContextKey, SpherePetnameWrite, + }, + stream::{walk_versioned_map_elements, walk_versioned_map_elements_and}, +}; + +/// Access levels available when simulating a [SphereContext] +pub enum SimulationAccess { + /// Access to the related [SphereContext] is read-only + Readonly, + /// Access to the related [SphereContext] is read+write + ReadWrite, +} + +/// Create a temporary, non-persisted [SphereContext] that tracks usage +/// internally. This is intended for use in docs and tests, and should otherwise +/// be ignored. When creating the simulated [SphereContext], you can pass a +/// [SimulationAccess] to control the kind of access the emphemeral credentials +/// have to the [SphereContext]. +pub async fn simulated_sphere_context( + profile: SimulationAccess, + db: Option>>, +) -> Result<( + Arc>>>, + Mnemonic, +)> { + let mut db = match db { + Some(db) => db, + None => { + let storage_provider = TrackingStorage::wrap(MemoryStorage::default()); + SphereDb::new(&storage_provider).await? + } + }; + + let owner_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key())); + let owner_did = owner_key.get_did().await?; + + let (sphere, proof, mnemonic) = Sphere::generate(&owner_did, &mut db).await?; + + let sphere_identity = sphere.get_identity().await?; + let author = Author { + key: owner_key, + authorization: match profile { + SimulationAccess::Readonly => None, + SimulationAccess::ReadWrite => Some(proof), + }, + }; + + db.set_version(&sphere_identity, sphere.cid()).await?; + + Ok(( + Arc::new(Mutex::new( + SphereContext::new(sphere_identity, author, db, None).await?, + )), + mnemonic, + )) +} + +#[cfg(docs)] +use crate::data::MemoIpld; + +/// Attempt to walk an entire sphere, touching every block up to and including +/// any [MemoIpld] nodes, but excluding those memo's body content. This helper +/// is useful for asserting that the blocks expected to be sent during +/// replication have in fact been sent. +pub async fn touch_all_sphere_blocks(sphere: &Sphere) -> Result<()> +where + S: BlockStore + 'static, +{ + trace!("Touching content blocks..."); + let content = sphere.get_content().await?; + let _ = content.load_changelog().await?; + + walk_versioned_map_elements(content).await?; + + trace!("Touching identity blocks..."); + let identities = sphere.get_address_book().await?.get_identities().await?; + let _ = identities.load_changelog().await?; + + walk_versioned_map_elements_and( + identities, + sphere.store().clone(), + |_, identity, store| async move { + let ucan_store = UcanStore(store); + if let Some(record) = identity.link_record(&ucan_store).await { + record.collect_proofs(&ucan_store).await?; + } + Ok(()) + }, + ) + .await?; + + trace!("Touching authority blocks..."); + let authority = sphere.get_authority().await?; + + trace!("Touching delegation blocks..."); + let delegations = authority.get_delegations().await?; + walk_versioned_map_elements(delegations).await?; + + trace!("Touching revocation blocks..."); + let revocations = authority.get_revocations().await?; + walk_versioned_map_elements(revocations).await?; + + Ok(()) +} + +/// A type of [HasMutableSphereContext] that uses [TrackingStorage] internally +pub type TrackedHasMutableSphereContext = Arc>>>; + +/// Create a series of spheres where each sphere has the next as resolved +/// entry in its address book; return a [HasMutableSphereContext] for the +/// first sphere in the sequence. +pub async fn make_sphere_context_with_peer_chain( + peer_chain: &[String], +) -> Result<(TrackedHasMutableSphereContext, Vec)> { + let (origin_sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None) + .await + .unwrap(); + + let mut db = origin_sphere_context + .sphere_context() + .await + .unwrap() + .db() + .clone(); + + let mut contexts = vec![origin_sphere_context.clone()]; + + for name in peer_chain.iter() { + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, Some(db.clone())) + .await + .unwrap(); + + sphere_context + .write("my-name", &ContentType::Subtext, name.as_bytes(), None) + .await + .unwrap(); + sphere_context.save(None).await.unwrap(); + + contexts.push(sphere_context); + } + + let mut next_sphere_context: Option = None; + let mut dids = Vec::new(); + + for mut sphere_context in contexts.into_iter().rev() { + dids.push(sphere_context.identity().await?); + if let Some(next_sphere_context) = next_sphere_context { + let version = next_sphere_context.version().await.unwrap(); + + let next_author = next_sphere_context + .sphere_context() + .await + .unwrap() + .author() + .clone(); + let next_identity = next_sphere_context.identity().await.unwrap(); + + let link_record = LinkRecord::from( + UcanBuilder::default() + .issued_by(&next_author.key) + .for_audience(&next_identity) + .witnessed_by( + &next_author + .authorization + .as_ref() + .unwrap() + .as_ucan(&db) + .await + .unwrap(), + None, + ) + .claiming_capability(&generate_capability( + &next_identity, + SphereAbility::Publish, + )) + .with_lifetime(120) + .with_fact(LINK_RECORD_FACT_NAME, version.to_string()) + .build() + .unwrap() + .sign() + .await + .unwrap(), + ); + + let mut name = String::new(); + let mut file = next_sphere_context.read("my-name").await.unwrap().unwrap(); + file.contents.read_to_string(&mut name).await.unwrap(); + + debug!("Adopting {name}"); + sphere_context + .set_petname(&name, Some(next_identity)) + .await?; + sphere_context.save(None).await?; + + sphere_context + .set_petname_record(&name, &link_record) + .await + .unwrap(); + let identity = sphere_context.identity().await?; + + db.set_version(&identity, &sphere_context.save(None).await.unwrap()) + .await + .unwrap(); + } + + next_sphere_context = Some(sphere_context); + } + + Ok((origin_sphere_context, dids)) +} diff --git a/rust/noosphere-core/src/helpers.rs b/rust/noosphere-core/src/helpers/link.rs similarity index 100% rename from rust/noosphere-core/src/helpers.rs rename to rust/noosphere-core/src/helpers/link.rs diff --git a/rust/noosphere-core/src/helpers/mod.rs b/rust/noosphere-core/src/helpers/mod.rs new file mode 100644 index 000000000..e6b0cbc2c --- /dev/null +++ b/rust/noosphere-core/src/helpers/mod.rs @@ -0,0 +1,8 @@ +//! Generic helper utilities intended to be used exclusively in tests and hidden +//! setup for examples + +mod context; +mod link; + +pub use context::*; +pub use link::*; diff --git a/rust/noosphere-core/src/lib.rs b/rust/noosphere-core/src/lib.rs index 414745718..42a04f932 100644 --- a/rust/noosphere-core/src/lib.rs +++ b/rust/noosphere-core/src/lib.rs @@ -1,8 +1,21 @@ +#![warn(missing_docs)] + +//! This crate embodies the core implementation of the Noosphere protocol. +//! +//! It includes facilities to: +//! - Get low-level access to Noosphere data structures ([view] and [data]) +//! - Interact with a Noosphere Gateway ([api::Client]) +//! - Read, update, and sync spheres via a high-level API ([context]) +//! - And more! + #[macro_use] extern crate tracing as extern_tracing; +pub mod api; pub mod authority; +pub mod context; pub mod data; +pub mod stream; pub mod view; pub mod error; diff --git a/rust/noosphere-core/src/stream/block.rs b/rust/noosphere-core/src/stream/block.rs new file mode 100644 index 000000000..2654af752 --- /dev/null +++ b/rust/noosphere-core/src/stream/block.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use libipld_core::raw::RawCodec; +use noosphere_storage::BlockStore; +use tokio_stream::{Stream, StreamExt}; + +/// Helper to put blocks from a [Stream] into any implementor of [BlockStore] +/// +/// Implementation note: this is a stand-alone helper because defining this +/// async function on a trait (necessitating use of `#[async_trait]`) creates an +/// unergonomic `Send` bound on returned [Future]. +pub async fn put_block_stream(mut store: S, stream: Str) -> Result<()> +where + S: BlockStore, + Str: Stream)>>, +{ + tokio::pin!(stream); + + let mut stream_count = 0usize; + + while let Some((cid, block)) = stream.try_next().await? { + stream_count += 1; + trace!(?cid, "Putting streamed block {stream_count}..."); + + store.put_block(&cid, &block).await?; + + match cid.codec() { + codec_id if codec_id == u64::from(DagCborCodec) => { + store.put_links::(&cid, &block).await?; + } + codec_id if codec_id == u64::from(RawCodec) => { + store.put_links::(&cid, &block).await?; + } + codec_id => warn!("Unrecognized codec {}; skipping...", codec_id), + } + } + + trace!("Successfully put {stream_count} blocks from stream..."); + + Ok(()) +} diff --git a/rust/noosphere-core/src/stream/car.rs b/rust/noosphere-core/src/stream/car.rs new file mode 100644 index 000000000..32926ae3f --- /dev/null +++ b/rust/noosphere-core/src/stream/car.rs @@ -0,0 +1,160 @@ +use anyhow::Result; +use async_stream::try_stream; +use bytes::Bytes; +use cid::Cid; +use futures_util::{sink::SinkExt, TryStreamExt}; +use iroh_car::{CarHeader, CarReader, CarWriter}; +use noosphere_common::ConditionalSend; +use std::io::{Error as IoError, ErrorKind as IoErrorKind}; +use tokio::sync::mpsc::channel; +use tokio_stream::Stream; +use tokio_util::{ + io::{CopyToBytes, SinkWriter, StreamReader}, + sync::PollSender, +}; + +/// Takes a [Bytes] stream and interprets it as a CARv1, returning a stream of +/// `(Cid, Vec)` blocks. +pub fn from_car_stream( + stream: S, +) -> impl Stream)>> + ConditionalSend + 'static +where + E: std::error::Error + Send + Sync + 'static, + S: Stream> + ConditionalSend + 'static, +{ + let stream = stream.map_err(|error| std::io::Error::new(std::io::ErrorKind::Other, error)); + + try_stream! { + tokio::pin!(stream); + + let reader = CarReader::new(StreamReader::new(stream)).await?; + let stream = reader.stream(); + + tokio::pin!(stream); + + while let Some(entry) = tokio_stream::StreamExt::try_next(&mut stream).await? { + yield entry; + } + } +} + +/// Takes a list of roots and a stream of blocks (pairs of [Cid] and +/// corresponding [Vec]), and produces an async byte stream that yields a +/// valid [CARv1](https://ipld.io/specs/transport/car/carv1/) +pub fn to_car_stream( + mut roots: Vec, + block_stream: S, +) -> impl Stream> + ConditionalSend +where + S: Stream)>> + ConditionalSend, +{ + if roots.is_empty() { + roots = vec![Cid::default()] + } + + try_stream! { + let (tx, mut rx) = channel::(16); + let sink = + PollSender::new(tx).sink_map_err(|error| { + error!("Failed to send CAR frame: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + }); + + let mut car_buffer = SinkWriter::new(CopyToBytes::new(sink)); + let car_header = CarHeader::new_v1(roots); + let mut car_writer = CarWriter::new(car_header, &mut car_buffer); + let mut sent_blocks = false; + + for await item in block_stream { + sent_blocks = true; + let (cid, block) = item.map_err(|error| { + error!("Failed to stream blocks: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + })?; + + car_writer.write(cid, block).await.map_err(|error| { + error!("Failed to write CAR frame: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + })?; + + car_writer.flush().await.map_err(|error| { + error!("Failed to flush CAR frames: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + })?; + + while let Ok(block) = rx.try_recv() { + yield block; + } + } + + if !sent_blocks { + car_writer.write_header().await.map_err(|error| { + error!("Failed to write CAR frame: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + })?; + car_writer.flush().await.map_err(|error| { + error!("Failed to flush CAR frames: {}", error); + IoError::from(IoErrorKind::BrokenPipe) + })?; + + while let Ok(block) = rx.try_recv() { + yield block; + } + } + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use async_stream::try_stream; + use cid::Cid; + use futures_util::Stream; + use libipld_cbor::DagCborCodec; + use noosphere_common::helpers::TestEntropy; + use noosphere_storage::{BlockStore, MemoryStorage, Storage}; + + use rand::Rng; + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + use crate::stream::{from_car_stream, put_block_stream, to_car_stream}; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_converts_block_streams_to_car_streams_and_back() -> Result<()> { + let test_entropy = TestEntropy::default(); + let rng_one = test_entropy.to_rng(); + let storage = MemoryStorage::default(); + let store_one = storage.get_block_store("one").await?; + let store_two = storage.get_block_store("two").await?; + + let block_stream = { + let mut store_one = store_one.clone(); + try_stream! { + for _ in 0..10 { + let block_bytes = rng_one.lock().await.gen::<[u8; 32]>(); + let block_cid = store_one.save::(block_bytes.as_ref()).await?; + + yield (block_cid, Vec::from(block_bytes)); + } + } + }; + // See: https://github.com/tokio-rs/async-stream/issues/33#issuecomment-1261435381 + let _: &dyn Stream)>> = &block_stream; + + let car_stream = to_car_stream(vec![], block_stream); + + let output_block_stream = from_car_stream(car_stream); + + put_block_stream(store_two.clone(), output_block_stream).await?; + + assert_eq!(store_one.entries.lock().await.len(), 10); + store_one.expect_replica_in(&store_two).await?; + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/stream/memo.rs b/rust/noosphere-core/src/stream/memo.rs new file mode 100644 index 000000000..3da8482f5 --- /dev/null +++ b/rust/noosphere-core/src/stream/memo.rs @@ -0,0 +1,283 @@ +use std::str::FromStr; + +use crate::{ + authority::collect_ucan_proofs, + data::{ContentType, Link, MemoIpld, SphereIpld}, + view::Sphere, +}; +use anyhow::{anyhow, Result}; +use async_stream::try_stream; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use noosphere_common::{spawn, ConditionalSend, TaskQueue}; +use noosphere_storage::{BlockStore, BlockStoreTap, UcanStore}; +use tokio::select; +use tokio_stream::{Stream, StreamExt}; +use ucan::{store::UcanJwtStore, Ucan}; + +use crate::stream::walk::{ + walk_versioned_map_changes_and, walk_versioned_map_elements, walk_versioned_map_elements_and, +}; +use crate::view::BodyChunkDecoder; + +/// Stream all the blocks required to reconstruct the history of a sphere since a +/// given point in time (or else the beginning of the history). +// TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy warning +#[allow(clippy::let_with_type_underscore)] +#[instrument(level = "trace", skip(store))] +pub fn memo_history_stream( + store: S, + latest: &Link, + since: Option<&Link>, + include_content: bool, +) -> impl Stream)>> + ConditionalSend +where + S: BlockStore + 'static, +{ + debug!("Streaming history via memo..."); + + let latest = latest.clone(); + let since = since.cloned(); + + try_stream! { + + let (store, mut rx) = BlockStoreTap::new(store.clone(), 64); + let memo = store.load::(&latest).await?; + + match memo.content_type() { + Some(ContentType::Sphere) => { + let mut history_task = Box::pin(spawn(async move { + let sphere = Sphere::from_memo(&memo, &store)?; + let identity = sphere.get_identity().await?; + let mut tasks = TaskQueue::default(); + + let mut previous_sphere_body_version = None; + let mut previous_sphere_body: Option = None; + + let history_stream = sphere.into_history_stream(since.as_ref()); + + tokio::pin!(history_stream); + + while let Some((version, sphere)) = history_stream.try_next().await? { + if let Some(previous_sphere_body_version) = previous_sphere_body_version { + let memo = sphere.to_memo().await?; + if memo.body == previous_sphere_body_version { + warn!("Skipping {version} delta for {identity}, no sphere changes detected..."); + continue; + } + } + + debug!("Replicating {version} delta for {identity}"); + + let sphere_body = sphere.to_body().await?; + let (replicate_authority, replicate_address_book, replicate_content) = { + if let Some(previous_sphere_body) = previous_sphere_body { + (previous_sphere_body.authority != sphere_body.authority, + previous_sphere_body.address_book != sphere_body.address_book, + previous_sphere_body.content != sphere_body.content) + } else { + (true, true, true) + } + }; + + if replicate_authority { + debug!("Replicating authority..."); + let authority = sphere.get_authority().await?; + let store = store.clone(); + + tasks.spawn(async move { + let delegations = authority.get_delegations().await?; + + walk_versioned_map_changes_and(delegations, store, |_, delegation, store| async move { + let ucan_store = UcanStore(store); + + collect_ucan_proofs(&Ucan::from_str(&ucan_store.require_token(&delegation.jwt).await?)?, &ucan_store).await?; + + Ok(()) + }).await?; + + let revocations = authority.get_revocations().await?; + revocations.load_changelog().await?; + + Ok(()) as Result<_, anyhow::Error> + }); + } + + if replicate_address_book { + debug!("Replicating address book..."); + let address_book = sphere.get_address_book().await?; + let identities = address_book.get_identities().await?; + + tasks.spawn(walk_versioned_map_changes_and(identities, store.clone(), |name, identity, store| async move { + let ucan_store = UcanStore(store); + trace!("Replicating proofs for {}", name); + if let Some(link_record) = identity.link_record(&ucan_store).await { + link_record.collect_proofs(&ucan_store).await?; + }; + + Ok(()) + })); + } + + if replicate_content { + debug!("Replicating content..."); + let content = sphere.get_content().await?; + let include_content = include_content; + + tasks.spawn(walk_versioned_map_changes_and(content, store.clone(), move |_, link, store| async move { + if include_content { + walk_memo_body(store, &link).await?; + } else { + link.load_from(&store).await?; + }; + Ok(()) + })); + } + + previous_sphere_body = Some(sphere_body); + previous_sphere_body_version = Some(sphere.to_memo().await?.body); + + drop(sphere); + } + + drop(store); + + tasks.join().await?; + + trace!("Done replicating!"); + + Ok(()) as Result<(), anyhow::Error> + })); + + let mut receiver_is_open = true; + let mut history_task_finished = false; + let mut yield_count = 0usize; + + while receiver_is_open { + select! { + next = rx.recv() => { + if let Some(block) = next { + trace!(cid = ?block.0, "Yielding block {yield_count}..."); + yield_count += 1; + yield block; + } else { + trace!("Receiver closed!"); + receiver_is_open = false; + } + Ok(Ok::<_, anyhow::Error>(())) + }, + history_result = &mut history_task, if !history_task_finished => { + trace!("History task completed!"); + history_task_finished = true; + history_result + } + }??; + } + + trace!("Done yielding {yield_count} blocks!"); + } + _ => { + Err(anyhow!("History streaming is only supported for spheres, but {latest} has content type {:?})", memo.content_type()))?; + } + } + } +} + +/// Stream all the blocks required to read the sphere at a given version (making no +/// assumptions of what historical data may already be available to the reader). +// TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy warning +#[allow(clippy::let_with_type_underscore)] +#[instrument(level = "trace", skip(store))] +pub fn memo_body_stream( + store: S, + memo_version: &Cid, +) -> impl Stream)>> + ConditionalSend +where + S: BlockStore + 'static, +{ + debug!("Streaming body via memo..."); + + let memo_version = *memo_version; + + try_stream! { + let (store, mut rx) = BlockStoreTap::new(store.clone(), 1024); + + let mut receiver_is_open = true; + let mut walk_memo_finished = false; + let mut walk_memo_finishes = Box::pin(walk_memo_body(store, &memo_version)); + + while receiver_is_open { + select! { + next = rx.recv() => { + if let Some(block) = next { + trace!("Yielding {}", block.0); + yield block; + } else { + receiver_is_open = false; + } + Ok::<_, anyhow::Error>(()) + }, + walk_memo_results = &mut walk_memo_finishes, if !walk_memo_finished => { + walk_memo_finished = true; + walk_memo_results + } + }?; + } + } +} + +#[allow(clippy::let_with_type_underscore)] +#[instrument(level = "trace", skip(store))] +async fn walk_memo_body(store: S, memo_version: &Cid) -> Result<()> +where + S: BlockStore + 'static, +{ + let memo = store.load::(memo_version).await?; + match memo.content_type() { + Some(ContentType::Sphere) => { + let sphere = Sphere::from_memo(&memo, &store)?; + let authority = sphere.get_authority().await?; + let address_book = sphere.get_address_book().await?; + let content = sphere.get_content().await?; + let identities = address_book.get_identities().await?; + let delegations = authority.get_delegations().await?; + let revocations = authority.get_revocations().await?; + + let mut tasks = TaskQueue::default(); + + tasks.spawn(walk_versioned_map_elements_and( + identities, + store.clone(), + |_, identity, store| async move { + let ucan_store = UcanStore(store); + if let Some(link_record) = identity.link_record(&ucan_store).await { + link_record.collect_proofs(&ucan_store).await?; + }; + Ok(()) + }, + )); + tasks.spawn(walk_versioned_map_elements_and( + content, + store.clone(), + move |_, link, store| async move { + link.load_from(&store).await?; + Ok(()) + }, + )); + tasks.spawn(walk_versioned_map_elements(delegations)); + tasks.spawn(walk_versioned_map_elements(revocations)); + + tasks.join().await?; + } + Some(_) => { + let stream = BodyChunkDecoder(&memo.body, &store).stream(); + + tokio::pin!(stream); + + while (stream.try_next().await?).is_some() {} + } + None => (), + }; + + Ok(()) +} diff --git a/rust/noosphere-core/src/stream/mod.rs b/rust/noosphere-core/src/stream/mod.rs new file mode 100644 index 000000000..36496667a --- /dev/null +++ b/rust/noosphere-core/src/stream/mod.rs @@ -0,0 +1,406 @@ +//! Utilities to support producing streams of blocks, as well as converting +//! streams of blocks to and from CARv1-encoded byte streams + +mod block; +mod car; +mod memo; +mod walk; + +pub use block::*; +pub use car::*; +pub use memo::*; +pub use walk::*; + +#[cfg(test)] +mod tests { + use anyhow::Result; + use std::collections::BTreeSet; + use ucan::store::UcanJwtStore; + + use crate::{ + context::{ + HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite, + }, + data::{BodyChunkIpld, ContentType, LinkRecord, MemoIpld}, + helpers::{ + make_valid_link_record, simulated_sphere_context, touch_all_sphere_blocks, + SimulationAccess, + }, + stream::{from_car_stream, memo_body_stream, memo_history_stream, to_car_stream}, + tracing::initialize_tracing, + view::{BodyChunkDecoder, Sphere}, + }; + use libipld_cbor::DagCborCodec; + use noosphere_storage::{BlockStore, MemoryStore, UcanStore}; + use tokio_stream::StreamExt; + + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_includes_all_link_records_and_proofs_from_the_address_book() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let mut db = sphere_context.sphere_context().await?.db().clone(); + + let (foo_did, foo_link_record, foo_link_record_link) = + make_valid_link_record(&mut db).await?; + + sphere_context.set_petname("foo", Some(foo_did)).await?; + sphere_context.save(None).await?; + sphere_context + .set_petname_record("foo", &foo_link_record) + .await?; + let final_version = sphere_context.save(None).await?; + + let mut other_store = MemoryStore::default(); + + let stream = memo_body_stream( + sphere_context.sphere_context().await?.db().clone(), + &final_version, + ); + + tokio::pin!(stream); + + while let Some((cid, block)) = stream.try_next().await? { + debug!("Received {cid}"); + other_store.put_block(&cid, &block).await?; + } + + let ucan_store = UcanStore(other_store); + + let link_record = + LinkRecord::try_from(ucan_store.require_token(&foo_link_record_link).await?)?; + + assert_eq!(link_record, foo_link_record); + + link_record.collect_proofs(&ucan_store).await?; + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_stream_all_blocks_in_a_sphere_version() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let changes = vec![ + (vec!["dogs", "birds"], vec!["alice", "bob"]), + (vec!["cats", "dogs"], vec!["gordon"]), + (vec!["birds"], vec!["cdata"]), + (vec!["cows", "beetles"], vec!["jordan", "ben"]), + ]; + + for (content_change, petname_change) in changes.iter() { + for slug in content_change { + sphere_context + .write( + slug, + &ContentType::Subtext, + format!("{} are cool", slug).as_bytes(), + None, + ) + .await?; + } + + for petname in petname_change { + sphere_context + .set_petname(petname, Some(format!("did:key:{}", petname).into())) + .await?; + } + + sphere_context.save(None).await?; + } + + let final_version = sphere_context.version().await?; + + let mut other_store = MemoryStore::default(); + + let mut received = BTreeSet::new(); + + let stream = memo_body_stream( + sphere_context.sphere_context().await?.db().clone(), + &final_version, + ); + + tokio::pin!(stream); + + while let Some((cid, block)) = stream.try_next().await? { + debug!("Received {cid}"); + assert!( + !received.contains(&cid), + "Got {cid} but we already received it", + ); + received.insert(cid); + other_store.put_block(&cid, &block).await?; + } + + let sphere = Sphere::at(&final_version, &other_store); + + let content = sphere.get_content().await?; + let identities = sphere.get_address_book().await?.get_identities().await?; + + for (content_change, petname_change) in changes.iter() { + for slug in content_change { + let _ = content.get(&slug.to_string()).await?.cloned().unwrap(); + } + + for petname in petname_change { + let _ = identities.get(&petname.to_string()).await?; + } + } + + touch_all_sphere_blocks(&sphere).await?; + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_stream_all_delta_blocks_for_a_range_of_history() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let changes = vec![ + (vec!["dogs", "birds"], vec!["alice", "bob"]), + (vec!["cats", "dogs"], vec!["gordon"]), + (vec!["birds"], vec!["cdata"]), + (vec!["cows", "beetles"], vec!["jordan", "ben"]), + ]; + + let original_store = sphere_context.sphere_context().await?.db().clone(); + let mut versions = Vec::new(); + + for (content_change, petname_change) in changes.iter() { + for slug in content_change { + sphere_context + .write( + slug, + &ContentType::Subtext, + format!("{} are cool", slug).as_bytes(), + None, + ) + .await?; + } + + for petname in petname_change { + let (id, record, _) = + make_valid_link_record(&mut UcanStore(original_store.clone())).await?; + sphere_context.set_petname(petname, Some(id)).await?; + versions.push(sphere_context.save(None).await?); + sphere_context.set_petname_record(petname, &record).await?; + } + + versions.push(sphere_context.save(None).await?); + } + + let mut other_store = MemoryStore::default(); + + let first_version = versions.first().unwrap(); + let stream = memo_body_stream(original_store.clone(), first_version); + + tokio::pin!(stream); + + while let Some((cid, block)) = stream.try_next().await? { + other_store.put_block(&cid, &block).await?; + } + + let sphere = Sphere::at(first_version, &other_store); + + touch_all_sphere_blocks(&sphere).await?; + + for i in 1..=3 { + let version = versions.get(i).unwrap(); + let sphere = Sphere::at(version, &other_store); + + assert!(touch_all_sphere_blocks(&sphere).await.is_err()); + } + + let stream = memo_history_stream( + original_store, + versions.last().unwrap(), + Some(first_version), + false, + ); + + tokio::pin!(stream); + + while let Some((cid, block)) = stream.try_next().await? { + other_store.put_block(&cid, &block).await?; + } + + for i in 1..=3 { + let version = versions.get(i).unwrap(); + let sphere = Sphere::at(version, &other_store); + sphere.hydrate().await?; + + touch_all_sphere_blocks(&sphere).await?; + } + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_stream_all_blocks_in_some_sphere_content() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + let mut db = sphere_context.sphere_context().await?.db_mut().clone(); + + let chunks = [b"foo", b"bar", b"baz"]; + + let mut next_chunk_cid = None; + + for bytes in chunks.iter().rev() { + next_chunk_cid = Some( + db.save::(&BodyChunkIpld { + bytes: bytes.to_vec(), + next: next_chunk_cid, + }) + .await?, + ); + } + + let content_cid = sphere_context + .link("foo", &ContentType::Bytes, &next_chunk_cid.unwrap(), None) + .await?; + + let stream = memo_body_stream( + sphere_context.sphere_context().await?.db().clone(), + &content_cid, + ); + + let mut store = MemoryStore::default(); + + tokio::pin!(stream); + + while let Some((cid, block)) = stream.try_next().await? { + store.put_block(&cid, &block).await?; + } + + let memo = store.load::(&content_cid).await?; + + let mut buffer = Vec::new(); + let body_stream = BodyChunkDecoder(&memo.body, &store).stream(); + + tokio::pin!(body_stream); + + while let Some(bytes) = body_stream.try_next().await? { + buffer.append(&mut Vec::from(bytes)); + } + + assert_eq!(buffer.as_slice(), b"foobarbaz"); + + Ok(()) + } + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] + async fn it_can_stream_all_blocks_in_a_sphere_version_as_a_car() -> Result<()> { + initialize_tracing(None); + + let (mut sphere_context, _) = + simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; + + let changes = vec![ + (vec!["dogs", "birds"], vec!["alice", "bob"]), + (vec!["cats", "dogs"], vec!["gordon"]), + (vec!["birds"], vec!["cdata"]), + (vec!["cows", "beetles"], vec!["jordan", "ben"]), + ]; + + for (content_change, petname_change) in changes.iter() { + for slug in content_change { + sphere_context + .write( + slug, + &ContentType::Subtext, + format!("{} are cool", slug).as_bytes(), + None, + ) + .await?; + } + + for petname in petname_change { + sphere_context + .set_petname(petname, Some(format!("did:key:{}", petname).into())) + .await?; + } + + sphere_context.save(None).await?; + } + + let mut db = sphere_context.sphere_context().await?.db().clone(); + let (id, link_record, _) = make_valid_link_record(&mut db).await?; + sphere_context.set_petname("hasrecord", Some(id)).await?; + sphere_context.save(None).await?; + sphere_context + .set_petname_record("hasrecord", &link_record) + .await?; + sphere_context.save(None).await?; + + let final_version = sphere_context.version().await?; + + let mut other_store = MemoryStore::default(); + + let stream = to_car_stream( + vec![final_version.clone().into()], + memo_body_stream(db.clone(), &final_version), + ); + + let block_stream = from_car_stream(stream); + + let mut received = BTreeSet::new(); + tokio::pin!(block_stream); + + while let Some((cid, block)) = block_stream.try_next().await? { + debug!("Received {cid}"); + assert!( + !received.contains(&cid), + "Got {cid} but we already received it", + ); + received.insert(cid); + other_store.put_block(&cid, &block).await?; + } + + let sphere = Sphere::at(&final_version, &other_store); + + let content = sphere.get_content().await?; + let identities = sphere.get_address_book().await?.get_identities().await?; + + for (content_change, petname_change) in changes.iter() { + for slug in content_change { + let _ = content.get(&slug.to_string()).await?.cloned().unwrap(); + } + + for petname in petname_change { + let _ = identities.get(&petname.to_string()).await?; + } + } + + let has_record = identities.get(&"hasrecord".into()).await?.unwrap(); + let has_record_version = has_record.link_record(&UcanStore(other_store)).await; + + assert!( + has_record_version.is_some(), + "We got a resolved link record from the stream" + ); + + touch_all_sphere_blocks(&sphere).await?; + + Ok(()) + } +} diff --git a/rust/noosphere-core/src/stream/walk.rs b/rust/noosphere-core/src/stream/walk.rs new file mode 100644 index 000000000..b205b7717 --- /dev/null +++ b/rust/noosphere-core/src/stream/walk.rs @@ -0,0 +1,72 @@ +use crate::{ + data::{MapOperation, VersionedMapKey, VersionedMapValue}, + view::VersionedMap, +}; +use anyhow::Result; +use noosphere_storage::BlockStore; +use std::ops::Fn; +use tokio_stream::StreamExt; + +/// Given a [VersionedMap], visit its changelog and all of its underlying entries +pub async fn walk_versioned_map_elements( + versioned_map: VersionedMap, +) -> Result<()> +where + K: VersionedMapKey + 'static, + V: VersionedMapValue + 'static, + S: BlockStore + 'static, +{ + versioned_map.get_changelog().await?; + let stream = versioned_map.into_stream().await?; + tokio::pin!(stream); + while (stream.try_next().await?).is_some() {} + Ok(()) +} + +/// Given a [VersionedMap] and [BlockStore], visit the [VersionedMap]'s +/// changelog and all of its underlying entries, invoking a callback for each +/// entry +pub async fn walk_versioned_map_elements_and( + versioned_map: VersionedMap, + store: S, + callback: F, +) -> Result<()> +where + K: VersionedMapKey + 'static, + V: VersionedMapValue + 'static, + S: BlockStore + 'static, + Fut: std::future::Future>, + F: 'static + Fn(K, V, S) -> Fut, +{ + versioned_map.get_changelog().await?; + let stream = versioned_map.into_stream().await?; + tokio::pin!(stream); + while let Some((key, value)) = stream.try_next().await? { + callback(key, value, store.clone()).await?; + } + Ok(()) +} + +/// Given a [VersionedMap] and [BlockStore], visit the [VersionedMap]'s +/// changelog; then, invoke the provided callback with each entry associated +/// with an 'add' operation in the changelog +pub async fn walk_versioned_map_changes_and( + versioned_map: VersionedMap, + store: S, + callback: F, +) -> Result<()> +where + K: VersionedMapKey + 'static, + V: VersionedMapValue + 'static, + S: BlockStore + 'static, + Fut: std::future::Future>, + F: 'static + Fn(K, V, S) -> Fut, +{ + let changelog = versioned_map.load_changelog().await?; + for op in changelog.changes { + if let MapOperation::Add { key, value } = op { + callback(key, value, store.clone()).await?; + } + } + Ok(()) +} diff --git a/rust/noosphere-core/src/tracing.rs b/rust/noosphere-core/src/tracing.rs index 027e9b0e4..181b4c1e9 100644 --- a/rust/noosphere-core/src/tracing.rs +++ b/rust/noosphere-core/src/tracing.rs @@ -9,6 +9,7 @@ pub static NOOSPHERE_LOG_LEVEL_CRATES: &[&str] = &[ "noosphere", "noosphere_core", "noosphere_storage", + "noosphere_common", "noosphere_sphere", "noosphere_into", "noosphere_gateway", @@ -116,16 +117,22 @@ impl Default for NoosphereLogFormat { /// [`env-filter`](https://docs.rs/env_logger/0.10.0/env_logger/#enabling-logging) #[derive(Clone, Display, EnumString)] pub enum NoosphereLogLevel { + /// Equivalent to [tracing::Level::TRACE] #[strum(serialize = "trace")] Trace, + /// Equivalent to [tracing::Level::DEBUG] #[strum(serialize = "debug")] Debug, + /// Equivalent to [tracing::Level::INFO] #[strum(serialize = "info")] Info, + /// Equivalent to [tracing::Level::WARN] #[strum(serialize = "warn")] Warn, + /// Equivalent to [tracing::Level::ERROR] #[strum(serialize = "error")] Error, + /// Disables logging entirely #[strum(serialize = "off")] Off, } @@ -159,6 +166,8 @@ mod inner { use std::sync::Once; static INITIALIZE_TRACING: Once = Once::new(); + /// Initialize tracing-based logging throughout the Noosphere body of code, + /// as well as dependencies that implement tracing-based logging. pub fn initialize_tracing(_noosphere_log: Option) { INITIALIZE_TRACING.call_once(|| { console_error_panic_hook::set_once(); diff --git a/rust/noosphere-core/src/view/authority.rs b/rust/noosphere-core/src/view/authority.rs index f1ed4ac6c..e67c2d378 100644 --- a/rust/noosphere-core/src/view/authority.rs +++ b/rust/noosphere-core/src/view/authority.rs @@ -23,10 +23,13 @@ impl Authority where S: BlockStore, { + /// Get the [Cid] that refers to the underlying [AuthorityIpld] pub fn cid(&self) -> &Cid { &self.cid } + /// Initialize an [Authority] view over the [AuthorityIpld] referred to by + /// the specified [Cid] pub fn at(cid: &Cid, store: &S) -> Self { Authority { cid: *cid, @@ -45,6 +48,8 @@ where .clone()) } + /// Helper to initialize an empty [AuthorityIpld] in the case that no [Cid] + /// is specified pub async fn at_or_empty(cid: Option, store: &mut S) -> Result> where C: Deref, @@ -55,6 +60,8 @@ where }) } + /// Initialize an empty [AuthorityIpld] and return an [Authority] view over + /// it pub async fn empty(store: &mut S) -> Result { let ipld = AuthorityIpld::empty(store).await?; let cid = store.save::(ipld).await?; @@ -66,12 +73,14 @@ where }) } + /// Initialize the [Delegations] associated with this [Authority] pub async fn get_delegations(&self) -> Result> { let ipld = self.to_body().await?; Delegations::at_or_empty(Some(ipld.delegations), &mut self.store.clone()).await } + /// Initialize the [Revocations] associated with this [Authority] pub async fn get_revocations(&self) -> Result> { let ipld = self.to_body().await?; diff --git a/rust/noosphere-core/src/view/content.rs b/rust/noosphere-core/src/view/content.rs new file mode 100644 index 000000000..cfc13029c --- /dev/null +++ b/rust/noosphere-core/src/view/content.rs @@ -0,0 +1,29 @@ +use crate::data::BodyChunkIpld; +use async_stream::try_stream; +use bytes::Bytes; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use noosphere_storage::BlockStore; +use tokio_stream::Stream; + +/// Helper to easily decode a linked list of `BodyChunkIpld` as a byte stream +pub struct BodyChunkDecoder<'a, 'b, S: BlockStore>(pub &'a Cid, pub &'b S); + +impl<'a, 'b, S: BlockStore> BodyChunkDecoder<'a, 'b, S> { + /// Consume the [BodyChunkDecoder] and return an async [Stream] of bytes + /// representing the raw body contents + pub fn stream(self) -> impl Stream> + Unpin { + let mut next = Some(*self.0); + let store = self.1.clone(); + Box::pin(try_stream! { + while let Some(cid) = next { + debug!("Unpacking block {}...", cid); + let chunk = store.load::(&cid).await.map_err(|error| { + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, error.to_string()) + })?; + yield Bytes::from(chunk.bytes); + next = chunk.next; + } + }) + } +} diff --git a/rust/noosphere-core/src/view/mod.rs b/rust/noosphere-core/src/view/mod.rs index 940b1e5b1..457817e6b 100644 --- a/rust/noosphere-core/src/view/mod.rs +++ b/rust/noosphere-core/src/view/mod.rs @@ -1,11 +1,16 @@ +//! Views over raw Noosphere data to support efficient reading and traversal, as +//! well as provide higher-level operations related to same. + mod address; mod authority; +mod content; mod mutation; mod sphere; mod timeline; mod versioned_map; pub use authority::*; +pub use content::*; pub use mutation::*; pub use sphere::*; pub use timeline::*; diff --git a/rust/noosphere-core/src/view/mutation.rs b/rust/noosphere-core/src/view/mutation.rs index 03069311c..882d51504 100644 --- a/rust/noosphere-core/src/view/mutation.rs +++ b/rust/noosphere-core/src/view/mutation.rs @@ -12,9 +12,22 @@ use crate::{ use noosphere_storage::{BlockStore, UcanStore}; +#[cfg(doc)] +use crate::{ + data::VersionedMapIpld, + view::versioned_map::{Content, Delegations, Identities, Revocations}, +}; + +/// A [VersionedMapMutation] that corresponds to [Content] pub type ContentMutation = VersionedMapMutation>; + +/// A [VersionedMapMutation] that corresponds to [Identities] pub type IdentitiesMutation = VersionedMapMutation; + +/// A [VersionedMapMutation] that corresponds to [Delegations] pub type DelegationsMutation = VersionedMapMutation, DelegationIpld>; + +/// A [VersionedMapMutation] that corresponds to [Revocations] pub type RevocationsMutation = VersionedMapMutation, RevocationIpld>; #[cfg(doc)] @@ -27,12 +40,19 @@ use crate::view::Sphere; /// history by the sphere's key. #[derive(Debug)] pub struct SphereRevision { + /// The [Did] of the sphere that this revision corresponds to pub sphere_identity: Did, + /// A [BlockStore] that contains blocks assocaited with the sphere that this + /// revision corresponds to pub store: S, + /// The unsigned memo that wraps the root of the [SphereIpld] for this + /// sphere revision pub memo: MemoIpld, } impl SphereRevision { + /// Sign the [SphereRevision] with the provided credential and return the + /// [Link] pointing to the root of the new, signed revision pub async fn sign( &mut self, credential: &Credential, @@ -82,6 +102,8 @@ pub struct SphereMutation { } impl SphereMutation { + /// Initialize a new [SphereMutation] with the [Did] of the author of the + /// mutation pub fn new(did: &str) -> Self { SphereMutation { did: did.into(), @@ -108,38 +130,55 @@ impl SphereMutation { self.revocations = RevocationsMutation::new(&self.did); } + /// The [Did] of the author of the mutation pub fn did(&self) -> &str { &self.did } + /// A mutable reference to the changes to sphere content, given as a + /// [ContentMutation] pub fn content_mut(&mut self) -> &mut ContentMutation { &mut self.content } + /// An immutable reference to the changes to sphere content, given as a + /// [ContentMutation] pub fn content(&self) -> &ContentMutation { &self.content } + /// A mutable reference to the changes to sphere identities (petnames), + /// given as a [IdentitiesMutation] pub fn identities_mut(&mut self) -> &mut IdentitiesMutation { &mut self.identities } + /// An immutable reference to the changes to sphere identities (petnames), + /// given as a [IdentitiesMutation] pub fn identities(&self) -> &IdentitiesMutation { &self.identities } + /// A mutable reference to the changes to sphere delegations (of authority), + /// given as a [DelegationsMutation] pub fn delegations_mut(&mut self) -> &mut DelegationsMutation { &mut self.delegations } + /// An immutable reference to the changes to sphere delegations (of + /// authority), given as a [DelegationsMutation] pub fn delegations(&self) -> &DelegationsMutation { &self.delegations } + /// A mutable reference to the changes to sphere revocations (of authority), + /// given as a [RevocationsMutation] pub fn revocations_mut(&mut self) -> &mut RevocationsMutation { &mut self.revocations } + /// An immutable reference to the changes to sphere revocations (of + /// authority), given as a [RevocationsMutation] pub fn revocations(&self) -> &RevocationsMutation { &self.revocations } @@ -154,6 +193,7 @@ impl SphereMutation { } } +/// A generalized expression of a mutation to a [VersionedMapIpld] #[derive(Default, Debug)] pub struct VersionedMapMutation where @@ -169,6 +209,9 @@ where K: VersionedMapKey, V: VersionedMapValue, { + /// Set the changes as expressed by a [ChangelogIpld] to this + /// [VersionedMapMutation]; the mutation will adopt the author of the + /// changelog as its own author pub fn apply_changelog(&mut self, changelog: &ChangelogIpld>) -> Result<()> { let did = changelog .did @@ -188,20 +231,26 @@ where Ok(()) } + /// Initialize a new [VersionedMapMutation] whose author has the given [Did] pub fn new(did: &str) -> Self { VersionedMapMutation { did: did.into(), changes: Default::default(), } } + + /// Get the [Did] of the author of the [VersionedMapMutation] pub fn did(&self) -> &str { &self.did } + /// Get the changes (as [MapOperation]s) represented by this + /// [VersionedMapMutation] pub fn changes(&self) -> &[MapOperation] { &self.changes } + /// Record a change to the related [VersionedMapIpld] by key and value pub fn set(&mut self, key: &K, value: &V) { self.changes.push(MapOperation::Add { key: key.clone(), @@ -209,6 +258,7 @@ where }); } + /// Remove a change from the [VersionedMapMutation] pub fn remove(&mut self, key: &K) { self.changes.push(MapOperation::Remove { key: key.clone() }); } diff --git a/rust/noosphere-core/src/view/sphere.rs b/rust/noosphere-core/src/view/sphere.rs index 914d8cc2a..a4ad89fab 100644 --- a/rust/noosphere-core/src/view/sphere.rs +++ b/rust/noosphere-core/src/view/sphere.rs @@ -31,6 +31,10 @@ use noosphere_storage::{base64_decode, block_serialize, BlockStore, SphereDb, St use super::{address::AddressBook, Authority, Delegations, Identities, Revocations, Timeslice}; +/// An arbitrarily long value for sphere authorizations that should outlive the +/// keys they authorize +// TODO: Recent UCAN versions allow setting a null expiry; we should use that +// instead pub const SPHERE_LIFETIME: u64 = 315360000000; // 10,000 years (arbitrarily high) /// High-level Sphere I/O @@ -224,10 +228,14 @@ impl Sphere { }) } + /// Get an immutable reference to the [BlockStore] that is in use by this + /// [Sphere] view pub fn store(&self) -> &S { &self.store } + /// Get a mutable reference to the [BlockStore] that is in use by this + /// [Sphere] view pub fn store_mut(&mut self) -> &mut S { &mut self.store } @@ -307,6 +315,8 @@ impl Sphere { Ok(Authority::at(&sphere.authority, &self.store.clone())) } + /// Attempt to load the [AddressBook] of this sphere. If no address book is + /// found, an empty one is initialized. pub async fn get_address_book(&self) -> Result> { let sphere = self.to_body().await?; diff --git a/rust/noosphere-core/src/view/timeline.rs b/rust/noosphere-core/src/view/timeline.rs index afef5b29b..d9985ece9 100644 --- a/rust/noosphere-core/src/view/timeline.rs +++ b/rust/noosphere-core/src/view/timeline.rs @@ -11,16 +11,26 @@ use noosphere_storage::BlockStore; // Assumptions: // - network operations are _always_ mediated by a "remote" agent (no client-to-client syncing) // - the "remote" always has the authoritative state (we always rebase merge onto remote's tip) + +#[cfg(doc)] +use tokio_stream::Stream; + +/// A helper for turning contiguous ranges of [Link]s into [Timeslice]s. #[derive(Debug)] pub struct Timeline<'a, S: BlockStore> { + /// The [BlockStore] that will be shared with any [Timeslice]s produced by + /// this [Timeline] pub store: &'a S, } impl<'a, S: BlockStore> Timeline<'a, S> { + /// Initialize a new [Timeline] with a backing [BlockStore] pub fn new(store: &'a S) -> Self { Timeline { store } } + /// Produce a [Timeslice], which represents a reverse-chronological series + /// of [Link] that occur between a specified bounds. pub fn slice( &'a self, future: &'a Link, @@ -37,6 +47,9 @@ impl<'a, S: BlockStore> Timeline<'a, S> { // TODO(#263): Consider using async-stream crate for this // TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy // warning + /// Produce a [TryStream] whose items are a series of [(Link, + /// MemoIpld)], each one the ancestor of the last, yielded in + /// reverse-chronological order. #[allow(clippy::let_with_type_underscore)] #[instrument(level = "trace", skip(self))] pub fn stream( @@ -74,30 +87,46 @@ impl<'a, S: BlockStore> Timeline<'a, S> { } } +/// A [Timeslice] represents a bounded chronological range of [Link] +/// within a [Timeline]. #[derive(Debug)] pub struct Timeslice<'a, S: BlockStore> { + /// The associated [Timeline] of this [Timeslice] pub timeline: &'a Timeline<'a, S>, + /// The bound in the chronological "past," e.g., the earliest version; + /// `None` means "the (inclusive) beginning" pub past: Option<&'a Link>, + /// The bound in the chronological "future" e.g., the most recent version pub future: &'a Link, + /// Whether or not to exclude the configured `past` from any iteration over + /// the series of versions pub exclude_past: bool, } impl<'a, S: BlockStore> Timeslice<'a, S> { + /// Produce a [TryStream] from this [Timeslice] that yields sphere versions + /// and their memos in reverse-chronological order pub fn stream(&self) -> impl TryStream, MemoIpld)>> { self.timeline .stream(self.future, self.past, self.exclude_past) } + /// Configure the [Timeslice] to be inclusive of the `past` bound pub fn include_past(mut self) -> Self { self.exclude_past = false; self } + /// Configure the [Timeslice] to be exclusive of the `past` bound pub fn exclude_past(mut self) -> Self { self.exclude_past = true; self } + /// Aggregate an array of versions in chronological order and return it; + /// note that this can be quite memory costly (depending on how much history + /// is being aggregated), so it is better to stream in reverse-chronological + /// order if possible. pub async fn to_chronological(&self) -> Result, MemoIpld)>> { let mut chronological = VecDeque::new(); let mut stream = Box::pin(self.stream()); diff --git a/rust/noosphere-core/src/view/versioned_map.rs b/rust/noosphere-core/src/view/versioned_map.rs index e3aa56b66..ef4525826 100644 --- a/rust/noosphere-core/src/view/versioned_map.rs +++ b/rust/noosphere-core/src/view/versioned_map.rs @@ -20,9 +20,13 @@ use noosphere_storage::{block_serialize, BlockStore}; use super::VersionedMapMutation; +/// A [VersionedMap] that represents the content space of a sphere pub type Content = VersionedMap, S>; +/// A [VersionedMap] that represents the petname space of a sphere pub type Identities = VersionedMap; +/// A [VersionedMap] that represents the key authorizations in a sphere pub type Delegations = VersionedMap, DelegationIpld, S>; +/// A [VersionedMap] that represents the authority revocations in a sphere pub type Revocations = VersionedMap, RevocationIpld, S>; /// A view over a [VersionedMapIpld] which provides high-level traversal of the @@ -61,12 +65,16 @@ where .clone()) } + /// Get the [ChangelogIpld] for the underlying [VersionedMapIpld], loading + /// it from storage if it is not available pub async fn get_changelog(&self) -> Result<&ChangelogIpld>> { self.changelog .get_or_try_init(|| async { self.load_changelog().await }) .await } + /// Load the [ChangelogIpld] for the underlying [VersionedMapIpld] from + /// storage pub async fn load_changelog(&self) -> Result>> { let ipld = self.to_body().await?; self.store @@ -74,17 +82,22 @@ where .await } + /// Get the [Hamt] for the underlying [VersionedMapIpld], loading it from + /// storage if it is not available pub async fn get_hamt(&self) -> Result<&Hamt> { self.hamt .get_or_try_init(|| async { self.load_hamt().await }) .await } + /// Load the [Hamt] for the underlying [VersionedMapIpld] from storage async fn load_hamt(&self) -> Result> { let ipld = self.to_body().await?; ipld.load_hamt(&self.store).await } + /// Initialize the [VersionedMap] over a [VersionedMapIpld] referred to by its [Cid] if known, or else + /// a newly-initialized, empty [VersionedMapIpld]. pub async fn at_or_empty(cid: Option, store: &mut S) -> Result> where C: Deref, @@ -95,10 +108,12 @@ where }) } + /// Get the [Cid] of the underlying [VersionedMapIpld] pub fn cid(&self) -> &Cid { &self.cid } + /// Initialize the [VersionedMap] over a [VersionedMapIpld] referred to by its [Cid] pub fn at(cid: &Cid, store: &S) -> VersionedMap { VersionedMap { cid: *cid, @@ -109,6 +124,8 @@ where } } + /// Initialize and store an empty [VersionedMapIpld], configuring a [VersionedMap] to + /// point to it by its [Cid] pub async fn empty(store: &mut S) -> Result> { let ipld = VersionedMapIpld::::empty(store).await?; let cid = store.save::(ipld).await?; @@ -183,6 +200,8 @@ where .ok_or_else(|| anyhow!("Key {} not found!", key)) } + /// Apply a [VersionedMapMutation] to the underlying [VersionedMapIpld] by iterating + /// over the changes in the mutation and performing them one at a time. pub async fn apply_with_cid( cid: Option, mutation: &VersionedMapMutation, @@ -219,6 +238,9 @@ where store.save::(&links_ipld).await } + /// Iterate over the keys and values of the underlying [VersionedMapIpld] + /// sequentially. Note: consider using [VersionedMap::stream] instead for + /// better ergonomics. pub async fn for_each(&self, for_each: ForEach) -> Result<()> where ForEach: FnMut(&K, &V) -> Result<()>, @@ -226,6 +248,8 @@ where self.get_hamt().await?.for_each(for_each).await } + /// Produce a [Stream] that yields `(key, value)` tuples for all entries + /// in the underlying [VersionedMapIpld]. pub async fn stream<'a>( &'a self, ) -> Result> + 'a>>> { @@ -239,6 +263,8 @@ where V: VersionedMapValue + 'static, S: BlockStore + 'static, { + /// Consume the [VersionedMap] and produce a [Stream] that yields `(key, + /// value)` tuples for all entries in the underlying [VersionedMapIpld]. pub async fn into_stream(self) -> Result>> { Ok(self.load_hamt().await?.into_stream()) } diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml index 359aeeb6a..5fa9aa6fd 100644 --- a/rust/noosphere-gateway/Cargo.toml +++ b/rust/noosphere-gateway/Cargo.toml @@ -15,6 +15,9 @@ repository = "https://github.com/subconsciousnetwork/noosphere" homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" +[dependencies] +tracing = { workspace = true } + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls", "stream"] } @@ -33,8 +36,8 @@ tower = { workspace = true } tower-http = { workspace = true, features = ["cors", "trace"] } async-trait = "~0.1" async-stream = { workspace = true } -tracing = { workspace = true } wnfs-namefilter = { version = "0.1.21" } +wnfs-common = "=0.1.23" url = { workspace = true, features = ["serde"] } mime_guess = "^2" @@ -43,8 +46,6 @@ noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs" } noosphere-core = { version = "0.15.2", path = "../noosphere-core" } noosphere-ns = { version = "0.10.2", path = "../noosphere-ns" } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } -noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" } -noosphere-api = { version = "0.12.2", path = "../noosphere-api" } ucan = { workspace = true } ucan-key-support = { workspace = true } cid = { workspace = true } diff --git a/rust/noosphere-gateway/src/authority.rs b/rust/noosphere-gateway/src/authority.rs index 4be66c298..7b81988df 100644 --- a/rust/noosphere-gateway/src/authority.rs +++ b/rust/noosphere-gateway/src/authority.rs @@ -10,7 +10,7 @@ use axum::{ }; use libipld_core::cid::Cid; use noosphere_core::authority::{SphereAbility, SphereReference, SPHERE_SEMANTICS}; -use noosphere_sphere::SphereContext; +use noosphere_core::context::SphereContext; use noosphere_storage::NativeStorage; use tokio::sync::Mutex; diff --git a/rust/noosphere-gateway/src/error.rs b/rust/noosphere-gateway/src/error.rs new file mode 100644 index 000000000..bd529cc1e --- /dev/null +++ b/rust/noosphere-gateway/src/error.rs @@ -0,0 +1,62 @@ +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::Json; +use noosphere_core::api::v0alpha2::PushError; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct GatewayError { + pub error: String, +} + +pub struct GatewayErrorResponse(StatusCode, GatewayError); + +impl IntoResponse for GatewayErrorResponse { + fn into_response(self) -> axum::response::Response { + (self.0, Json(self.1)).into_response() + } +} + +impl From for GatewayErrorResponse { + fn from(value: anyhow::Error) -> Self { + GatewayErrorResponse( + StatusCode::INTERNAL_SERVER_ERROR, + GatewayError { + error: value.to_string(), + }, + ) + } +} + +impl From for GatewayErrorResponse { + fn from(value: PushError) -> Self { + GatewayErrorResponse( + StatusCode::from(&value), + GatewayError { + error: value.to_string(), + }, + ) + } +} + +impl From for GatewayErrorResponse { + fn from(value: StatusCode) -> Self { + GatewayErrorResponse( + value, + GatewayError { + error: value.to_string(), + }, + ) + } +} + +impl From for GatewayErrorResponse { + fn from(value: axum::Error) -> Self { + GatewayErrorResponse( + StatusCode::INTERNAL_SERVER_ERROR, + GatewayError { + error: value.to_string(), + }, + ) + } +} diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs index f72914e77..9a0fe28f7 100644 --- a/rust/noosphere-gateway/src/gateway.rs +++ b/rust/noosphere-gateway/src/gateway.rs @@ -3,19 +3,20 @@ use axum::extract::DefaultBodyLimit; use axum::http::{HeaderValue, Method}; use axum::routing::{get, put}; use axum::{Extension, Router, Server}; +use noosphere_core::context::HasMutableSphereContext; use noosphere_core::data::Did; use noosphere_ipfs::KuboClient; -use noosphere_sphere::HasMutableSphereContext; use noosphere_storage::Storage; use std::net::TcpListener; use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; use url::Url; -use noosphere_api::route::Route as GatewayRoute; +use noosphere_core::api::{v0alpha1, v0alpha2}; +// use noosphere_core::api::route::Route as GatewayRoute; use crate::{ - route::{did_route, fetch_route, identify_route, push_route, replicate_route}, + handlers, worker::{ start_ipfs_syndication, start_name_system, NameSystemConfiguration, NameSystemConnectionType, @@ -85,17 +86,31 @@ where ); let app = Router::new() - .route(&GatewayRoute::Did.to_string(), get(did_route)) .route( - &GatewayRoute::Replicate(None).to_string(), - get(replicate_route::), + &v0alpha1::Route::Did.to_string(), + get(handlers::v0alpha1::did_route), ) .route( - &GatewayRoute::Identify.to_string(), - get(identify_route::), + &v0alpha1::Route::Replicate(None).to_string(), + get(handlers::v0alpha1::replicate_route::), + ) + .route( + &v0alpha1::Route::Identify.to_string(), + get(handlers::v0alpha1::identify_route::), + ) + .route( + &v0alpha1::Route::Push.to_string(), + #[allow(deprecated)] + put(handlers::v0alpha1::push_route::), + ) + .route( + &v0alpha2::Route::Push.to_string(), + put(handlers::v0alpha2::push_route::), + ) + .route( + &v0alpha1::Route::Fetch.to_string(), + get(handlers::v0alpha1::fetch_route::), ) - .route(&GatewayRoute::Push.to_string(), put(push_route::)) - .route(&GatewayRoute::Fetch.to_string(), get(fetch_route::)) .layer(Extension(sphere_context.clone())) .layer(Extension(gateway_scope.clone())) .layer(Extension(ipfs_client)) diff --git a/rust/noosphere-gateway/src/handlers/mod.rs b/rust/noosphere-gateway/src/handlers/mod.rs new file mode 100644 index 000000000..9f424705d --- /dev/null +++ b/rust/noosphere-gateway/src/handlers/mod.rs @@ -0,0 +1,2 @@ +pub mod v0alpha1; +pub mod v0alpha2; diff --git a/rust/noosphere-gateway/src/route/did.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/did.rs similarity index 100% rename from rust/noosphere-gateway/src/route/did.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/did.rs diff --git a/rust/noosphere-gateway/src/route/fetch.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs similarity index 92% rename from rust/noosphere-gateway/src/route/fetch.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs index 37231affb..8067394da 100644 --- a/rust/noosphere-gateway/src/route/fetch.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs @@ -4,14 +4,15 @@ use anyhow::Result; use axum::{body::StreamBody, extract::Query, http::StatusCode, Extension}; use bytes::Bytes; -use noosphere_api::data::FetchParameters; use noosphere_core::{ + api::v0alpha1::FetchParameters, authority::{generate_capability, SphereAbility}, + context::HasSphereContext, data::{Link, MemoIpld}, + stream::{memo_history_stream, to_car_stream}, view::Sphere, }; use noosphere_ipfs::{IpfsStore, KuboClient}; -use noosphere_sphere::{car_stream, memo_history_stream, HasSphereContext}; use noosphere_storage::{BlockStoreRetry, SphereDb, Storage}; use tokio_stream::{Stream, StreamExt}; @@ -71,7 +72,7 @@ where .map(|cid| cid.to_string()) .unwrap_or_else(|| "the beginning...".into()) ); - return Ok(Box::pin(car_stream(vec![], tokio_stream::empty()))); + return Ok(Box::pin(to_car_stream(vec![], tokio_stream::empty()))); } debug!( @@ -84,7 +85,7 @@ where let store = BlockStoreRetry::from(IpfsStore::new(db.clone(), Some(ipfs_client))); - let stream = memo_history_stream(store.clone(), &latest_local_sphere_cid, since); + let stream = memo_history_stream(store.clone(), &latest_local_sphere_cid, since, false); debug!("Resolving latest counterpart sphere version..."); @@ -116,18 +117,19 @@ where .unwrap_or_else(|| "the beginning".into()) ); - return Ok(Box::pin(car_stream( + return Ok(Box::pin(to_car_stream( vec![latest_local_sphere_cid.into()], stream.merge(memo_history_stream( store, latest_counterpart_sphere_cid, since.as_ref(), + false, )), ))); } None => { warn!("No revisions found for counterpart {}!", scope.counterpart); - Ok(Box::pin(car_stream( + Ok(Box::pin(to_car_stream( vec![latest_local_sphere_cid.into()], stream, ))) diff --git a/rust/noosphere-gateway/src/route/identify.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs similarity index 92% rename from rust/noosphere-gateway/src/route/identify.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs index df42026cd..7d522d21f 100644 --- a/rust/noosphere-gateway/src/route/identify.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs @@ -1,8 +1,8 @@ use crate::{authority::GatewayAuthority, GatewayScope}; use axum::{http::StatusCode, response::IntoResponse, Extension, Json}; -use noosphere_api::data::IdentifyResponse; +use noosphere_core::api::v0alpha1::IdentifyResponse; use noosphere_core::authority::{generate_capability, SphereAbility}; -use noosphere_sphere::HasSphereContext; +use noosphere_core::context::HasSphereContext; use noosphere_storage::Storage; pub async fn identify_route( @@ -12,7 +12,7 @@ pub async fn identify_route( ) -> Result where C: HasSphereContext, - S: Storage, + S: Storage + 'static, { debug!("Invoking identify route..."); diff --git a/rust/noosphere-gateway/src/route/mod.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/mod.rs similarity index 100% rename from rust/noosphere-gateway/src/route/mod.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/mod.rs diff --git a/rust/noosphere-gateway/src/route/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs similarity index 98% rename from rust/noosphere-gateway/src/route/push.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/push.rs index 7888314b1..858f5a5d6 100644 --- a/rust/noosphere-gateway/src/route/push.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs @@ -4,13 +4,13 @@ use anyhow::Result; use axum::{http::StatusCode, Extension}; -use noosphere_api::data::{PushBody, PushError, PushResponse}; +use noosphere_core::api::v0alpha1::{PushBody, PushError, PushResponse}; +use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor}; use noosphere_core::{ authority::{generate_capability, SphereAbility}, data::{Bundle, Link, LinkRecord, MapOperation, MemoIpld}, view::Sphere, }; -use noosphere_sphere::{HasMutableSphereContext, SphereContentWrite, SphereCursor}; use noosphere_storage::Storage; use tokio::sync::mpsc::UnboundedSender; use tokio_stream::StreamExt; @@ -23,6 +23,7 @@ use crate::{ }; // #[debug_handler] +#[deprecated(since = "0.8.1", note = "Please migrate to v0alpha2")] #[instrument( level = "debug", skip( diff --git a/rust/noosphere-gateway/src/route/replicate.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs similarity index 96% rename from rust/noosphere-gateway/src/route/replicate.rs rename to rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs index 860612b77..0d213fc11 100644 --- a/rust/noosphere-gateway/src/route/replicate.rs +++ b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs @@ -10,15 +10,14 @@ use axum::{ use bytes::Bytes; use cid::Cid; use libipld_cbor::DagCborCodec; -use noosphere_api::data::ReplicateParameters; +use noosphere_core::api::v0alpha1::ReplicateParameters; +use noosphere_core::context::HasMutableSphereContext; +use noosphere_core::stream::{memo_body_stream, memo_history_stream, to_car_stream}; use noosphere_core::{ authority::{generate_capability, SphereAbility}, data::{ContentType, MemoIpld}, }; use noosphere_ipfs::{IpfsStore, KuboClient}; -use noosphere_sphere::{ - car_stream, memo_body_stream, memo_history_stream, HasMutableSphereContext, -}; use noosphere_storage::{BlockStore, BlockStoreRetry, Storage}; use tokio_stream::Stream; @@ -85,9 +84,9 @@ where // of usage. Maybe somewhere in the ballpark of 1~10k revisions. It // should be a large-but-finite number. debug!("Streaming revisions from {} to {}", since, memo_version); - return Ok(StreamBody::new(Box::pin(car_stream( + return Ok(StreamBody::new(Box::pin(to_car_stream( vec![memo_version], - memo_history_stream(store, &memo_version.into(), Some(&since)), + memo_history_stream(store, &memo_version.into(), Some(&since), false), )))); } else { error!("Suggested version {since} is not a valid ancestor of {memo_version}"); @@ -104,7 +103,7 @@ where debug!("Streaming entire version for {}", memo_version); // Always fall back to a full replication - Ok(StreamBody::new(Box::pin(car_stream( + Ok(StreamBody::new(Box::pin(to_car_stream( vec![memo_version], memo_body_stream(store, &memo_version), )))) @@ -159,16 +158,16 @@ mod tests { use super::is_allowed_to_replicate_incrementally; use anyhow::Result; + use noosphere_core::context::{ + HasMutableSphereContext, HasSphereContext, SphereContext, SphereContextKey, SphereCursor, + }; + use noosphere_core::helpers::{simulated_sphere_context, SimulationAccess}; use noosphere_core::{ authority::{ generate_capability, generate_ed25519_key, Author, Authorization, SphereAbility, }, data::{DelegationIpld, RevocationIpld}, }; - use noosphere_sphere::{ - helpers::{simulated_sphere_context, SimulationAccess}, - HasMutableSphereContext, HasSphereContext, SphereContext, SphereContextKey, SphereCursor, - }; use tokio::sync::Mutex; use ucan::{builder::UcanBuilder, crypto::KeyMaterial}; diff --git a/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs b/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs new file mode 100644 index 000000000..621b20604 --- /dev/null +++ b/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs @@ -0,0 +1,3 @@ +mod push; + +pub use push::*; diff --git a/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs new file mode 100644 index 000000000..09b5f110c --- /dev/null +++ b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs @@ -0,0 +1,376 @@ +use std::{collections::BTreeSet, marker::PhantomData}; + +use anyhow::Result; + +use async_stream::try_stream; +use axum::{body::StreamBody, extract::BodyStream, Extension}; + +use bytes::Bytes; +use cid::Cid; +use libipld_cbor::DagCborCodec; +use noosphere_core::api::v0alpha2::{PushBody, PushError, PushResponse}; +use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor}; +use noosphere_core::stream::{ + from_car_stream, memo_history_stream, put_block_stream, to_car_stream, +}; +use noosphere_core::{ + authority::{generate_capability, SphereAbility}, + data::{Link, LinkRecord, MapOperation, MemoIpld}, + view::Sphere, +}; +use noosphere_storage::{block_deserialize, block_serialize, Storage}; +use tokio::sync::mpsc::UnboundedSender; +use tokio_stream::{Stream, StreamExt}; + +use crate::{ + authority::GatewayAuthority, + error::GatewayErrorResponse, + worker::{NameSystemJob, SyndicationJob}, + GatewayScope, +}; + +// #[debug_handler] +#[instrument( + level = "debug", + skip( + authority, + gateway_scope, + sphere_context, + syndication_tx, + name_system_tx, + stream + ) +)] +pub async fn push_route( + authority: GatewayAuthority, + Extension(sphere_context): Extension, + Extension(gateway_scope): Extension, + Extension(syndication_tx): Extension>>, + Extension(name_system_tx): Extension>>, + stream: BodyStream, +) -> Result>>, GatewayErrorResponse> +where + C: HasMutableSphereContext, + S: Storage + 'static, +{ + debug!("Invoking push route..."); + + authority.try_authorize(&generate_capability( + &gateway_scope.counterpart, + SphereAbility::Push, + ))?; + + let gateway_push_routine = GatewayPushRoutine { + sphere_context, + gateway_scope, + syndication_tx, + name_system_tx, + block_stream: Box::pin(from_car_stream(stream)), + storage_type: PhantomData, + }; + + Ok(StreamBody::new(gateway_push_routine.invoke().await?)) +} + +pub struct GatewayPushRoutine +where + C: HasMutableSphereContext + 'static, + S: Storage + 'static, + St: Stream)>> + Unpin + 'static, +{ + sphere_context: C, + gateway_scope: GatewayScope, + syndication_tx: UnboundedSender>, + name_system_tx: UnboundedSender>, + block_stream: St, + storage_type: PhantomData, +} + +impl GatewayPushRoutine +where + C: HasMutableSphereContext + 'static, + S: Storage + 'static, + St: Stream)>> + Unpin + 'static, +{ + #[instrument(level = "debug", skip(self))] + pub async fn invoke( + mut self, + ) -> Result> + Send + 'static, PushError> { + debug!("Invoking gateway push..."); + + let push_body = self.verify_history().await?; + + debug!(?push_body, "Received valid push body..."); + + self.incorporate_history(&push_body).await?; + self.synchronize_names(&push_body).await?; + + let (next_version, new_blocks) = self.update_gateway_sphere().await?; + + // These steps are order-independent + let _ = tokio::join!( + self.notify_name_resolver(&push_body), + self.notify_ipfs_syndicator(next_version.clone()) + ); + + let roots = vec![next_version.clone().into()]; + + let block_stream = try_stream! { + yield block_serialize::(PushResponse::Accepted { + new_tip: next_version + })?; + + for await block in new_blocks { + yield block?; + } + }; + + Ok(to_car_stream(roots, block_stream)) + } + + /// Ensure that the pushed history is not in direct conflict with our + /// history (in which case the pusher typically must sync first), and that + /// there is no missing history implied by the pushed history (in which + /// case, there is probably a sync bug in the client implementation). + async fn verify_history(&mut self) -> Result { + debug!("Verifying pushed sphere history..."); + + let push_body = if let Some((_, first_block)) = self.block_stream.try_next().await? { + block_deserialize::(&first_block)? + } else { + return Err(PushError::UnexpectedBody); + }; + + let gateway_sphere_tip = self.sphere_context.version().await?; + if Some(&gateway_sphere_tip) != push_body.counterpart_tip.as_ref() { + warn!( + "Gateway sphere conflict; we have {gateway_sphere_tip}, they have {:?}", + push_body.counterpart_tip + ); + return Err(PushError::Conflict); + } + + let sphere_identity = &push_body.sphere; + let gateway_sphere_context = self.sphere_context.sphere_context().await?; + let db = gateway_sphere_context.db(); + + let local_sphere_base_cid = db.get_version(sphere_identity).await?.map(|cid| cid.into()); + let request_sphere_base_cid = push_body.local_base.clone(); + + match (local_sphere_base_cid, request_sphere_base_cid) { + (Some(mine), theirs) => { + // TODO(#26): Probably should do some diligence here to check if + // their base is even in our lineage. Note that this condition + // will be hit if theirs is ahead of mine, which actually + // should be a "missing revisions" condition. + let conflict = match &theirs { + Some(cid) if cid != &mine => true, + None => true, + _ => false, + }; + + if conflict { + warn!( + "Counterpart sphere conflict; we have {mine}, they have {:?}", + theirs + ); + return Err(PushError::Conflict); + } + + if push_body.local_tip == mine { + warn!("No new changes in push body!"); + return Err(PushError::UpToDate); + } + } + (None, Some(_)) => { + error!("Missing local lineage!"); + return Err(PushError::MissingHistory); + } + _ => (), + }; + + Ok(push_body) + } + + /// Incorporate the pushed history into our storage, hydrating each new + /// revision in the history as we go. Then, update our local pointer to the + /// tip of the pushed history. + async fn incorporate_history(&mut self, push_body: &PushBody) -> Result<(), PushError> { + { + debug!("Merging pushed sphere history..."); + let mut sphere_context = self.sphere_context.sphere_context_mut().await?; + + put_block_stream(sphere_context.db_mut().clone(), &mut self.block_stream).await?; + + let PushBody { + local_base: base, + local_tip: tip, + .. + } = &push_body; + + let history: Vec, Sphere<_>)>> = + Sphere::at(tip, sphere_context.db()) + .into_history_stream(base.as_ref()) + .collect() + .await; + + for step in history.into_iter().rev() { + let (cid, sphere) = step?; + debug!("Hydrating {}", cid); + sphere.hydrate().await?; + } + + debug!( + "Setting {} tip to {}...", + self.gateway_scope.counterpart, tip + ); + + sphere_context + .db_mut() + .set_version(&self.gateway_scope.counterpart, tip) + .await?; + } + + self.sphere_context + .link_raw(&self.gateway_scope.counterpart, &push_body.local_tip) + .await?; + + Ok(()) + } + + async fn synchronize_names(&mut self, push_body: &PushBody) -> Result<(), PushError> { + debug!("Synchronizing name changes to local sphere..."); + + let my_sphere = self.sphere_context.to_sphere().await?; + let my_names = my_sphere.get_address_book().await?.get_identities().await?; + + let sphere = Sphere::at(&push_body.local_tip, my_sphere.store()); + let stream = sphere.into_history_stream(push_body.local_base.as_ref()); + + tokio::pin!(stream); + + let mut updated_names = BTreeSet::::new(); + let mut removed_names = BTreeSet::::new(); + + // Walk backwards through the history of the pushed sphere and aggregate + // name changes into a single mutation + while let Ok(Some((_, sphere))) = stream.try_next().await { + let changed_names = sphere + .get_address_book() + .await? + .get_identities() + .await? + .load_changelog() + .await?; + for operation in changed_names.changes { + match operation { + MapOperation::Add { key, value } => { + // Since we are walking backwards through history, we + // can ignore changes to names in the past that we have + // already encountered in the future + if updated_names.contains(&key) || removed_names.contains(&key) { + trace!("Skipping name add for '{}' (already seen)...", key); + continue; + } + + let my_value = my_names.get(&key).await?; + + // Only add to the mutation if the value has actually + // changed to avoid redundantly recording updates made + // on the client due to a previous sync + if my_value != Some(&value) { + debug!("Adding name '{}' ({})...", key, value.did); + self.sphere_context + .sphere_context_mut() + .await? + .mutation_mut() + .identities_mut() + .set(&key, &value); + } + + updated_names.insert(key); + } + MapOperation::Remove { key } => { + removed_names.insert(key.clone()); + + if updated_names.contains(&key) { + trace!("Skipping name removal for '{}' (already seen)...", key); + continue; + } + + debug!("Removing name '{}'...", key); + self.sphere_context + .sphere_context_mut() + .await? + .mutation_mut() + .identities_mut() + .remove(&key); + + updated_names.insert(key); + } + } + } + } + + Ok(()) + } + + /// Apply any mutations accrued during the push operation to the local + /// sphere and return the new version, along with the blocks needed to + /// synchronize the pusher with the latest local history. + async fn update_gateway_sphere( + &mut self, + ) -> Result<(Link, impl Stream)>>), PushError> { + debug!("Updating the gateway's sphere..."); + + // NOTE CDATA: "Previous version" doesn't cover all cases; this needs to be a version given + // in the push body, or else we don't know how far back we actually have to go (e.g., the name + // system may have created a new version in the mean time. + let previous_version = self.sphere_context.version().await?; + let next_version = SphereCursor::latest(self.sphere_context.clone()) + .save(None) + .await?; + + let db = self.sphere_context.sphere_context().await?.db().clone(); + let block_stream = memo_history_stream(db, &next_version, Some(&previous_version), false); + + Ok((next_version, block_stream)) + } + + /// Notify the name system that new names may need to be resolved + async fn notify_name_resolver(&self, push_body: &PushBody) -> Result<()> { + if let Some(name_record) = &push_body.name_record { + if let Err(error) = self.name_system_tx.send(NameSystemJob::Publish { + context: self.sphere_context.clone(), + record: LinkRecord::try_from(name_record)?, + republish: false, + }) { + warn!("Failed to request name record publish: {}", error); + } + } + + if let Err(error) = self.name_system_tx.send(NameSystemJob::ResolveSince { + context: self.sphere_context.clone(), + since: push_body.local_base.clone(), + }) { + warn!("Failed to request name system resolutions: {}", error); + }; + + Ok(()) + } + + /// Request that new history be syndicated to IPFS + async fn notify_ipfs_syndicator(&self, next_version: Link) -> Result<()> { + // TODO(#156): This should not be happening on every push, but rather on + // an explicit publish action. Move this to the publish handler when we + // have added it to the gateway. + if let Err(error) = self.syndication_tx.send(SyndicationJob { + revision: next_version, + context: self.sphere_context.clone(), + }) { + warn!("Failed to queue IPFS syndication job: {}", error); + }; + + Ok(()) + } +} diff --git a/rust/noosphere-gateway/src/lib.rs b/rust/noosphere-gateway/src/lib.rs index 88eca547e..f988c0250 100644 --- a/rust/noosphere-gateway/src/lib.rs +++ b/rust/noosphere-gateway/src/lib.rs @@ -1,4 +1,3 @@ -#[cfg(not(target_arch = "wasm32"))] #[macro_use] extern crate tracing; @@ -15,7 +14,10 @@ mod extractor; mod worker; #[cfg(not(target_arch = "wasm32"))] -mod route; +mod handlers; + +#[cfg(not(target_arch = "wasm32"))] +mod error; #[cfg(not(target_arch = "wasm32"))] mod gateway; diff --git a/rust/noosphere-gateway/src/worker/name_system.rs b/rust/noosphere-gateway/src/worker/name_system.rs index 059a0f5be..ea255cce7 100644 --- a/rust/noosphere-gateway/src/worker/name_system.rs +++ b/rust/noosphere-gateway/src/worker/name_system.rs @@ -1,28 +1,27 @@ use crate::try_or_reset::TryOrReset; use anyhow::anyhow; use anyhow::Result; -use noosphere_core::data::Link; -use noosphere_core::data::MemoIpld; -use noosphere_core::data::{ContentType, Did, IdentityIpld, LinkRecord, MapOperation}; +use noosphere_core::{ + context::{ + HasMutableSphereContext, SphereContentRead, SphereContentWrite, SphereCursor, + SpherePetnameRead, SpherePetnameWrite, COUNTERPART, + }, + data::{ContentType, Did, IdentityIpld, Link, LinkRecord, MapOperation, MemoIpld}, +}; use noosphere_ipfs::{IpfsStore, KuboClient}; use noosphere_ns::{server::HttpClient as NameSystemHttpClient, NameResolver}; -use noosphere_sphere::{ - HasMutableSphereContext, SphereCursor, SpherePetnameRead, SpherePetnameWrite, -}; -use noosphere_sphere::{SphereContentRead, SphereContentWrite, COUNTERPART}; -use noosphere_storage::KeyValueStore; -use noosphere_storage::{BlockStoreRetry, Storage, UcanStore}; -use std::fmt::Display; -use std::future::Future; +use noosphere_storage::{BlockStoreRetry, KeyValueStore, Storage, UcanStore}; use std::{ collections::{BTreeMap, BTreeSet}, + fmt::Display, + future::Future, string::ToString, sync::Arc, time::Duration, }; use strum_macros::Display; -use tokio::io::AsyncReadExt; use tokio::{ + io::AsyncReadExt, sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, oneshot::Sender, @@ -514,9 +513,9 @@ mod tests { use noosphere_core::{ authority::{generate_capability, SphereAbility}, data::LINK_RECORD_FACT_NAME, + helpers::{simulated_sphere_context, SimulationAccess}, }; use noosphere_ns::helpers::KeyValueNameResolver; - use noosphere_sphere::helpers::{simulated_sphere_context, SimulationAccess}; use ucan::builder::UcanBuilder; use super::*; diff --git a/rust/noosphere-gateway/src/worker/syndication.rs b/rust/noosphere-gateway/src/worker/syndication.rs index 90c1154f2..d5810e356 100644 --- a/rust/noosphere-gateway/src/worker/syndication.rs +++ b/rust/noosphere-gateway/src/worker/syndication.rs @@ -2,15 +2,15 @@ use std::{io::Cursor, sync::Arc}; use anyhow::Result; use libipld_cbor::DagCborCodec; +use noosphere_core::context::{ + metadata::COUNTERPART, HasMutableSphereContext, SphereContentRead, SphereContentWrite, + SphereCursor, +}; use noosphere_core::{ data::{ContentType, Did, Link, MemoIpld}, view::Timeline, }; use noosphere_ipfs::{IpfsClient, KuboClient}; -use noosphere_sphere::{ - metadata::COUNTERPART, HasMutableSphereContext, SphereContentRead, SphereContentWrite, - SphereCursor, -}; use noosphere_storage::{block_deserialize, block_serialize, BlockStore, KeyValueStore, Storage}; use serde::{Deserialize, Serialize}; use tokio::{ @@ -158,11 +158,10 @@ where let stream = db.query_links(&cid, { let filter = Arc::new(syndicated_blocks.clone()); - let kubo_client = kubo_client.clone(); move |cid| { let filter = filter.clone(); - let kubo_client = kubo_client.clone(); + // let kubo_client = kubo_client.clone(); let cid = *cid; async move { @@ -177,11 +176,7 @@ where return Ok(true); } - // This will probably end up being rather noisy for the - // IPFS node, but hopefully checking for a pin is not - // overly costly. We may have to come up with a - // different strategy if this turns out to be too noisy. - Ok(!kubo_client.block_is_pinned(&cid).await?) + Ok(false) } } }); diff --git a/rust/noosphere-into/Cargo.toml b/rust/noosphere-into/Cargo.toml index 7b5ad78eb..07fd3cf57 100644 --- a/rust/noosphere-into/Cargo.toml +++ b/rust/noosphere-into/Cargo.toml @@ -19,7 +19,6 @@ readme = "README.md" [dependencies] noosphere-core = { version = "0.15.2", path = "../noosphere-core" } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } -noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" } subtext = { version = "0.3.2", features = ["stream"] } async-trait = "~0.1" url = { workspace = true } @@ -36,7 +35,7 @@ tokio-stream = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "test-util"] } async-stream = { workspace = true } -futures = { version = "~0.3" } +futures = { workspace = true } async-compat = { version = "~0.2" } async-utf8-decoder = { version = "~0.3" } @@ -44,7 +43,7 @@ ucan = { workspace = true } ucan-key-support = { workspace = true } [dev-dependencies] -noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere", features = ["helpers"] } +noosphere-core = { version = "0.15.1", path = "../noosphere-core", features = ["helpers"] } wasm-bindgen-test = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] diff --git a/rust/noosphere-into/examples/notes-to-html/implementation.rs b/rust/noosphere-into/examples/notes-to-html/implementation.rs index b945378dd..0f323d073 100644 --- a/rust/noosphere-into/examples/notes-to-html/implementation.rs +++ b/rust/noosphere-into/examples/notes-to-html/implementation.rs @@ -1,9 +1,9 @@ use anyhow::{anyhow, Result}; use axum::{error_handling::HandleErrorLayer, http::StatusCode, routing::get_service}; -use noosphere_into::{sphere_into_html, NativeFs}; -use noosphere_sphere::{ +use noosphere_core::context::{ HasMutableSphereContext, SphereContentWrite, SphereContext, SphereContextKey, SphereCursor, }; +use noosphere_into::{sphere_into_html, NativeFs}; use std::{ffi::OsStr, net::SocketAddr, path::Path, sync::Arc}; use tempfile::TempDir; use tokio::{ diff --git a/rust/noosphere-into/src/into/html/sphere.rs b/rust/noosphere-into/src/into/html/sphere.rs index c3b8752d2..86c6c7800 100644 --- a/rust/noosphere-into/src/into/html/sphere.rs +++ b/rust/noosphere-into/src/into/html/sphere.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeSet, io::Cursor, path::PathBuf, sync::Arc}; use anyhow::{anyhow, Result}; use cid::Cid; -use noosphere_sphere::{HasSphereContext, SphereContentRead, SphereCursor}; +use noosphere_core::context::{HasSphereContext, SphereContentRead, SphereCursor}; use noosphere_storage::Storage; use tokio::sync::Mutex; use tokio_stream::StreamExt; @@ -160,17 +160,16 @@ where pub mod tests { use std::path::PathBuf; - use noosphere_core::data::{ContentType, Header}; - #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::wasm_bindgen_test; wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); use crate::write::MemoryWriteTarget; - use noosphere_sphere::{ + use noosphere_core::{ + context::{HasMutableSphereContext, SphereContentWrite, SphereCursor}, + data::{ContentType, Header}, helpers::{simulated_sphere_context, SimulationAccess}, - HasMutableSphereContext, SphereContentWrite, SphereCursor, }; use super::sphere_into_html; diff --git a/rust/noosphere-into/src/transcluder/content.rs b/rust/noosphere-into/src/transcluder/content.rs index 8249c7fbf..7ea9a75d9 100644 --- a/rust/noosphere-into/src/transcluder/content.rs +++ b/rust/noosphere-into/src/transcluder/content.rs @@ -3,8 +3,8 @@ use std::marker::PhantomData; use crate::{ResolvedLink, TextTransclude, Transclude, Transcluder}; use anyhow::Result; use async_trait::async_trait; +use noosphere_core::context::{HasSphereContext, SphereContentRead}; use noosphere_core::data::Header; -use noosphere_sphere::{HasSphereContext, SphereContentRead}; use noosphere_storage::Storage; use subtext::{block::Block, primitive::Entity, Peer}; use tokio_stream::StreamExt; diff --git a/rust/noosphere-into/src/transform/file.rs b/rust/noosphere-into/src/transform/file.rs index 85b5068e5..26f1834ab 100644 --- a/rust/noosphere-into/src/transform/file.rs +++ b/rust/noosphere-into/src/transform/file.rs @@ -1,7 +1,7 @@ use async_stream::stream; use futures::Stream; +use noosphere_core::context::SphereFile; use noosphere_core::data::ContentType; -use noosphere_sphere::SphereFile; use tokio::io::AsyncRead; use crate::{subtext_to_html_document_stream, subtext_to_html_fragment_stream, Transform}; diff --git a/rust/noosphere-into/src/transform/sphere/html.rs b/rust/noosphere-into/src/transform/sphere/html.rs index 581845b9d..dc35bd7fb 100644 --- a/rust/noosphere-into/src/transform/sphere/html.rs +++ b/rust/noosphere-into/src/transform/sphere/html.rs @@ -1,9 +1,9 @@ use crate::{html_document_envelope, subtext_to_html_fragment_stream, Transform, TransformStream}; use async_stream::stream; use futures::Stream; +use noosphere_core::context::{HasSphereContext, SphereFile}; use noosphere_core::data::{ContentType, Header}; use noosphere_core::view::Sphere; -use noosphere_sphere::{HasSphereContext, SphereFile}; use noosphere_storage::{BlockStore, Storage}; /// Given a [Transform] and a [Sphere], produce a stream that yields the file diff --git a/rust/noosphere-into/src/transform/subtext/html/document.rs b/rust/noosphere-into/src/transform/subtext/html/document.rs index 3e2787858..c8895d964 100644 --- a/rust/noosphere-into/src/transform/subtext/html/document.rs +++ b/rust/noosphere-into/src/transform/subtext/html/document.rs @@ -1,7 +1,7 @@ use crate::{html_document_envelope, subtext_to_html_fragment_stream, Transform}; use async_stream::stream; use futures::Stream; -use noosphere_sphere::SphereFile; +use noosphere_core::context::SphereFile; use tokio::io::AsyncRead; /// Given a [Transform] and a [SphereFile], produce a stream that yields the diff --git a/rust/noosphere-into/src/transform/subtext/html/fragment.rs b/rust/noosphere-into/src/transform/subtext/html/fragment.rs index c73651225..8fc8b9405 100644 --- a/rust/noosphere-into/src/transform/subtext/html/fragment.rs +++ b/rust/noosphere-into/src/transform/subtext/html/fragment.rs @@ -6,8 +6,8 @@ use async_stream::stream; use futures::Stream; use horrorshow::{html, Raw}; +use noosphere_core::context::SphereFile; use noosphere_core::data::ContentType; -use noosphere_sphere::SphereFile; use subtext::{block::Block, primitive::Entity, Slashlink}; use tokio::io::AsyncRead; diff --git a/rust/noosphere-into/src/transform/transform_implementation.rs b/rust/noosphere-into/src/transform/transform_implementation.rs index 44fb81e87..e1b78227b 100644 --- a/rust/noosphere-into/src/transform/transform_implementation.rs +++ b/rust/noosphere-into/src/transform/transform_implementation.rs @@ -1,4 +1,4 @@ -use noosphere_sphere::HasSphereContext; +use noosphere_core::context::HasSphereContext; use noosphere_storage::Storage; use crate::{Resolver, SphereContentTranscluder, StaticHtmlResolver, Transcluder}; diff --git a/rust/noosphere-ipfs/Cargo.toml b/rust/noosphere-ipfs/Cargo.toml index 3c9f44b22..26ddda0da 100644 --- a/rust/noosphere-ipfs/Cargo.toml +++ b/rust/noosphere-ipfs/Cargo.toml @@ -22,7 +22,7 @@ readme = "README.md" [features] default = ["storage"] storage = ["ucan"] -test_kubo = [] +test-kubo = [] [dependencies] anyhow = { workspace = true } @@ -40,6 +40,7 @@ tokio-stream = { workspace = true } tracing = { workspace = true } url = { workspace = true, features = [ "serde" ] } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } +noosphere-common = { version = "0.1.0", path = "../noosphere-common" } ucan = { workspace = true, optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] @@ -48,8 +49,6 @@ hyper-multipart-rfc7578 = "~0.8" ipfs-api-prelude = "0.6" [dev-dependencies] - -[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] rand = { workspace = true } iroh-car = { workspace = true } libipld-cbor = { workspace = true } diff --git a/rust/noosphere-ipfs/src/client/kubo.rs b/rust/noosphere-ipfs/src/client/kubo.rs index 41b52c584..0979cbbed 100644 --- a/rust/noosphere-ipfs/src/client/kubo.rs +++ b/rust/noosphere-ipfs/src/client/kubo.rs @@ -163,7 +163,7 @@ impl IpfsClient for KuboClient { // Note that these tests require that there is a locally available IPFS Kubo // node running with the RPC API enabled -#[cfg(all(test, feature = "test_kubo"))] +#[cfg(all(test, feature = "test-kubo"))] mod tests { use std::io::Cursor; diff --git a/rust/noosphere-ipfs/src/storage.rs b/rust/noosphere-ipfs/src/storage.rs index 67b785c76..f088eb845 100644 --- a/rust/noosphere-ipfs/src/storage.rs +++ b/rust/noosphere-ipfs/src/storage.rs @@ -2,6 +2,7 @@ use crate::IpfsClient; use anyhow::Result; use async_trait::async_trait; use cid::Cid; +use noosphere_common::ConditionalSync; use noosphere_storage::{BlockStore, Storage}; use std::sync::Arc; use tokio::sync::RwLock; @@ -36,24 +37,24 @@ where } } -#[cfg(not(target_arch = "wasm32"))] -pub trait IpfsStorageConditionalSendSync: Send + Sync {} +// #[cfg(not(target_arch = "wasm32"))] +// pub trait IpfsStorageConditionalSendSync: Send + Sync {} -#[cfg(not(target_arch = "wasm32"))] -impl IpfsStorageConditionalSendSync for S where S: Send + Sync {} +// #[cfg(not(target_arch = "wasm32"))] +// impl IpfsStorageConditionalSendSync for S where S: Send + Sync {} -#[cfg(target_arch = "wasm32")] -pub trait IpfsStorageConditionalSendSync {} +// #[cfg(target_arch = "wasm32")] +// pub trait IpfsStorageConditionalSendSync {} -#[cfg(target_arch = "wasm32")] -impl IpfsStorageConditionalSendSync for S {} +// #[cfg(target_arch = "wasm32")] +// impl IpfsStorageConditionalSendSync for S {} #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storage for IpfsStorage where - S: Storage + IpfsStorageConditionalSendSync, - C: IpfsClient + IpfsStorageConditionalSendSync, + S: Storage + ConditionalSync, + C: IpfsClient + ConditionalSync, { type BlockStore = IpfsStore; @@ -79,7 +80,7 @@ where pub struct IpfsStore where B: BlockStore, - C: IpfsClient + IpfsStorageConditionalSendSync, + C: IpfsClient + ConditionalSync, { local_store: Arc>, ipfs_client: Option, @@ -88,7 +89,7 @@ where impl IpfsStore where B: BlockStore, - C: IpfsClient + IpfsStorageConditionalSendSync, + C: IpfsClient + ConditionalSync, { pub fn new(block_store: B, ipfs_client: Option) -> Self { IpfsStore { @@ -103,7 +104,7 @@ where impl BlockStore for IpfsStore where B: BlockStore, - C: IpfsClient + IpfsStorageConditionalSendSync, + C: IpfsClient + ConditionalSync, { #[instrument(skip(self), level = "trace")] async fn put_block(&mut self, cid: &Cid, block: &[u8]) -> Result<()> { @@ -141,7 +142,7 @@ where // Note that these tests require that there is a locally available IPFS Kubo // node running with the RPC API enabled -#[cfg(all(test, feature = "test_kubo"))] +#[cfg(all(test, feature = "test-kubo", not(target_arch = "wasm32")))] mod tests { use std::time::Duration; diff --git a/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs b/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs index 7846f6906..b5a632fd7 100644 --- a/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs +++ b/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs @@ -263,7 +263,7 @@ pub mod test { assert_eq!(&socket.to_string(), expectation); } - for failure_addr in vec![ + for failure_addr in [ CLIAddress::Url("http://127.0.0.1:6666".parse()?), CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?), ] { @@ -303,7 +303,8 @@ pub mod test { ); } - for failure_addr in vec![CLIAddress::Url("http://127.0.0.1:6666".parse()?)] { + { + let failure_addr = CLIAddress::Url("http://127.0.0.1:6666".parse()?); let result: Result = failure_addr.try_into(); assert!(result.is_err()); } @@ -329,7 +330,8 @@ pub mod test { assert_eq!(&url.to_string(), expectation); } - for failure_addr in vec![CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?)] { + { + let failure_addr = CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?); let result: Result = failure_addr.try_into(); assert!(result.is_err()); } diff --git a/rust/noosphere-sphere/src/cursor.rs b/rust/noosphere-sphere/src/cursor.rs index 3cb6f61c0..eb2cbbf77 100644 --- a/rust/noosphere-sphere/src/cursor.rs +++ b/rust/noosphere-sphere/src/cursor.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use noosphere_api::data::ReplicateParameters; use noosphere_core::{ data::{Link, MemoIpld}, + stream::put_block_stream, view::{Sphere, Timeline}, }; use noosphere_storage::Storage; @@ -192,14 +193,14 @@ where let replicate_parameters = since.as_ref().map(|since| ReplicateParameters { since: Some(since.clone()), }); - let (mut db, client) = { + let (db, client) = { let sphere_context = cursor.sphere_context().await?; (sphere_context.db().clone(), sphere_context.client().await?) }; let stream = client .replicate(&version, replicate_parameters.as_ref()) .await?; - db.put_block_stream(stream).await?; + put_block_stream(db.clone(), stream).await?; // If this was incremental replication, we have to hydrate... if let Some(since) = since { diff --git a/rust/noosphere-sphere/src/internal.rs b/rust/noosphere-sphere/src/internal.rs index d6913ed17..b736381f5 100644 --- a/rust/noosphere-sphere/src/internal.rs +++ b/rust/noosphere-sphere/src/internal.rs @@ -10,6 +10,7 @@ use cid::Cid; use noosphere_core::{ authority::Access, data::{ContentType, Header, Link, MemoIpld}, + stream::put_block_stream, }; /// A module-private trait for internal trait methods; this is a workaround for @@ -54,7 +55,7 @@ where sphere_revision: &Cid, memo_link: Link, ) -> Result>> { - let mut db = self.sphere_context().await?.db().clone(); + let db = self.sphere_context().await?.db().clone(); let memo = memo_link.load_from(&db).await?; // If we have a memo, but not the content it refers to, we should try to @@ -76,7 +77,7 @@ where // into the local DB let stream = client.replicate(&memo_link, None).await?; - db.put_block_stream(stream).await?; + put_block_stream(db.clone(), stream).await?; } let content_type = match memo.get_first_header(&Header::ContentType) { diff --git a/rust/noosphere-sphere/src/lib.rs b/rust/noosphere-sphere/src/lib.rs index 680dfa861..6a0bf6dd7 100644 --- a/rust/noosphere-sphere/src/lib.rs +++ b/rust/noosphere-sphere/src/lib.rs @@ -11,43 +11,52 @@ //! traversing the Noosphere content graph. //! //! ```rust -//! use anyhow::Result; -//! use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess}; -//! -//! use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SphereContentWrite}; -//! -//! #[tokio::main(flavor = "multi_thread")] -//! async fn main() -> Result<()> { -//! let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; -//! -//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?; -//! sphere_context.save(None).await?; -//! -//! Ok(()) -//! } +//! # use anyhow::Result; +//! # #[cfg(feature = "helpers")] +//! # use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess}; +//! # +//! # use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SphereContentWrite}; +//! # +//! # #[cfg(feature = "helpers")] +//! # #[tokio::main(flavor = "multi_thread")] +//! # async fn main() -> Result<()> { +//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # +//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?; +//! sphere_context.save(None).await?; +//! # +//! # Ok(()) +//! # } +//! # +//! # #[cfg(not(feature = "helpers"))] +//! # fn main() {} //! ``` //! //! You can also use a [SphereContext] to access petnames in the sphere: //! //! ```rust -//! use anyhow::Result; -//! use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess}; -//! use noosphere_core::data::Did; -//! -//! use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite}; -//! -//! #[tokio::main(flavor = "multi_thread")] -//! async fn main() -> Result<()> { -//! let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; -//! -//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?; -//! sphere_context.save(None).await?; -//! -//! Ok(()) -//! } +//! # use anyhow::Result; +//! # #[cfg(feature = "helpers")] +//! # use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess}; +//! # use noosphere_core::data::Did; +//! # +//! # use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite}; +//! # +//! # #[cfg(feature = "helpers")] +//! # #[tokio::main(flavor = "multi_thread")] +//! # async fn main() -> Result<()> { +//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; +//! # +//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?; +//! sphere_context.save(None).await?; +//! # +//! # Ok(()) +//! # } +//! # +//! # #[cfg(not(feature = "helpers"))] +//! # fn main() {} //! ``` //! -//! #![warn(missing_docs)] @@ -71,7 +80,7 @@ mod has; mod replication; mod walker; -#[cfg(any(test, feature = "helpers"))] +#[cfg(any(doctest, test, feature = "helpers"))] pub mod helpers; mod internal; diff --git a/rust/noosphere-sphere/src/replication/stream.rs b/rust/noosphere-sphere/src/replication/stream.rs index 6150afbb7..c3227e085 100644 --- a/rust/noosphere-sphere/src/replication/stream.rs +++ b/rust/noosphere-sphere/src/replication/stream.rs @@ -504,7 +504,7 @@ mod tests { simulated_sphere_context(SimulationAccess::ReadWrite, None).await?; let mut db = sphere_context.sphere_context().await?.db_mut().clone(); - let chunks = vec![b"foo", b"bar", b"baz"]; + let chunks = [b"foo", b"bar", b"baz"]; let mut next_chunk_cid = None; diff --git a/rust/noosphere-sphere/src/sync/strategy.rs b/rust/noosphere-sphere/src/sync/strategy.rs index 115816b3a..b04ac3f9a 100644 --- a/rust/noosphere-sphere/src/sync/strategy.rs +++ b/rust/noosphere-sphere/src/sync/strategy.rs @@ -5,6 +5,7 @@ use noosphere_api::data::{FetchParameters, PushBody, PushResponse}; use noosphere_core::{ authority::{generate_capability, SphereAbility}, data::{Did, IdentityIpld, Jwt, Link, MemoIpld, LINK_RECORD_FACT_NAME}, + stream::put_block_stream, view::{Sphere, Timeline}, }; use noosphere_storage::{KeyValueStore, SphereDb, Storage}; @@ -178,7 +179,7 @@ where } }; - context.db_mut().put_block_stream(block_stream).await?; + put_block_stream(context.db_mut().clone(), block_stream).await?; trace!("Finished putting block stream"); diff --git a/rust/noosphere-storage/Cargo.toml b/rust/noosphere-storage/Cargo.toml index b3ad41872..1a6042bdf 100644 --- a/rust/noosphere-storage/Cargo.toml +++ b/rust/noosphere-storage/Cargo.toml @@ -17,14 +17,13 @@ repository = "https://github.com/subconsciousnetwork/noosphere" homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = { workspace = true } async-trait = "~0.1" async-stream = { workspace = true } tokio-stream = { workspace = true } cid = { workspace = true } +noosphere-common = { version = "0.1.0", path = "../noosphere-common" } tracing = "~0.1" ucan = { workspace = true } libipld-core = { workspace = true } diff --git a/rust/noosphere-storage/src/block.rs b/rust/noosphere-storage/src/block.rs index 26aa0a2ac..22cc7180c 100644 --- a/rust/noosphere-storage/src/block.rs +++ b/rust/noosphere-storage/src/block.rs @@ -14,35 +14,12 @@ use libipld_core::{ codec::{Codec, Decode, Encode}, ipld::Ipld, }; +use noosphere_common::{ConditionalSend, ConditionalSync}; use serde::{de::DeserializeOwned, Serialize}; #[cfg(doc)] use serde::Deserialize; -#[cfg(not(target_arch = "wasm32"))] -pub trait BlockStoreSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl BlockStoreSendSync for T where T: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait BlockStoreSendSync {} - -#[cfg(target_arch = "wasm32")] -impl BlockStoreSendSync for T {} - -#[cfg(not(target_arch = "wasm32"))] -pub trait BlockStoreSend: Send {} - -#[cfg(not(target_arch = "wasm32"))] -impl BlockStoreSend for T where T: Send {} - -#[cfg(target_arch = "wasm32")] -pub trait BlockStoreSend {} - -#[cfg(target_arch = "wasm32")] -impl BlockStoreSend for T {} - /// An interface for storage backends that are suitable for storing blocks. A /// block is a chunk of bytes that can be addressed by a /// [CID](https://docs.ipfs.tech/concepts/content-addressing/#identifier-formats). @@ -50,7 +27,7 @@ impl BlockStoreSend for T {} /// retrieve blocks given a [Cid]. #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait BlockStore: Clone + BlockStoreSendSync { +pub trait BlockStore: Clone + ConditionalSync { /// Given a CID and a block, store the links (any [Cid] that is part of the /// encoded data) in a suitable location for later retrieval. This method is /// optional, and its default implementation is a no-op. It should be @@ -77,7 +54,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync { async fn put(&mut self, data: T) -> Result where C: Codec + Default, - T: Encode + BlockStoreSend, + T: Encode + ConditionalSend, Ipld: References, { let codec = C::default(); @@ -114,7 +91,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync { async fn save(&mut self, data: T) -> Result where C: Codec + Default, - T: Serialize + BlockStoreSend, + T: Serialize + ConditionalSend, Ipld: Encode + References, { self.put::(to_ipld(data)?).await @@ -127,7 +104,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync { async fn load(&self, cid: &Cid) -> Result where C: Codec + Default, - T: DeserializeOwned + BlockStoreSend, + T: DeserializeOwned + ConditionalSend, u64: From, Ipld: Decode, { diff --git a/rust/noosphere-storage/src/db.rs b/rust/noosphere-storage/src/db.rs index 09660d07d..e54501032 100644 --- a/rust/noosphere-storage/src/db.rs +++ b/rust/noosphere-storage/src/db.rs @@ -7,28 +7,17 @@ use libipld_core::{ ipld::Ipld, raw::RawCodec, }; +use noosphere_common::ConditionalSend; use serde::{de::DeserializeOwned, Serialize}; use std::future::Future; use std::{collections::BTreeSet, fmt::Debug}; -use tokio_stream::{Stream, StreamExt}; -use ucan::store::{UcanStore, UcanStoreConditionalSend}; +use tokio_stream::Stream; +use ucan::store::UcanStore; -use crate::{BlockStore, BlockStoreSend, KeyValueStore, MemoryStore, Storage}; +use crate::{BlockStore, KeyValueStore, MemoryStore, Storage}; use async_stream::try_stream; -#[cfg(not(target_arch = "wasm32"))] -pub trait SphereDbSendSync: Send + Sync {} - -#[cfg(not(target_arch = "wasm32"))] -impl SphereDbSendSync for T where T: Send + Sync {} - -#[cfg(target_arch = "wasm32")] -pub trait SphereDbSendSync {} - -#[cfg(target_arch = "wasm32")] -impl SphereDbSendSync for T {} - pub const BLOCK_STORE: &str = "blocks"; pub const LINK_STORE: &str = "links"; pub const VERSION_STORE: &str = "versions"; @@ -203,36 +192,6 @@ where } } - pub async fn put_block_stream(&mut self, stream: Str) -> Result<()> - where - Str: Stream)>>, - { - tokio::pin!(stream); - - let mut stream_count = 0usize; - - while let Some((cid, block)) = stream.try_next().await? { - stream_count += 1; - trace!(?cid, "Putting streamed block {stream_count}..."); - - self.put_block(&cid, &block).await?; - - match cid.codec() { - codec_id if codec_id == u64::from(DagCborCodec) => { - self.put_links::(&cid, &block).await?; - } - codec_id if codec_id == u64::from(RawCodec) => { - self.put_links::(&cid, &block).await?; - } - codec_id => warn!("Unrecognized codec {}; skipping...", codec_id), - } - } - - trace!("Loaded {stream_count} blocks from stream..."); - - Ok(()) - } - /// Get an owned copy of the underlying primitive [BlockStore] for this /// [SphereDb] pub fn to_block_store(&self) -> S::BlockStore { @@ -278,23 +237,23 @@ where { async fn set_key(&mut self, key: K, value: V) -> Result<()> where - K: AsRef<[u8]> + BlockStoreSend, - V: Serialize + BlockStoreSend, + K: AsRef<[u8]> + ConditionalSend, + V: Serialize + ConditionalSend, { self.metadata_store.set_key(key, value).await } async fn unset_key(&mut self, key: K) -> Result<()> where - K: AsRef<[u8]> + BlockStoreSend, + K: AsRef<[u8]> + ConditionalSend, { self.metadata_store.unset_key(key).await } async fn get_key(&self, key: K) -> Result> where - K: AsRef<[u8]> + BlockStoreSend, - V: DeserializeOwned + BlockStoreSend, + K: AsRef<[u8]> + ConditionalSend, + V: DeserializeOwned + ConditionalSend, { self.metadata_store.get_key(key).await } @@ -310,7 +269,7 @@ where self.get::(cid).await } - async fn write + UcanStoreConditionalSend + Debug>( + async fn write + ConditionalSend + Debug>( &mut self, token: T, ) -> Result { diff --git a/rust/noosphere/Cargo.toml b/rust/noosphere/Cargo.toml index 5d0227fdc..d594a8420 100644 --- a/rust/noosphere/Cargo.toml +++ b/rust/noosphere/Cargo.toml @@ -19,6 +19,7 @@ crate-type = ["rlib", "staticlib", "cdylib"] default = [] headers = ["safer-ffi/headers"] ipfs-storage = ["noosphere-ipfs"] +test-kubo = [] [dependencies] anyhow = { workspace = true } @@ -39,15 +40,20 @@ libipld-cbor = { workspace = true } bytes = "^1" noosphere-core = { version = "0.15.2", path = "../noosphere-core" } -noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" } noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" } -noosphere-api = { version = "0.12.2", path = "../noosphere-api" } noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs", optional = true } ucan = { workspace = true } ucan-key-support = { workspace = true } [dev-dependencies] libipld-core = { workspace = true } +rand = { workspace = true } +serde_json = { workspace = true } +noosphere-cli = { version = "0.14.1", path = "../noosphere-cli", features = ["helpers"] } +noosphere-core = { version = "0.15.1", path = "../noosphere-core", features = ["helpers"] } +noosphere-common = { version = "0.1.0", path = "../noosphere-common", features = ["helpers"] } +noosphere-gateway = { version = "0.8.1", path = "../noosphere-gateway" } +noosphere-ns = { version = "0.10.1", path = "../noosphere-ns" } [target.'cfg(target_arch = "wasm32")'.dependencies] # TODO: We should eventually support gateway storage as a specialty target only, @@ -60,7 +66,7 @@ js-sys = { workspace = true } noosphere-into = { version = "0.10.6", path = "../noosphere-into" } [target.'cfg(target_arch = "wasm32")'.dependencies.web-sys] -version = "~0.3" +workspace = true features = [ "CryptoKey", ] @@ -71,6 +77,7 @@ tokio = { workspace = true, features = ["full"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tempfile = { workspace = true } +reqwest = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = { workspace = true } diff --git a/rust/noosphere/src/ffi/authority.rs b/rust/noosphere/src/ffi/authority.rs index dca3400ad..7c1d0a14c 100644 --- a/rust/noosphere/src/ffi/authority.rs +++ b/rust/noosphere/src/ffi/authority.rs @@ -1,11 +1,13 @@ use anyhow::anyhow; use cid::Cid; +use noosphere_core::context::{ + HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker, +}; use noosphere_core::{ authority::Authorization, data::{Did, Jwt, Link, Mnemonic}, error::NoosphereError, }; -use noosphere_sphere::{HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker}; use safer_ffi::{char_p::InvalidNulTerminator, prelude::*}; use std::ffi::c_void; diff --git a/rust/noosphere/src/ffi/context.rs b/rust/noosphere/src/ffi/context.rs index 524d51d29..dd756dad9 100644 --- a/rust/noosphere/src/ffi/context.rs +++ b/rust/noosphere/src/ffi/context.rs @@ -15,7 +15,7 @@ use crate::{ platform::{PlatformSphereChannel, PlatformStorage}, }; -use noosphere_sphere::{ +use noosphere_core::context::{ AsyncFileBody, HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, SphereContext, SphereCursor, SphereFile, SphereReplicaRead, SphereWalker, }; diff --git a/rust/noosphere/src/ffi/petname.rs b/rust/noosphere/src/ffi/petname.rs index 5cc73ead1..a13c6001e 100644 --- a/rust/noosphere/src/ffi/petname.rs +++ b/rust/noosphere/src/ffi/petname.rs @@ -5,8 +5,8 @@ use std::{ffi::c_void, str::FromStr}; use anyhow::anyhow; use cid::Cid; +use noosphere_core::context::{SpherePetnameRead, SpherePetnameWrite, SphereWalker}; use noosphere_core::data::Did; -use noosphere_sphere::{SpherePetnameRead, SpherePetnameWrite, SphereWalker}; use safer_ffi::{char_p::InvalidNulTerminator, prelude::*}; use crate::ffi::{NsError, TryOrInitialize}; diff --git a/rust/noosphere/src/ffi/sphere.rs b/rust/noosphere/src/ffi/sphere.rs index b8f6a5dfa..32a3224a1 100644 --- a/rust/noosphere/src/ffi/sphere.rs +++ b/rust/noosphere/src/ffi/sphere.rs @@ -2,8 +2,8 @@ use std::ffi::c_void; use anyhow::anyhow; use cid::Cid; +use noosphere_core::context::{HasSphereContext, SphereSync, SyncRecovery}; use noosphere_core::{authority::Authorization, data::Did}; -use noosphere_sphere::{HasSphereContext, SphereSync, SyncRecovery}; use safer_ffi::char_p::InvalidNulTerminator; use safer_ffi::prelude::*; diff --git a/rust/noosphere/src/noosphere.rs b/rust/noosphere/src/noosphere.rs index 6541330f5..7759a984b 100644 --- a/rust/noosphere/src/noosphere.rs +++ b/rust/noosphere/src/noosphere.rs @@ -4,7 +4,7 @@ use anyhow::{anyhow, Result}; use noosphere_core::{authority::Authorization, data::Did}; use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; -use noosphere_sphere::{SphereContext, SphereCursor}; +use noosphere_core::context::{SphereContext, SphereCursor}; use tokio::sync::Mutex; use url::Url; diff --git a/rust/noosphere/src/platform.rs b/rust/noosphere/src/platform.rs index 3b1fa0839..71353adfa 100644 --- a/rust/noosphere/src/platform.rs +++ b/rust/noosphere/src/platform.rs @@ -146,7 +146,7 @@ mod inner { use std::sync::Arc; pub use inner::*; -use noosphere_sphere::{SphereContext, SphereCursor}; +use noosphere_core::context::{SphereContext, SphereCursor}; use tokio::sync::Mutex; use crate::sphere::SphereChannel; diff --git a/rust/noosphere/src/sphere/builder.rs b/rust/noosphere/src/sphere/builder.rs index b76dea9a4..ce401d0ed 100644 --- a/rust/noosphere/src/sphere/builder.rs +++ b/rust/noosphere/src/sphere/builder.rs @@ -19,7 +19,7 @@ use noosphere_storage::{KeyValueStore, MemoryStore, SphereDb}; use ucan::crypto::KeyMaterial; use url::Url; -use noosphere_sphere::{ +use noosphere_core::context::{ metadata::{AUTHORIZATION, IDENTITY, USER_KEY_NAME}, SphereContext, SphereContextKey, }; @@ -365,7 +365,7 @@ mod tests { use wasm_bindgen_test::wasm_bindgen_test; use crate::{key::KeyStorage, platform::make_temporary_platform_primitives}; - use noosphere_sphere::SphereContext; + use noosphere_core::context::SphereContext; #[cfg(target_arch = "wasm32")] wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); @@ -373,7 +373,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_create_a_sphere_and_later_open_it() { - let (storage_path, key_storage, temporary_directories) = + let (storage_path, key_storage, _temporary_directories) = make_temporary_platform_primitives().await.unwrap(); key_storage.create_key("foo").await.unwrap(); @@ -404,14 +404,12 @@ mod tests { .into(); assert_eq!(&sphere_identity, context.identity()); - - drop(temporary_directories); } #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_create_a_scoped_sphere_and_later_open_it() { - let (storage_path, key_storage, temporary_directories) = + let (storage_path, key_storage, _temporary_directories) = make_temporary_platform_primitives().await.unwrap(); key_storage.create_key("foo").await.unwrap(); @@ -444,14 +442,12 @@ mod tests { .into(); assert_eq!(&sphere_identity, context.identity()); - - drop(temporary_directories); } #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn it_can_initialize_a_sphere_to_sync_from_elsewhere() { - let (storage_path, key_storage, temporary_directories) = + let (storage_path, key_storage, _temporary_directories) = make_temporary_platform_primitives().await.unwrap(); key_storage.create_key("foo").await.unwrap(); @@ -469,7 +465,5 @@ mod tests { let context: SphereContext<_> = artifacts.into(); assert_eq!(context.identity().as_str(), "did:key:foo"); - - drop(temporary_directories); } } diff --git a/rust/noosphere/src/sphere/channel.rs b/rust/noosphere/src/sphere/channel.rs index 894ae4b34..64a2a57cf 100644 --- a/rust/noosphere/src/sphere/channel.rs +++ b/rust/noosphere/src/sphere/channel.rs @@ -1,6 +1,8 @@ use std::{marker::PhantomData, sync::Arc}; -use noosphere_sphere::{HasMutableSphereContext, HasSphereContext, SphereContext, SphereCursor}; +use noosphere_core::context::{ + HasMutableSphereContext, HasSphereContext, SphereContext, SphereCursor, +}; use noosphere_storage::Storage; use tokio::sync::Mutex; @@ -14,7 +16,7 @@ use tokio::sync::Mutex; #[derive(Clone)] pub struct SphereChannel where - S: Storage, + S: Storage + 'static, Ci: HasSphereContext, Cm: HasMutableSphereContext, { @@ -25,7 +27,7 @@ where impl SphereChannel where - S: Storage, + S: Storage + 'static, Ci: HasSphereContext, Cm: HasMutableSphereContext, { diff --git a/rust/noosphere/src/wasm/file.rs b/rust/noosphere/src/wasm/file.rs index 9ff4dfcd3..1d49459bb 100644 --- a/rust/noosphere/src/wasm/file.rs +++ b/rust/noosphere/src/wasm/file.rs @@ -9,7 +9,7 @@ use tokio_stream::StreamExt; use anyhow::{anyhow, Result}; use js_sys::{Function, Promise, Uint8Array}; -use noosphere_sphere::{ +use noosphere_core::context::{ AsyncFileBody, HasSphereContext, SphereContext, SphereCursor, SphereFile as SphereFileImpl, }; use tokio::{io::AsyncReadExt, sync::Mutex}; diff --git a/rust/noosphere/src/wasm/fs.rs b/rust/noosphere/src/wasm/fs.rs index 7bb5c53bc..2f13585bb 100644 --- a/rust/noosphere/src/wasm/fs.rs +++ b/rust/noosphere/src/wasm/fs.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use crate::{platform::PlatformStorage, wasm::SphereFile}; use js_sys::{Array, Function}; -use noosphere_sphere::{ +use noosphere_core::context::{ HasMutableSphereContext, SphereContentRead, SphereContentWrite, SphereContext, SphereCursor, SphereWalker, }; diff --git a/rust/noosphere/src/wasm/sphere.rs b/rust/noosphere/src/wasm/sphere.rs index 04b3185c5..b05daa6be 100644 --- a/rust/noosphere/src/wasm/sphere.rs +++ b/rust/noosphere/src/wasm/sphere.rs @@ -2,7 +2,7 @@ use anyhow::Result; use cid::Cid; use crate::{platform::PlatformSphereChannel, wasm::SphereFs}; -use noosphere_sphere::SphereCursor; +use noosphere_core::context::SphereCursor; use wasm_bindgen::prelude::*; #[wasm_bindgen] diff --git a/rust/noosphere/tests/cli.rs b/rust/noosphere/tests/cli.rs new file mode 100644 index 000000000..55b912c94 --- /dev/null +++ b/rust/noosphere/tests/cli.rs @@ -0,0 +1,155 @@ +#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))] + +//! Integration tests to demonstrate that the Noosphere CLI, aka "orb", works +//! end-to-end in concert with the name system and backing block syndication + +use anyhow::Result; +use noosphere_cli::helpers::CliSimulator; +use noosphere_cli::paths::SPHERE_DIRECTORY; +use noosphere_common::helpers::wait; +use noosphere_core::tracing::initialize_tracing; +use serde_json::Value; + +#[tokio::test(flavor = "multi_thread")] +async fn orb_status_errors_on_empty_directory() -> Result<()> { + initialize_tracing(None); + let client = CliSimulator::new()?; + + assert!(client.orb(&["sphere", "status"]).await.is_err()); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn orb_sphere_create_initializes_a_sphere() -> Result<()> { + initialize_tracing(None); + let client = CliSimulator::new()?; + + client.orb(&["key", "create", "foobar"]).await?; + client + .orb(&["sphere", "create", "--owner-key", "foobar"]) + .await?; + + assert!(client.sphere_directory().join(SPHERE_DIRECTORY).exists()); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn orb_can_enable_multiple_replicas_to_synchronize() -> Result<()> { + initialize_tracing(None); + + let first_replica = CliSimulator::new()?; + let second_replica = CliSimulator::new()?; + + first_replica.orb(&["key", "create", "foo"]).await?; + first_replica + .orb(&["sphere", "create", "--owner-key", "foo"]) + .await?; + + let client_sphere_id = first_replica + .orb_with_output(&["sphere", "status", "--id"]) + .await? + .join("\n"); + + let gateway = CliSimulator::new()?; + + gateway.orb(&["key", "create", "gateway"]).await?; + + gateway + .orb(&["sphere", "create", "--owner-key", "gateway"]) + .await?; + + gateway + .orb(&["sphere", "config", "set", "counterpart", &client_sphere_id]) + .await?; + + let gateway_task = tokio::task::spawn(async move { gateway.orb(&["serve"]).await }); + + wait(1).await; + + first_replica + .orb(&[ + "sphere", + "config", + "set", + "gateway-url", + "http://127.0.0.1:4433", + ]) + .await?; + + second_replica.orb(&["key", "create", "bar"]).await?; + let second_replica_id = match serde_json::from_str( + &second_replica + .orb_with_output(&["key", "list", "--as-json"]) + .await? + .join("\n"), + )? { + Value::Object(keys) => keys.get("bar").unwrap().as_str().unwrap().to_owned(), + _ => panic!(), + }; + + tokio::fs::write( + first_replica.sphere_directory().join("foo.subtext"), + "foobar", + ) + .await?; + + first_replica.orb(&["sphere", "save"]).await?; + + first_replica + .orb(&["sphere", "auth", "add", &second_replica_id]) + .await?; + + let second_replica_auth = match serde_json::from_str( + &first_replica + .orb_with_output(&["sphere", "auth", "list", "--as-json"]) + .await? + .join("\n"), + )? { + Value::Array(auths) => match auths + .iter() + .filter(|auth| { + auth.as_object() + .unwrap() + .get("name") + .unwrap() + .as_str() + .unwrap() + != "(OWNER)" + }) + .take(1) + .next() + .unwrap() + { + Value::Object(auth) => auth.get("link").unwrap().as_str().unwrap().to_owned(), + _ => panic!(), + }, + _ => panic!(), + }; + + first_replica.orb(&["sphere", "sync"]).await?; + + second_replica + .orb(&[ + "sphere", + "join", + "--authorization", + &second_replica_auth, + "--local-key", + "bar", + "--gateway-url", + "http://127.0.0.1:4433", + &client_sphere_id, + ]) + .await?; + + let foo_contents = + tokio::fs::read_to_string(second_replica.sphere_directory().join("foo.subtext")).await?; + + assert_eq!(foo_contents.as_str(), "foobar"); + + gateway_task.abort(); + + Ok(()) +} diff --git a/rust/noosphere-cli/tests/peer_to_peer.rs b/rust/noosphere/tests/distributed_basic.rs similarity index 87% rename from rust/noosphere-cli/tests/peer_to_peer.rs rename to rust/noosphere/tests/distributed_basic.rs index 991e6b1e1..83efc3f35 100644 --- a/rust/noosphere-cli/tests/peer_to_peer.rs +++ b/rust/noosphere/tests/distributed_basic.rs @@ -1,28 +1,32 @@ -#![cfg(all(feature = "test_kubo", not(target_arch = "wasm32")))] +#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))] + +//! Integration tests that expect "full stack" Noosphere to be available, including +//! name system and block syndication backend (e.g., IPFS Kubo). The tests in this +//! module represent basic distributed system scenarios. #[macro_use] extern crate tracing; -mod helpers; use anyhow::Result; -use helpers::{start_name_system_server, wait, SpherePair}; -use noosphere_core::data::{ContentType, Did, Link, MemoIpld}; -use noosphere_core::tracing::initialize_tracing; -use noosphere_ns::{server::HttpClient, NameResolver}; -use noosphere_sphere::{ - HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, SphereCursor, - SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, SphereSync, SphereWalker, - SyncRecovery, +use noosphere_cli::helpers::{start_name_system_server, SpherePair}; +use noosphere_common::helpers::wait; +use noosphere_core::{ + context::{ + HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, + SphereCursor, SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, SphereSync, + SphereWalker, SyncRecovery, + }, + data::{ContentType, Did, Link, MemoIpld}, + stream::memo_history_stream, + tracing::initialize_tracing, }; -use rand::Rng; +use noosphere_ns::{server::HttpClient, NameResolver}; use std::collections::BTreeSet; use std::sync::Arc; use tokio::io::AsyncReadExt; use tokio_stream::StreamExt; use url::Url; -use crate::helpers::TestEntropy; - #[tokio::test] async fn gateway_publishes_and_resolves_petnames_configured_by_the_client() -> Result<()> { initialize_tracing(None); @@ -673,15 +677,12 @@ async fn local_lineage_remains_sparse_as_graph_changes_accrue_over_time() -> Res Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> { +#[tokio::test] +async fn all_of_client_history_is_made_manifest_on_the_gateway_after_sync() -> Result<()> { initialize_tracing(None); - let entropy = TestEntropy::default(); - let rng = entropy.to_rng(); - - let ipfs_url = Url::parse("http://127.0.0.1:5001").unwrap(); - let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await.unwrap(); + let ipfs_url = Url::parse("http://127.0.0.1:5001")?; + let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await?; let mut pair_1 = SpherePair::new("ONE", &ipfs_url, &ns_url).await?; let mut pair_2 = SpherePair::new("TWO", &ipfs_url, &ns_url).await?; @@ -689,95 +690,59 @@ async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> { pair_1.start_gateway().await?; pair_2.start_gateway().await?; - let peer_2_identity = pair_2.client.workspace.sphere_identity().await?; - let pair_2_rng = rng.clone(); - - pair_2 + let _ = pair_2 .spawn(|mut ctx| async move { - let mut rng = pair_2_rng.lock().await; - - // Long history, small-ish files - for _ in 0..1000 { - let random_index = rng.gen_range(0..100); - let mut random_bytes = Vec::from(rng.gen::<[u8; 32]>()); - let slug = format!("slug{}", random_index); - - let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { - let mut file_bytes = Vec::new(); - file.contents.read_to_end(&mut file_bytes).await?; - file_bytes.append(&mut random_bytes); - file_bytes - } else { - random_bytes - }; - - ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) - .await?; - ctx.save(None).await?; - } - + ctx.write("foo", &ContentType::Text, "bar".as_bytes(), None) + .await?; + ctx.save(None).await?; Ok(ctx.sync(SyncRecovery::Retry(3)).await?) }) .await?; - let pair_1_rng = rng.clone(); + let sphere_2_identity = pair_2.client.identity.clone(); - pair_1 - .spawn(|mut ctx| async move { - let mut rng = pair_1_rng.lock().await; - - // Modest history, large-ish files - for _ in 0..100 { - let mut random_bytes = (0..1000).fold(Vec::new(), |mut bytes, _| { - bytes.append(&mut Vec::from(rng.gen::<[u8; 32]>())); - bytes - }); - let random_index = rng.gen_range(0..10); - let slug = format!("slug{}", random_index); - - let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { - let mut file_bytes = Vec::new(); - file.contents.read_to_end(&mut file_bytes).await?; - file_bytes.append(&mut random_bytes); - file_bytes - } else { - random_bytes - }; - - ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) + let final_client_version = pair_1 + .spawn(move |mut ctx| async move { + for value in ["one", "two", "three"] { + ctx.write(value, &ContentType::Text, value.as_bytes(), None) .await?; - ctx.save(None).await?; } ctx.sync(SyncRecovery::Retry(3)).await?; - ctx.set_petname("peer2", Some(peer_2_identity)).await?; + ctx.set_petname("two", Some(sphere_2_identity)).await?; ctx.save(None).await?; ctx.sync(SyncRecovery::Retry(3)).await?; - // TODO(#606): Implement this part of the test when we "fix" latency asymmetry between - // name system and syndication workers. We should be able to test traversing to a peer - // after a huge update as been added to the name system. - /* - wait(1).await; + for value in ["one", "two", "three"] { + ctx.set_petname(value, Some(Did(format!("did:key:{}", value)))) + .await?; + ctx.save(None).await?; + } ctx.sync(SyncRecovery::Retry(3)).await?; wait(1).await; - let cursor = SphereCursor::latest(ctx); - let _peer2_ctx = cursor - .traverse_by_petnames(&["peer2".into()]) - .await? - .unwrap(); - */ - Ok(()) + Ok(ctx.sync(SyncRecovery::Retry(3)).await?) }) .await?; + // Stream all of the blocks of client history as represented in gateway's storage + let block_stream = memo_history_stream( + pair_1.gateway.workspace.db().await?, + &final_client_version, + None, + true, + ); + + tokio::pin!(block_stream); + + while let Some(_) = block_stream.try_next().await? {} + ns_task.abort(); Ok(()) diff --git a/rust/noosphere-cli/tests/cli.rs b/rust/noosphere/tests/distributed_stress.rs similarity index 67% rename from rust/noosphere-cli/tests/cli.rs rename to rust/noosphere/tests/distributed_stress.rs index 426c8972f..0eb4d3e13 100644 --- a/rust/noosphere-cli/tests/cli.rs +++ b/rust/noosphere/tests/distributed_stress.rs @@ -1,168 +1,146 @@ -#![cfg(not(target_arch = "wasm32"))] +#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))] -mod helpers; +//! Integration tests that expect "full stack" Noosphere to be available, including +//! name system and block syndication backend (e.g., IPFS Kubo). The tests in this +//! module represent sophisticated, complicated, nuanced or high-latency scenarios. -use anyhow::Result; -use helpers::CliSimulator; -use noosphere_cli::paths::SPHERE_DIRECTORY; -use noosphere_core::tracing::initialize_tracing; -use serde_json::Value; - -use crate::helpers::wait; - -#[tokio::test(flavor = "multi_thread")] -async fn orb_status_errors_on_empty_directory() -> Result<()> { - initialize_tracing(None); - let client = CliSimulator::new()?; - - assert!(client.orb(&["sphere", "status"]).await.is_err()); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn orb_sphere_create_initializes_a_sphere() -> Result<()> { - initialize_tracing(None); - let client = CliSimulator::new()?; - - client.orb(&["key", "create", "foobar"]).await?; - client - .orb(&["sphere", "create", "--owner-key", "foobar"]) - .await?; +mod latency { + use anyhow::Result; + use noosphere_cli::helpers::{start_name_system_server, SpherePair}; + use noosphere_common::helpers::TestEntropy; + use noosphere_core::{ + context::{ + HasMutableSphereContext, SphereContentRead, SphereContentWrite, SpherePetnameWrite, + SphereSync, SyncRecovery, + }, + data::ContentType, + tracing::initialize_tracing, + }; + use rand::Rng; + use tokio::io::AsyncReadExt; + use url::Url; - assert!(client.sphere_directory().join(SPHERE_DIRECTORY).exists()); + #[tokio::test(flavor = "multi_thread")] + async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> { + initialize_tracing(None); - Ok(()) -} + let entropy = TestEntropy::default(); + let rng = entropy.to_rng(); -#[tokio::test(flavor = "multi_thread")] -async fn orb_can_enable_multiple_replicas_to_synchronize() -> Result<()> { - initialize_tracing(None); + let ipfs_url = Url::parse("http://127.0.0.1:5001").unwrap(); + let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await.unwrap(); - let first_replica = CliSimulator::new()?; - let second_replica = CliSimulator::new()?; + let mut pair_1 = SpherePair::new("ONE", &ipfs_url, &ns_url).await?; + let mut pair_2 = SpherePair::new("TWO", &ipfs_url, &ns_url).await?; - first_replica.orb(&["key", "create", "foo"]).await?; - first_replica - .orb(&["sphere", "create", "--owner-key", "foo"]) - .await?; + pair_1.start_gateway().await?; + pair_2.start_gateway().await?; - let client_sphere_id = first_replica - .orb_with_output(&["sphere", "status", "--id"]) - .await? - .join("\n"); + let peer_2_identity = pair_2.client.workspace.sphere_identity().await?; + let pair_2_rng = rng.clone(); - let gateway = CliSimulator::new()?; + pair_2 + .spawn(|mut ctx| async move { + let mut rng = pair_2_rng.lock().await; + + // Long history, small-ish files + for _ in 0..1000 { + let random_index = rng.gen_range(0..100); + let mut random_bytes = Vec::from(rng.gen::<[u8; 32]>()); + let slug = format!("slug{}", random_index); + + let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { + let mut file_bytes = Vec::new(); + file.contents.read_to_end(&mut file_bytes).await?; + file_bytes.append(&mut random_bytes); + file_bytes + } else { + random_bytes + }; + + ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) + .await?; + ctx.save(None).await?; + } - gateway.orb(&["key", "create", "gateway"]).await?; + Ok(ctx.sync(SyncRecovery::Retry(3)).await?) + }) + .await?; - gateway - .orb(&["sphere", "create", "--owner-key", "gateway"]) - .await?; + let pair_1_rng = rng.clone(); + + pair_1 + .spawn(|mut ctx| async move { + let mut rng = pair_1_rng.lock().await; + + // Modest history, large-ish files + for _ in 0..100 { + let mut random_bytes = (0..1000).fold(Vec::new(), |mut bytes, _| { + bytes.append(&mut Vec::from(rng.gen::<[u8; 32]>())); + bytes + }); + let random_index = rng.gen_range(0..10); + let slug = format!("slug{}", random_index); + + let next_bytes = if let Some(mut file) = ctx.read(&slug).await? { + let mut file_bytes = Vec::new(); + file.contents.read_to_end(&mut file_bytes).await?; + file_bytes.append(&mut random_bytes); + file_bytes + } else { + random_bytes + }; + + ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None) + .await?; - gateway - .orb(&["sphere", "config", "set", "counterpart", &client_sphere_id]) - .await?; + ctx.save(None).await?; + } - let gateway_task = tokio::task::spawn(async move { gateway.orb(&["serve"]).await }); + ctx.sync(SyncRecovery::Retry(3)).await?; - wait(1).await; + ctx.set_petname("peer2", Some(peer_2_identity)).await?; - first_replica - .orb(&[ - "sphere", - "config", - "set", - "gateway-url", - "http://127.0.0.1:4433", - ]) - .await?; + ctx.save(None).await?; - second_replica.orb(&["key", "create", "bar"]).await?; - let second_replica_id = match serde_json::from_str( - &second_replica - .orb_with_output(&["key", "list", "--as-json"]) - .await? - .join("\n"), - )? { - Value::Object(keys) => keys.get("bar").unwrap().as_str().unwrap().to_owned(), - _ => panic!(), - }; + ctx.sync(SyncRecovery::Retry(3)).await?; - tokio::fs::write( - first_replica.sphere_directory().join("foo.subtext"), - "foobar", - ) - .await?; + // TODO(#606): Implement this part of the test when we "fix" latency asymmetry between + // name system and syndication workers. We should be able to test traversing to a peer + // after a huge update as been added to the name system. + // wait(1).await; - first_replica.orb(&["sphere", "save"]).await?; + // ctx.sync(SyncRecovery::Retry(3)).await?; - first_replica - .orb(&["sphere", "auth", "add", &second_replica_id]) - .await?; + // wait(1).await; - let second_replica_auth = match serde_json::from_str( - &first_replica - .orb_with_output(&["sphere", "auth", "list", "--as-json"]) - .await? - .join("\n"), - )? { - Value::Array(auths) => match auths - .iter() - .filter(|auth| { - auth.as_object() - .unwrap() - .get("name") - .unwrap() - .as_str() - .unwrap() - != "(OWNER)" + // let cursor = SphereCursor::latest(ctx); + // let _peer2_ctx = cursor + // .traverse_by_petnames(&["peer2".into()]) + // .await? + // .unwrap(); + Ok(()) }) - .take(1) - .next() - .unwrap() - { - Value::Object(auth) => auth.get("link").unwrap().as_str().unwrap().to_owned(), - _ => panic!(), - }, - _ => panic!(), - }; - - first_replica.orb(&["sphere", "sync"]).await?; - - second_replica - .orb(&[ - "sphere", - "join", - "--authorization", - &second_replica_auth, - "--local-key", - "bar", - "--gateway-url", - "http://127.0.0.1:4433", - &client_sphere_id, - ]) - .await?; - - let foo_contents = - tokio::fs::read_to_string(second_replica.sphere_directory().join("foo.subtext")).await?; - - assert_eq!(foo_contents.as_str(), "foobar"); + .await?; - gateway_task.abort(); + ns_task.abort(); - Ok(()) + Ok(()) + } } -#[cfg(feature = "test_kubo")] mod multiplayer { - use crate::helpers::{start_name_system_server, wait, CliSimulator, SpherePair}; use anyhow::Result; - use noosphere_core::{data::Did, tracing::initialize_tracing}; - use noosphere_sphere::{ - HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SpherePetnameWrite, - SphereSync, SyncRecovery, + use noosphere_cli::helpers::{start_name_system_server, CliSimulator, SpherePair}; + use noosphere_common::helpers::wait; + use noosphere_core::{ + context::{ + HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SpherePetnameWrite, + SphereSync, SyncRecovery, + }, + data::Did, + tracing::initialize_tracing, }; use serde_json::Value; use url::Url; diff --git a/rust/noosphere-cli/tests/gateway.rs b/rust/noosphere/tests/gateway.rs similarity index 87% rename from rust/noosphere-cli/tests/gateway.rs rename to rust/noosphere/tests/gateway.rs index c594acb1c..637382774 100644 --- a/rust/noosphere-cli/tests/gateway.rs +++ b/rust/noosphere/tests/gateway.rs @@ -1,8 +1,12 @@ #![cfg(not(target_arch = "wasm32"))] +//! Integration tests that span the distance between a client and a gateway; +//! tests in this module should be able to run without an available backing +//! IPFS-like block syndication layer. + use anyhow::Result; use noosphere::key::KeyStorage; -use noosphere_sphere::{ +use noosphere_core::context::{ HasMutableSphereContext, HasSphereContext, SphereAuthorityWrite, SphereContentRead, SphereContentWrite, SphereCursor, SphereSync, SyncRecovery, }; @@ -12,23 +16,21 @@ use tokio::io::AsyncReadExt; use tokio_stream::StreamExt; use url::Url; -use noosphere_api::route::Route; +use noosphere_core::api::v0alpha1; use noosphere_core::data::{ContentType, Did}; use ucan::crypto::KeyMaterial; -use noosphere_cli::commands::{ - key::key_create, - sphere::{sphere_create, sphere_join}, +use noosphere_cli::{ + commands::{ + key::key_create, + sphere::{sphere_create, sphere_join}, + }, + helpers::{temporary_workspace, SpherePair}, }; use noosphere_core::tracing::initialize_tracing; use noosphere_gateway::{start_gateway, GatewayScope}; -mod helpers; -use helpers::temporary_workspace; - -use crate::helpers::SpherePair; - #[tokio::test] async fn gateway_tells_you_its_identity() -> Result<()> { initialize_tracing(None); @@ -81,7 +83,7 @@ async fn gateway_tells_you_its_identity() -> Result<()> { gateway_address.port() )) .unwrap(); - url.set_path(&Route::Did.to_string()); + url.set_path(&v0alpha1::Route::Did.to_string()); let did_response = client.get(url).send().await.unwrap().text().await.unwrap(); @@ -388,6 +390,60 @@ async fn gateway_receives_sphere_revisions_from_a_client() -> Result<()> { Ok(()) } +// #[tokio::test] +// async fn all_of_client_history_is_made_manifest_on_the_gateway_after_sync() -> Result<()> { +// initialize_tracing(None); + +// let mut sphere_pair = SpherePair::new( +// "one", +// &Url::parse("http://127.0.0.1:5001")?, +// &Url::parse("http://127.0.0.1:6667")?, +// ) +// .await?; + +// sphere_pair.start_gateway().await?; + +// let final_client_version = sphere_pair +// .spawn(move |mut client_sphere_context| async move { +// for value in ["one", "two", "three"] { +// client_sphere_context +// .write(value, &ContentType::Text, value.as_bytes(), None) +// .await?; +// client_sphere_context.save(None).await?; +// } + +// client_sphere_context.sync(SyncRecovery::None).await?; + +// for value in ["one", "two", "three"] { +// client_sphere_context +// .set_petname(value, Some(Did(format!("did:key:{}", value)))) +// .await?; +// client_sphere_context.save(None).await?; +// } + +// client_sphere_context.sync(SyncRecovery::None).await?; + +// wait(1).await; + +// Ok(client_sphere_context.sync(SyncRecovery::None).await?) +// }) +// .await?; + +// // Stream all of the blocks of client history as represented in gateway's storage +// let block_stream = memo_history_stream( +// sphere_pair.gateway.workspace.db().await?, +// &final_client_version, +// None, +// true, +// ); + +// tokio::pin!(block_stream); + +// while let Some(_) = block_stream.try_next().await? {} + +// Ok(()) +// } + #[tokio::test] async fn gateway_can_sync_an_authorized_sphere_across_multiple_replicas() -> Result<()> { initialize_tracing(None); diff --git a/rust/noosphere/tests/integration.rs b/rust/noosphere/tests/sphere_channel.rs similarity index 97% rename from rust/noosphere/tests/integration.rs rename to rust/noosphere/tests/sphere_channel.rs index ab7d3d38e..6631e601b 100644 --- a/rust/noosphere/tests/integration.rs +++ b/rust/noosphere/tests/sphere_channel.rs @@ -1,4 +1,5 @@ -#![cfg(test)] +//! Integration tests to demonstrate that a [SphereChannel] works as expected in +//! end-to-end flows. use std::pin::Pin; @@ -10,8 +11,8 @@ use std::time::Duration; use anyhow::Result; use async_stream::try_stream; use bytes::Bytes; +use noosphere_core::context::{HasMutableSphereContext, SphereContentRead, SphereContentWrite}; use noosphere_core::{data::ContentType, tracing::initialize_tracing}; -use noosphere_sphere::{HasMutableSphereContext, SphereContentRead, SphereContentWrite}; use tokio::io::AsyncReadExt; use tokio_stream::Stream;