diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml
index 1bd9d838e..928c12b18 100644
--- a/.github/workflows/run_test_suite.yaml
+++ b/.github/workflows/run_test_suite.yaml
@@ -55,7 +55,7 @@ jobs:
ipfs_version: v0.17.0
run_daemon: true
- name: 'Run Rust native target tests'
- run: cargo test --features test_kubo
+ run: cargo test --features test-kubo,helpers
env:
NOOSPHERE_LOG: deafening
@@ -96,7 +96,7 @@ jobs:
ipfs_version: v0.17.0
run_daemon: true
- name: 'Run Rust native target tests'
- run: NOOSPHERE_LOG=deafening cargo test --features test_kubo,headers
+ run: NOOSPHERE_LOG=deafening cargo test --features test-kubo,headers
run-test-suite-linux-c:
runs-on: ubuntu-latest
steps:
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 5876957c0..af68d05a2 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -11,6 +11,6 @@
]
},
"rust-analyzer.cargo.features": [
- "test_kubo"
+ "test-kubo"
]
}
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 49de2def0..77ae6c161 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -176,24 +176,23 @@ dependencies = [
[[package]]
name = "anstream"
-version = "0.3.2"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
- "is-terminal",
"utf8parse",
]
[[package]]
name = "anstyle"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"
[[package]]
name = "anstyle-parse"
@@ -215,9 +214,9 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
-version = "1.0.1"
+version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
dependencies = [
"anstyle",
"windows-sys",
@@ -349,9 +348,9 @@ dependencies = [
[[package]]
name = "async-lock"
-version = "2.7.0"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7"
+checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
dependencies = [
"event-listener",
]
@@ -574,9 +573,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.3.3"
+version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "bitvec"
@@ -777,14 +776,14 @@ dependencies = [
[[package]]
name = "chrono"
-version = "0.4.26"
+version = "0.4.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
+checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f"
dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
- "winapi",
+ "windows-targets",
]
[[package]]
@@ -831,33 +830,31 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.3.19"
+version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
+checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6"
dependencies = [
"clap_builder",
"clap_derive",
- "once_cell",
]
[[package]]
name = "clap_builder"
-version = "4.3.19"
+version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
+checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
- "once_cell",
"strsim",
]
[[package]]
name = "clap_derive"
-version = "4.3.12"
+version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
+checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
dependencies = [
"heck",
"proc-macro2",
@@ -867,9 +864,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.5.0"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
[[package]]
name = "colorchoice"
@@ -889,7 +886,7 @@ dependencies = [
"http",
"mime",
"mime_guess",
- "rand 0.8.5",
+ "rand",
"thiserror",
]
@@ -1113,18 +1110,32 @@ dependencies = [
[[package]]
name = "curve25519-dalek"
-version = "4.0.0-rc.1"
+version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16"
+checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2"
dependencies = [
"cfg-if",
+ "cpufeatures",
+ "curve25519-dalek-derive",
+ "digest 0.10.7",
"fiat-crypto",
- "packed_simd_2",
"platforms",
+ "rustc_version",
"subtle",
"zeroize",
]
+[[package]]
+name = "curve25519-dalek-derive"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
[[package]]
name = "darling"
version = "0.14.4"
@@ -1390,24 +1401,25 @@ dependencies = [
[[package]]
name = "ed25519"
-version = "1.5.3"
+version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7"
+checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d"
dependencies = [
- "signature 1.6.4",
+ "pkcs8 0.10.2",
+ "signature 2.1.0",
]
[[package]]
name = "ed25519-dalek"
-version = "1.0.1"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d"
+checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980"
dependencies = [
- "curve25519-dalek 3.2.0",
+ "curve25519-dalek 4.0.0",
"ed25519",
- "rand 0.7.3",
+ "rand_core 0.6.4",
"serde",
- "sha2 0.9.9",
+ "sha2 0.10.7",
"zeroize",
]
@@ -1476,9 +1488,9 @@ dependencies = [
[[package]]
name = "encoding_rs"
-version = "0.8.32"
+version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
+checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
dependencies = [
"cfg-if",
]
@@ -1503,9 +1515,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
+checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
dependencies = [
"errno-dragonfly",
"libc",
@@ -1737,8 +1749,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd"
dependencies = [
"futures-io",
- "rustls 0.20.8",
- "webpki 0.22.0",
+ "rustls 0.20.9",
+ "webpki 0.22.1",
]
[[package]]
@@ -1880,6 +1892,27 @@ dependencies = [
"regex",
]
+[[package]]
+name = "gloo-net"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ac9e8288ae2c632fa9f8657ac70bfe38a1530f345282d7ba66a1f70b72b7dc4"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-sink",
+ "gloo-utils",
+ "http",
+ "js-sys",
+ "pin-project",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "gloo-timers"
version = "0.2.6"
@@ -1892,6 +1925,19 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "gloo-utils"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa"
+dependencies = [
+ "js-sys",
+ "serde",
+ "serde_json",
+ "wasm-bindgen",
+ "web-sys",
+]
+
[[package]]
name = "group"
version = "0.12.1"
@@ -1916,9 +1962,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.3.20"
+version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
+checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
dependencies = [
"bytes",
"fnv",
@@ -2096,9 +2142,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]]
name = "httpdate"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
@@ -2146,7 +2192,7 @@ dependencies = [
"futures-util",
"http",
"hyper",
- "rustls 0.21.6",
+ "rustls 0.21.7",
"tokio",
"tokio-rustls",
]
@@ -2280,7 +2326,7 @@ dependencies = [
"async-trait",
"bytes",
"log",
- "rand 0.8.5",
+ "rand",
"rtcp",
"rtp",
"thiserror",
@@ -2332,7 +2378,7 @@ dependencies = [
"socket2 0.5.3",
"widestring",
"windows-sys",
- "winreg 0.50.0",
+ "winreg",
]
[[package]]
@@ -2381,17 +2427,6 @@ dependencies = [
"unsigned-varint",
]
-[[package]]
-name = "is-terminal"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
-dependencies = [
- "hermit-abi",
- "rustix 0.38.7",
- "windows-sys",
-]
-
[[package]]
name = "itertools"
version = "0.11.0"
@@ -2530,12 +2565,6 @@ dependencies = [
"thiserror",
]
-[[package]]
-name = "libm"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a"
-
[[package]]
name = "libm"
version = "0.2.7"
@@ -2617,7 +2646,7 @@ dependencies = [
"parking_lot 0.12.1",
"pin-project",
"quick-protobuf",
- "rand 0.8.5",
+ "rand",
"rw-stream-sink",
"serde",
"smallvec",
@@ -2662,7 +2691,7 @@ dependencies = [
"prometheus-client",
"quick-protobuf",
"quick-protobuf-codec",
- "rand 0.8.5",
+ "rand",
"regex",
"serde",
"sha2 0.10.7",
@@ -2697,9 +2726,9 @@ dependencies = [
[[package]]
name = "libp2p-identity"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e2d584751cecb2aabaa56106be6be91338a60a0f4e420cf2af639204f596fc1"
+checksum = "276bb57e7af15d8f100d3c11cbdd32c6752b7eef4ba7a18ecf464972c07abcce"
dependencies = [
"bs58 0.4.0",
"ed25519-dalek",
@@ -2707,7 +2736,7 @@ dependencies = [
"multiaddr",
"multihash 0.17.0",
"quick-protobuf",
- "rand 0.8.5",
+ "rand",
"serde",
"sha2 0.10.7",
"thiserror",
@@ -2733,7 +2762,7 @@ dependencies = [
"libp2p-swarm",
"log",
"quick-protobuf",
- "rand 0.8.5",
+ "rand",
"serde",
"sha2 0.10.7",
"smallvec",
@@ -2756,7 +2785,7 @@ dependencies = [
"libp2p-identity",
"libp2p-swarm",
"log",
- "rand 0.8.5",
+ "rand",
"smallvec",
"socket2 0.4.9",
"tokio",
@@ -2791,7 +2820,7 @@ dependencies = [
"log",
"once_cell",
"quick-protobuf",
- "rand 0.8.5",
+ "rand",
"sha2 0.10.7",
"snow",
"static_assertions",
@@ -2816,8 +2845,8 @@ dependencies = [
"log",
"parking_lot 0.12.1",
"quinn-proto",
- "rand 0.8.5",
- "rustls 0.20.8",
+ "rand",
+ "rustls 0.20.9",
"thiserror",
"tokio",
]
@@ -2837,7 +2866,7 @@ dependencies = [
"libp2p-identity",
"libp2p-swarm-derive",
"log",
- "rand 0.8.5",
+ "rand",
"smallvec",
"tokio",
"void",
@@ -2882,9 +2911,9 @@ dependencies = [
"libp2p-identity",
"rcgen 0.10.0",
"ring",
- "rustls 0.20.8",
+ "rustls 0.20.9",
"thiserror",
- "webpki 0.22.0",
+ "webpki 0.22.1",
"x509-parser 0.14.0",
"yasna",
]
@@ -2909,7 +2938,7 @@ dependencies = [
"multihash 0.17.0",
"quick-protobuf",
"quick-protobuf-codec",
- "rand 0.8.5",
+ "rand",
"rcgen 0.9.3",
"serde",
"stun",
@@ -3317,15 +3346,20 @@ dependencies = [
"lazy_static",
"libipld-cbor",
"libipld-core",
- "noosphere-api",
+ "noosphere-cli",
+ "noosphere-common",
"noosphere-core",
+ "noosphere-gateway",
"noosphere-into",
"noosphere-ipfs",
- "noosphere-sphere",
+ "noosphere-ns",
"noosphere-storage",
"pkg-version",
+ "rand",
+ "reqwest",
"rexie",
"safer-ffi",
+ "serde_json",
"subtext",
"tempfile",
"thiserror",
@@ -3384,15 +3418,13 @@ dependencies = [
"libipld-core",
"mime_guess",
"noosphere",
- "noosphere-api",
"noosphere-core",
"noosphere-gateway",
"noosphere-ipfs",
"noosphere-ns",
- "noosphere-sphere",
"noosphere-storage",
"pathdiff",
- "rand 0.8.5",
+ "rand",
"reqwest",
"serde",
"serde_json",
@@ -3436,6 +3468,20 @@ dependencies = [
"wasm-bindgen-test",
]
+[[package]]
+name = "noosphere-common"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "futures",
+ "futures-util",
+ "rand",
+ "tokio",
+ "tracing",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
[[package]]
name = "noosphere-core"
version = "0.15.2"
@@ -3445,37 +3491,50 @@ dependencies = [
"async-recursion",
"async-stream",
"async-trait",
+ "axum",
"base64 0.21.2",
"byteorder",
+ "bytes",
"cid",
"console_error_panic_hook",
"ed25519-zebra",
"fastcdc",
"futures",
+ "futures-util",
"getrandom 0.2.10",
+ "gloo-net",
+ "iroh-car",
+ "js-sys",
"libipld-cbor",
"libipld-core",
"noosphere-collections",
+ "noosphere-common",
"noosphere-storage",
"once_cell",
- "rand 0.8.5",
+ "rand",
+ "reqwest",
"sentry-tracing",
"serde",
"serde_bytes",
"serde_json",
+ "serde_urlencoded",
"strum 0.25.0",
"strum_macros",
"thiserror",
"tiny-bip39",
"tokio",
"tokio-stream",
+ "tokio-util",
"tracing",
"tracing-subscriber",
"tracing-wasm",
"ucan",
"ucan-key-support",
"url",
+ "wasm-bindgen",
"wasm-bindgen-test",
+ "wasm-streams",
+ "web-sys",
]
[[package]]
@@ -3492,11 +3551,9 @@ dependencies = [
"libipld-cbor",
"libipld-core",
"mime_guess",
- "noosphere-api",
"noosphere-core",
"noosphere-ipfs",
"noosphere-ns",
- "noosphere-sphere",
"noosphere-storage",
"reqwest",
"serde",
@@ -3513,6 +3570,7 @@ dependencies = [
"ucan-key-support",
"url",
"wasm-bindgen",
+ "wnfs-common",
"wnfs-namefilter",
]
@@ -3532,7 +3590,6 @@ dependencies = [
"horrorshow",
"libipld-cbor",
"noosphere-core",
- "noosphere-sphere",
"noosphere-storage",
"subtext",
"tempfile",
@@ -3562,9 +3619,10 @@ dependencies = [
"iroh-car",
"libipld-cbor",
"libipld-core",
+ "noosphere-common",
"noosphere-core",
"noosphere-storage",
- "rand 0.8.5",
+ "rand",
"reqwest",
"serde",
"serde_json",
@@ -3593,7 +3651,7 @@ dependencies = [
"noosphere-core",
"noosphere-ipfs",
"noosphere-storage",
- "rand 0.8.5",
+ "rand",
"reqwest",
"serde",
"serde_json",
@@ -3652,6 +3710,7 @@ dependencies = [
"js-sys",
"libipld-cbor",
"libipld-core",
+ "noosphere-common",
"rexie",
"serde",
"sled",
@@ -3687,9 +3746,9 @@ dependencies = [
[[package]]
name = "num-bigint"
-version = "0.4.3"
+version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
dependencies = [
"autocfg",
"num-integer",
@@ -3704,11 +3763,11 @@ checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151"
dependencies = [
"byteorder",
"lazy_static",
- "libm 0.2.7",
+ "libm",
"num-integer",
"num-iter",
"num-traits",
- "rand 0.8.5",
+ "rand",
"smallvec",
"zeroize",
]
@@ -3741,7 +3800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
dependencies = [
"autocfg",
- "libm 0.2.7",
+ "libm",
]
[[package]]
@@ -3839,16 +3898,6 @@ dependencies = [
"sha2 0.10.7",
]
-[[package]]
-name = "packed_simd_2"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282"
-dependencies = [
- "cfg-if",
- "libm 0.1.4",
-]
-
[[package]]
name = "parking"
version = "2.1.0"
@@ -4041,9 +4090,9 @@ dependencies = [
[[package]]
name = "platforms"
-version = "3.0.2"
+version = "3.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630"
+checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8"
[[package]]
name = "polling"
@@ -4184,13 +4233,13 @@ dependencies = [
[[package]]
name = "prometheus-client-derive-encode"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b6a5217beb0ad503ee7fa752d451c905113d70721b937126158f3106a48cc1"
+checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
dependencies = [
"proc-macro2",
"quote",
- "syn 1.0.109",
+ "syn 2.0.28",
]
[[package]]
@@ -4228,15 +4277,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31999cfc7927c4e212e60fd50934ab40e8e8bfd2d493d6095d2d306bc0764d9"
dependencies = [
"bytes",
- "rand 0.8.5",
+ "rand",
"ring",
"rustc-hash",
- "rustls 0.20.8",
+ "rustls 0.20.9",
"slab",
"thiserror",
"tinyvec",
"tracing",
- "webpki 0.22.0",
+ "webpki 0.22.1",
]
[[package]]
@@ -4254,19 +4303,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
-[[package]]
-name = "rand"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
-dependencies = [
- "getrandom 0.1.16",
- "libc",
- "rand_chacha 0.2.2",
- "rand_core 0.5.1",
- "rand_hc",
-]
-
[[package]]
name = "rand"
version = "0.8.5"
@@ -4274,20 +4310,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
- "rand_chacha 0.3.1",
+ "rand_chacha",
"rand_core 0.6.4",
]
-[[package]]
-name = "rand_chacha"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.5.1",
-]
-
[[package]]
name = "rand_chacha"
version = "0.3.1"
@@ -4316,15 +4342,6 @@ dependencies = [
"getrandom 0.2.10",
]
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
-dependencies = [
- "rand_core 0.5.1",
-]
-
[[package]]
name = "rcgen"
version = "0.9.3"
@@ -4425,9 +4442,9 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
[[package]]
name = "reqwest"
-version = "0.11.18"
+version = "0.11.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
+checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
dependencies = [
"base64 0.21.2",
"bytes",
@@ -4446,7 +4463,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
- "rustls 0.21.6",
+ "rustls 0.21.7",
"rustls-pemfile",
"serde",
"serde_json",
@@ -4461,7 +4478,7 @@ dependencies = [
"wasm-streams",
"web-sys",
"webpki-roots",
- "winreg 0.10.1",
+ "winreg",
]
[[package]]
@@ -4581,7 +4598,7 @@ checksum = "a2a095411ff00eed7b12e4c6a118ba984d113e1079582570d56a5ee723f11f80"
dependencies = [
"async-trait",
"bytes",
- "rand 0.8.5",
+ "rand",
"serde",
"thiserror",
"webrtc-util",
@@ -4633,11 +4650,11 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.7"
+version = "0.38.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
+checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453"
dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.0",
"errno",
"libc",
"linux-raw-sys 0.4.5",
@@ -4659,21 +4676,21 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.20.8"
+version = "0.20.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
+checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99"
dependencies = [
"log",
"ring",
"sct 0.7.0",
- "webpki 0.22.0",
+ "webpki 0.22.1",
]
[[package]]
name = "rustls"
-version = "0.21.6"
+version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb"
+checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
dependencies = [
"log",
"ring",
@@ -4692,9 +4709,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.101.2"
+version = "0.101.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59"
+checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d"
dependencies = [
"ring",
"untrusted",
@@ -4800,7 +4817,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d22a5ef407871893fd72b4562ee15e4742269b173959db4b8df6f538c414e13"
dependencies = [
- "rand 0.8.5",
+ "rand",
"substring",
"thiserror",
"url",
@@ -4847,7 +4864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7163491708804a74446642ff2c80b3acd668d4b9e9f497f85621f3d250fd012b"
dependencies = [
"once_cell",
- "rand 0.8.5",
+ "rand",
"sentry-types",
"serde",
"serde_json",
@@ -5102,14 +5119,14 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
[[package]]
name = "snow"
-version = "0.9.2"
+version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733"
+checksum = "0c9d1425eb528a21de2755c75af4c9b5d57f50a0d4c3b7f1828a4cd03f8ba155"
dependencies = [
"aes-gcm 0.9.4",
"blake2",
"chacha20poly1305",
- "curve25519-dalek 4.0.0-rc.1",
+ "curve25519-dalek 4.0.0",
"rand_core 0.6.4",
"ring",
"rustc_version",
@@ -5210,7 +5227,7 @@ dependencies = [
"crc",
"lazy_static",
"md-5",
- "rand 0.8.5",
+ "rand",
"ring",
"subtle",
"thiserror",
@@ -5325,14 +5342,14 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
-version = "3.7.1"
+version = "3.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
dependencies = [
"cfg-if",
"fastrand 2.0.0",
"redox_syscall 0.3.5",
- "rustix 0.38.7",
+ "rustix 0.38.11",
"windows-sys",
]
@@ -5415,7 +5432,7 @@ dependencies = [
"hmac 0.12.1",
"once_cell",
"pbkdf2",
- "rand 0.8.5",
+ "rand",
"rustc-hash",
"sha2 0.10.7",
"thiserror",
@@ -5486,7 +5503,7 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
- "rustls 0.21.6",
+ "rustls 0.21.7",
"tokio",
]
@@ -5577,11 +5594,11 @@ dependencies = [
[[package]]
name = "tower-http"
-version = "0.4.3"
+version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82"
+checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.0",
"bytes",
"futures-core",
"futures-util",
@@ -5715,7 +5732,7 @@ dependencies = [
"idna 0.2.3",
"ipnet",
"lazy_static",
- "rand 0.8.5",
+ "rand",
"smallvec",
"socket2 0.4.9",
"thiserror",
@@ -5762,7 +5779,7 @@ dependencies = [
"futures",
"log",
"md-5",
- "rand 0.8.5",
+ "rand",
"ring",
"stun",
"thiserror",
@@ -5794,7 +5811,7 @@ dependencies = [
"libipld-core",
"libipld-json",
"log",
- "rand 0.8.5",
+ "rand",
"serde",
"serde_json",
"strum 0.24.1",
@@ -5839,9 +5856,9 @@ dependencies = [
[[package]]
name = "unicase"
-version = "2.6.0"
+version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check",
]
@@ -6120,9 +6137,9 @@ dependencies = [
[[package]]
name = "wasm-streams"
-version = "0.2.3"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
dependencies = [
"futures-util",
"js-sys",
@@ -6168,9 +6185,9 @@ dependencies = [
[[package]]
name = "webpki"
-version = "0.22.0"
+version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
+checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e"
dependencies = [
"ring",
"untrusted",
@@ -6178,12 +6195,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
-version = "0.22.6"
+version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
-dependencies = [
- "webpki 0.22.0",
-]
+checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
[[package]]
name = "webrtc"
@@ -6198,7 +6212,7 @@ dependencies = [
"interceptor",
"lazy_static",
"log",
- "rand 0.8.5",
+ "rand",
"rcgen 0.9.3",
"regex",
"ring",
@@ -6243,9 +6257,9 @@ dependencies = [
[[package]]
name = "webrtc-dtls"
-version = "0.7.1"
+version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "942be5bd85f072c3128396f6e5a9bfb93ca8c1939ded735d177b7bcba9a13d05"
+checksum = "c4a00f4242f2db33307347bd5be53263c52a0331c96c14292118c9a6bb48d267"
dependencies = [
"aes 0.6.0",
"aes-gcm 0.10.2",
@@ -6260,12 +6274,11 @@ dependencies = [
"hkdf",
"hmac 0.12.1",
"log",
- "oid-registry 0.6.1",
"p256 0.11.1",
"p384",
- "rand 0.8.5",
+ "rand",
"rand_core 0.6.4",
- "rcgen 0.9.3",
+ "rcgen 0.10.0",
"ring",
"rustls 0.19.1",
"sec1 0.3.0",
@@ -6278,7 +6291,7 @@ dependencies = [
"tokio",
"webpki 0.21.4",
"webrtc-util",
- "x25519-dalek 2.0.0-pre.1",
+ "x25519-dalek 2.0.0",
"x509-parser 0.13.2",
]
@@ -6292,7 +6305,7 @@ dependencies = [
"async-trait",
"crc",
"log",
- "rand 0.8.5",
+ "rand",
"serde",
"serde_json",
"stun",
@@ -6327,7 +6340,7 @@ checksum = "f72e1650a8ae006017d1a5280efb49e2610c19ccc3c0905b03b648aee9554991"
dependencies = [
"byteorder",
"bytes",
- "rand 0.8.5",
+ "rand",
"rtp",
"thiserror",
]
@@ -6343,7 +6356,7 @@ dependencies = [
"bytes",
"crc",
"log",
- "rand 0.8.5",
+ "rand",
"thiserror",
"tokio",
"webrtc-util",
@@ -6388,7 +6401,7 @@ dependencies = [
"libc",
"log",
"nix",
- "rand 0.8.5",
+ "rand",
"thiserror",
"tokio",
"winapi",
@@ -6551,22 +6564,13 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
[[package]]
name = "winnow"
-version = "0.5.4"
+version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64"
+checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc"
dependencies = [
"memchr",
]
-[[package]]
-name = "winreg"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
-dependencies = [
- "winapi",
-]
-
[[package]]
name = "winreg"
version = "0.50.0"
@@ -6604,7 +6608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "775ccf0bffa4cae6de1c534ffda97b032d7144d648bcc07ad56f73ac4b064b6b"
dependencies = [
"getopts",
- "rand 0.8.5",
+ "rand",
]
[[package]]
@@ -6671,12 +6675,13 @@ dependencies = [
[[package]]
name = "x25519-dalek"
-version = "2.0.0-pre.1"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5da623d8af10a62342bcbbb230e33e58a63255a58012f8653c578e54bab48df"
+checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96"
dependencies = [
- "curve25519-dalek 3.2.0",
+ "curve25519-dalek 4.0.0",
"rand_core 0.6.4",
+ "serde",
"zeroize",
]
@@ -6733,7 +6738,7 @@ dependencies = [
"log",
"nohash-hasher",
"parking_lot 0.12.1",
- "rand 0.8.5",
+ "rand",
"static_assertions",
]
diff --git a/Cargo.toml b/Cargo.toml
index c7a5a0087..5af9faa43 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
"rust/noosphere-api",
"rust/noosphere-cli",
"rust/noosphere-collections",
+ "rust/noosphere-common",
"rust/noosphere-core",
"rust/noosphere-gateway",
"rust/noosphere-into",
@@ -20,9 +21,13 @@ resolver = "2"
anyhow = { version = "1" }
async-stream = { version = "0.3" }
axum = { version = "^0.6.18" }
+bytes = { version = "^1" }
cid = { version = "0.10" }
directories = { version = "5" }
fastcdc = { version = "3.1" }
+futures = { version = "0.3" }
+futures-util = { version = "0.3" }
+gloo-net = { version = "0.4" }
gloo-timers = { version = "0.2", features = ["futures"] }
ignore = { version = "0.4.20" }
iroh-car = { version = "^0.3.0" }
@@ -32,9 +37,11 @@ libipld-core = { version = "0.16" }
libipld-cbor = { version = "0.16" }
pathdiff = { version = "0.2.1" }
rand = { version = "0.8" }
+reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream"] }
sentry-tracing = { version = "0.31.5" }
serde = { version = "^1" }
serde_json = { version = "^1" }
+serde_urlencoded = { version = "~0.7" }
strum = { version = "0.25" }
strum_macros = { version = "0.25" }
subtext = { version = "0.3.4" }
@@ -43,6 +50,7 @@ tempfile = { version = "^3" }
thiserror = { version = "1" }
tokio = { version = "^1" }
tokio-stream = { version = "~0.1" }
+tokio-util = { version = "0.7" }
tower = { version = "^0.4.13" }
tower-http = { version = "^0.4.3" }
tracing = { version = "0.1" }
@@ -54,6 +62,8 @@ void = { version = "1" }
wasm-bindgen = { version = "^0.2" }
wasm-bindgen-test = { version = "^0.3" }
wasm-bindgen-futures = { version = "^0.4" }
+wasm-streams = { version = "0.3.0" }
+web-sys = { version = "0.3" }
wnfs-namefilter = { version = "0.1.21" }
[profile.release]
diff --git a/release-please-config.json b/release-please-config.json
index dda8f6f76..b04d5105e 100644
--- a/release-please-config.json
+++ b/release-please-config.json
@@ -1,11 +1,16 @@
{
- "plugins": ["cargo-workspace"],
+ "plugins": [
+ "cargo-workspace"
+ ],
"changelog-path": "CHANGELOG.md",
"release-type": "rust",
"bump-minor-pre-major": true,
"bump-patch-for-minor-pre-major": true,
"packages": {
- "rust/noosphere-api": {},
+ "rust/noosphere-api": {
+ "release-as": "0.13.0+deprecated"
+ },
+ "rust/noosphere-common": {},
"rust/noosphere-cli": {
"draft": true
},
@@ -15,11 +20,13 @@
"rust/noosphere-ipfs": {},
"rust/noosphere-into": {},
"rust/noosphere-ns": {},
- "rust/noosphere-sphere": {},
+ "rust/noosphere-sphere": {
+ "release-as": "0.11.0+deprecated"
+ },
"rust/noosphere-storage": {},
"rust/noosphere": {
"draft": true
}
},
"$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
-}
+}
\ No newline at end of file
diff --git a/rust/noosphere-cli/Cargo.toml b/rust/noosphere-cli/Cargo.toml
index afec7ad12..d32fc4a6b 100644
--- a/rust/noosphere-cli/Cargo.toml
+++ b/rust/noosphere-cli/Cargo.toml
@@ -17,14 +17,12 @@ homepage = "https://github.com/subconsciousnetwork/noosphere"
readme = "README.md"
[features]
-test_kubo = []
+helpers = ["tracing-subscriber", "noosphere-ns"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
-tracing-subscriber = { workspace = true }
reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls", "stream"] }
-noosphere-ns = { version = "0.10.2", path = "../noosphere-ns" }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
tempfile = { workspace = true }
@@ -37,6 +35,7 @@ tower = { workspace = true }
tower-http = { workspace = true, features = ["cors", "trace"] }
async-trait = "~0.1"
tracing = { workspace = true }
+tracing-subscriber = { workspace = true, optional = true }
iroh-car = { workspace = true }
url = { workspace = true, features = ["serde"] }
@@ -47,10 +46,9 @@ globset = "~0.4"
noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs" }
noosphere-core = { version = "0.15.2", path = "../noosphere-core" }
-noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
-noosphere-api = { version = "0.12.2", path = "../noosphere-api" }
noosphere-gateway = { version = "0.8.2", path = "../noosphere-gateway" }
+noosphere-ns = { version = "0.10.2", path = "../noosphere-ns", optional = true }
noosphere = { version = "0.14.2", path = "../noosphere" }
ucan = { workspace = true }
ucan-key-support = { workspace = true }
diff --git a/rust/noosphere-cli/src/native/commands/sphere/auth.rs b/rust/noosphere-cli/src/native/commands/sphere/auth.rs
index a1d1f756d..977398f52 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/auth.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/auth.rs
@@ -2,8 +2,10 @@ use std::{collections::BTreeMap, convert::TryFrom};
use anyhow::{anyhow, Result};
use cid::Cid;
+use noosphere_core::context::{
+ HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker,
+};
use noosphere_core::data::{Did, Jwt, Link};
-use noosphere_sphere::{HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker};
use serde_json::{json, Value};
use ucan::{store::UcanJwtStore, Ucan};
diff --git a/rust/noosphere-cli/src/native/commands/sphere/config.rs b/rust/noosphere-cli/src/native/commands/sphere/config.rs
index 18da8a625..ee43ca96a 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/config.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/config.rs
@@ -1,6 +1,6 @@
use anyhow::Result;
+use noosphere_core::context::metadata::{COUNTERPART, GATEWAY_URL};
use noosphere_core::data::Did;
-use noosphere_sphere::metadata::{COUNTERPART, GATEWAY_URL};
use noosphere_storage::KeyValueStore;
use url::Url;
diff --git a/rust/noosphere-cli/src/native/commands/sphere/follow.rs b/rust/noosphere-cli/src/native/commands/sphere/follow.rs
index b18a460c8..680cd2eee 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/follow.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/follow.rs
@@ -2,10 +2,10 @@ use std::collections::BTreeMap;
use crate::native::{commands::sphere::save, workspace::Workspace};
use anyhow::{anyhow, Result};
-use noosphere_core::data::Did;
-use noosphere_sphere::{
+use noosphere_core::context::{
HasMutableSphereContext, SpherePetnameRead, SpherePetnameWrite, SphereWalker,
};
+use noosphere_core::data::Did;
use serde_json::{json, Value};
use tokio_stream::StreamExt;
diff --git a/rust/noosphere-cli/src/native/commands/sphere/history.rs b/rust/noosphere-cli/src/native/commands/sphere/history.rs
index 12e783be2..a8237f3fd 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/history.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/history.rs
@@ -1,6 +1,6 @@
use anyhow::Result;
+use noosphere_core::context::HasSphereContext;
use noosphere_core::data::MapOperation;
-use noosphere_sphere::HasSphereContext;
use tokio_stream::StreamExt;
use ucan::Ucan;
diff --git a/rust/noosphere-cli/src/native/commands/sphere/mod.rs b/rust/noosphere-cli/src/native/commands/sphere/mod.rs
index 2b39074d8..66e9de382 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/mod.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/mod.rs
@@ -27,8 +27,8 @@ use noosphere::{
key::KeyStorage,
sphere::{SphereContextBuilder, SphereContextBuilderArtifacts},
};
+use noosphere_core::context::{HasMutableSphereContext, SphereContext, SphereSync, SyncRecovery};
use noosphere_core::{authority::Authorization, data::Did};
-use noosphere_sphere::{HasMutableSphereContext, SphereContext, SphereSync, SyncRecovery};
use tokio::sync::Mutex;
use ucan::crypto::KeyMaterial;
diff --git a/rust/noosphere-cli/src/native/commands/sphere/save.rs b/rust/noosphere-cli/src/native/commands/sphere/save.rs
index fc9134d10..6a860fb90 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/save.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/save.rs
@@ -1,8 +1,8 @@
use anyhow::{anyhow, Result};
use cid::Cid;
use libipld_cbor::DagCborCodec;
+use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor};
use noosphere_core::data::Header;
-use noosphere_sphere::{HasMutableSphereContext, SphereContentWrite, SphereCursor};
use noosphere_storage::BlockStore;
use crate::native::{
diff --git a/rust/noosphere-cli/src/native/commands/sphere/status.rs b/rust/noosphere-cli/src/native/commands/sphere/status.rs
index fd7eaf3e9..783b2c922 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/status.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/status.rs
@@ -2,8 +2,8 @@ use std::collections::BTreeMap;
use crate::native::{content::Content, Workspace};
use anyhow::Result;
+use noosphere_core::context::{HasSphereContext, SphereCursor};
use noosphere_core::data::ContentType;
-use noosphere_sphere::{HasSphereContext, SphereCursor};
fn status_section(
name: &str,
diff --git a/rust/noosphere-cli/src/native/commands/sphere/sync.rs b/rust/noosphere-cli/src/native/commands/sphere/sync.rs
index 3bc8e1e13..b3d81ac51 100644
--- a/rust/noosphere-cli/src/native/commands/sphere/sync.rs
+++ b/rust/noosphere-cli/src/native/commands/sphere/sync.rs
@@ -1,6 +1,6 @@
use crate::native::{content::Content, workspace::Workspace};
use anyhow::{anyhow, Result};
-use noosphere_sphere::{SphereSync, SyncRecovery};
+use noosphere_core::context::{SphereSync, SyncRecovery};
/// Attempt to synchronize the local workspace with a configured gateway,
/// optionally automatically retrying a fixed number of times in case a rebase
diff --git a/rust/noosphere-cli/src/native/content.rs b/rust/noosphere-cli/src/native/content.rs
index ab5377546..d11a8d6ea 100644
--- a/rust/noosphere-cli/src/native/content.rs
+++ b/rust/noosphere-cli/src/native/content.rs
@@ -11,7 +11,7 @@ use subtext::util::to_slug;
use tokio::fs;
use tokio_stream::StreamExt;
-use noosphere_sphere::SphereWalker;
+use noosphere_core::context::SphereWalker;
use super::{extension::infer_content_type, paths::SpherePaths, workspace::Workspace};
diff --git a/rust/noosphere-cli/tests/helpers/cli.rs b/rust/noosphere-cli/src/native/helpers/cli.rs
similarity index 76%
rename from rust/noosphere-cli/tests/helpers/cli.rs
rename to rust/noosphere-cli/src/native/helpers/cli.rs
index c97a5c53f..a2f477b19 100644
--- a/rust/noosphere-cli/tests/helpers/cli.rs
+++ b/rust/noosphere-cli/src/native/helpers/cli.rs
@@ -1,18 +1,13 @@
-#![allow(dead_code)]
-
-use tracing::*;
-
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::Mutex;
use tempfile::TempDir;
-use tracing::Level;
-use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
-use tracing_subscriber::Layer;
+use tracing::{field, Level, Subscriber};
+use tracing_subscriber::{prelude::*, Layer};
+use crate::{cli::Cli, invoke_cli, workspace::Workspace};
use clap::Parser;
-use noosphere_cli::{cli::Cli, invoke_cli, workspace::Workspace};
#[derive(Default)]
struct InfoCaptureVisitor {
@@ -58,6 +53,8 @@ where
}
}
+/// Poll a [Future] to completion, capturing the log output from work performed
+/// in the span of that work.
pub fn run_and_capture_info<F>(fut: F) -> Result<Vec<String>>
where
F: std::future::Future<Output = Result<()>>,
@@ -83,6 +80,10 @@ where
Ok(lines)
}
+/// A helper that simulates using the `orb` CLI from a command line. When
+/// initialized, it produces a temporary directory for a sphere and for global
+/// Noosphere configuration, and ensures that any command that expects these
+/// things is configured to use them.
pub struct CliSimulator {
current_working_directory: PathBuf,
sphere_directory: TempDir,
@@ -90,6 +91,7 @@ pub struct CliSimulator {
}
impl CliSimulator {
+ /// Initialize a new [CliSimulator]
pub fn new() -> Result<Self> {
let sphere_directory = TempDir::new()?;
@@ -100,6 +102,9 @@ impl CliSimulator {
})
}
+ /// Logs a command to quickly move into the simulator's temporary
+ /// directory and ensure its temporary credentials are available in your
+ /// global Noosphere key storage location
pub fn print_debug_shell_command(&self) {
info!(
"cd {} && cp {}/keys/* $HOME/.config/noosphere/keys/",
@@ -108,22 +113,30 @@ impl CliSimulator {
);
}
+ /// The temporary root path for the [CliSimulator]'s sphere directory
pub fn sphere_directory(&self) -> &Path {
self.sphere_directory.path()
}
+ /// The temporary root path for the [CliSimulator]'s global Noosphere
+ /// directory
pub fn noosphere_directory(&self) -> &Path {
self.noosphere_directory.path()
}
+ /// Change the current working directory used by this [CliSimulator] when
+ /// simulating `orb` commands
pub fn cd(&mut self, path: &Path) {
self.current_working_directory = path.to_owned();
}
+ /// Change the current working directory to the temporary root sphere
+ /// directory in use by this [CliSimulator]
pub fn cd_to_sphere_directory(&mut self) {
self.current_working_directory = self.sphere_directory().to_owned();
}
+ /// Run an `orb` command, capturing and returning its logged output
pub async fn orb_with_output(&self, command: &[&str]) -> Result<Vec<String>> {
Ok(self
.run_orb_command(command, true)
@@ -131,6 +144,7 @@ impl CliSimulator {
.unwrap_or_default())
}
+ /// Run an `orb` command
pub async fn orb(&self, command: &[&str]) -> Result<()> {
self.run_orb_command(command, false).await?;
Ok(())
diff --git a/rust/noosphere-cli/src/native/helpers/mod.rs b/rust/noosphere-cli/src/native/helpers/mod.rs
new file mode 100644
index 000000000..1074556b3
--- /dev/null
+++ b/rust/noosphere-cli/src/native/helpers/mod.rs
@@ -0,0 +1,8 @@
+//! Generic helper utilities intended to be used exclusively in tests and hidden
+//! setup for examples
+
+mod cli;
+mod workspace;
+
+pub use cli::*;
+pub use workspace::*;
diff --git a/rust/noosphere-cli/tests/helpers/mod.rs b/rust/noosphere-cli/src/native/helpers/workspace.rs
similarity index 90%
rename from rust/noosphere-cli/tests/helpers/mod.rs
rename to rust/noosphere-cli/src/native/helpers/workspace.rs
index e8babb3a3..9f938b8db 100644
--- a/rust/noosphere-cli/tests/helpers/mod.rs
+++ b/rust/noosphere-cli/src/native/helpers/workspace.rs
@@ -1,26 +1,20 @@
-#![allow(dead_code)]
-
-mod cli;
-mod random;
-
-pub use crate::helpers::random::*;
-pub use cli::*;
-
use anyhow::Result;
use noosphere_ipfs::{IpfsStore, KuboClient};
use noosphere_storage::{BlockStoreRetry, MemoryStore, NativeStorage, UcanStore};
-use std::{net::TcpListener, sync::Arc, time::Duration};
+use std::{net::TcpListener, sync::Arc};
use tempfile::TempDir;
-use noosphere_cli::{
+use crate::{
cli::ConfigSetCommand,
commands::{key::key_create, sphere::config_set, sphere::sphere_create},
workspace::Workspace,
};
-use noosphere_core::data::Did;
+use noosphere_core::{
+ context::{HasSphereContext, SphereContext},
+ data::Did,
+};
use noosphere_gateway::{start_gateway, GatewayScope};
use noosphere_ns::{helpers::NameSystemNetwork, server::start_name_system_api_server};
-use noosphere_sphere::{HasSphereContext, SphereContext};
use tokio::{sync::Mutex, task::JoinHandle};
use url::Url;
@@ -42,14 +36,6 @@ pub fn temporary_workspace() -> Result<(Workspace, (TempDir, TempDir))> {
(root, global_root),
))
}
-
-/// Wait for the specified number of seconds; uses [tokio::time::sleep], so this
-/// will yield to the async runtime rather than block until the sleep time is
-/// elapsed.
-pub async fn wait(seconds: u64) {
- tokio::time::sleep(Duration::from_secs(seconds)).await;
-}
-
async fn start_gateway_for_workspace(
workspace: &Workspace,
client_sphere_identity: &Did,
@@ -89,6 +75,7 @@ async fn start_gateway_for_workspace(
Ok((gateway_url, join_handle))
}
+/// Start a Noosphere Name System server for use in a test scenario
pub async fn start_name_system_server(ipfs_url: &Url) -> Result<(Url, JoinHandle<()>)> {
// TODO(#267)
let use_validation = false;
@@ -119,7 +106,9 @@ pub async fn start_name_system_server(ipfs_url: &Url) -> Result<(Url, JoinHandle
/// Represents a single sphere and a corresponding workspace.
pub struct SphereData {
+ /// The identity of the sphere
pub identity: Did,
+ /// The temporary [Workspace] of the sphere
pub workspace: Workspace,
_temp_dirs: (tempfile::TempDir, tempfile::TempDir),
}
@@ -128,10 +117,15 @@ pub struct SphereData {
/// and provides high-level utility methods for synchronizing between
/// the two in order to support DSL-like integration tests.
pub struct SpherePair {
+ /// The human-readable name of the [SpherePair]
pub name: String,
+ /// The [SphereData] for the client half of the [SpherePair]
pub client: SphereData,
+ /// The [SphereData] for the gateway half of the [SpherePair]
pub gateway: SphereData,
+ /// The base [Url] for a separately-initialized Noosphere Name System REST API
pub ns_url: Url,
+ /// The base [Url] for a separately-initialized IPFS Kubo RPC API
pub ipfs_url: Url,
gateway_url: Option<Url>,
gateway_task: Option<JoinHandle<()>>,
@@ -226,6 +220,9 @@ impl SpherePair {
self.client.workspace.sphere_context().await
}
+ /// Invoke a closure that returns a [Future]. The closure receives the
+ /// client's [SphereContext] as its argument, and polls the returned
+ /// [Future] until it finishes
pub async fn spawn<T, F, Fut>(&self, f: F) -> Result<T>
where
T: Send + 'static,
diff --git a/rust/noosphere-cli/src/native/mod.rs b/rust/noosphere-cli/src/native/mod.rs
index 53313b13b..727304719 100644
--- a/rust/noosphere-cli/src/native/mod.rs
+++ b/rust/noosphere-cli/src/native/mod.rs
@@ -8,6 +8,9 @@ pub mod paths;
pub mod render;
pub mod workspace;
+#[cfg(any(test, feature = "helpers"))]
+pub mod helpers;
+
use anyhow::Result;
use noosphere_core::tracing::initialize_tracing;
diff --git a/rust/noosphere-cli/src/native/render/buffer.rs b/rust/noosphere-cli/src/native/render/buffer.rs
index 65d2afee8..a13a27552 100644
--- a/rust/noosphere-cli/src/native/render/buffer.rs
+++ b/rust/noosphere-cli/src/native/render/buffer.rs
@@ -1,7 +1,7 @@
use anyhow::{anyhow, Result};
use cid::Cid;
+use noosphere_core::context::{AsyncFileBody, SphereFile};
use noosphere_core::data::Did;
-use noosphere_sphere::{AsyncFileBody, SphereFile};
use std::{
collections::{BTreeMap, BTreeSet},
future::Future,
diff --git a/rust/noosphere-cli/src/native/render/job.rs b/rust/noosphere-cli/src/native/render/job.rs
index 3dd8804cf..b6cce6d59 100644
--- a/rust/noosphere-cli/src/native/render/job.rs
+++ b/rust/noosphere-cli/src/native/render/job.rs
@@ -2,11 +2,11 @@
use anyhow::{anyhow, Result};
use cid::Cid;
-use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld};
-use noosphere_sphere::{
+use noosphere_core::context::{
HasSphereContext, SphereContentRead, SphereCursor, SpherePetnameRead, SphereReplicaRead,
SphereWalker,
};
+use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld};
use noosphere_storage::Storage;
use std::{marker::PhantomData, sync::Arc};
use tokio::sync::mpsc::Sender;
diff --git a/rust/noosphere-cli/src/native/render/renderer.rs b/rust/noosphere-cli/src/native/render/renderer.rs
index a777a3394..244c92572 100644
--- a/rust/noosphere-cli/src/native/render/renderer.rs
+++ b/rust/noosphere-cli/src/native/render/renderer.rs
@@ -1,5 +1,5 @@
use anyhow::Result;
-use noosphere_sphere::HasSphereContext;
+use noosphere_core::context::HasSphereContext;
use noosphere_storage::Storage;
use std::{collections::BTreeSet, marker::PhantomData, sync::Arc, thread::available_parallelism};
diff --git a/rust/noosphere-cli/src/native/render/writer.rs b/rust/noosphere-cli/src/native/render/writer.rs
index 01179e0d7..25d4dd955 100644
--- a/rust/noosphere-cli/src/native/render/writer.rs
+++ b/rust/noosphere-cli/src/native/render/writer.rs
@@ -1,7 +1,7 @@
use anyhow::{anyhow, Result};
use cid::Cid;
+use noosphere_core::context::{AsyncFileBody, SphereFile};
use noosphere_core::data::Did;
-use noosphere_sphere::{AsyncFileBody, SphereFile};
use pathdiff::diff_paths;
use std::{
path::{Path, PathBuf},
diff --git a/rust/noosphere-cli/src/native/workspace.rs b/rust/noosphere-cli/src/native/workspace.rs
index abd0a5ea2..0c4680b66 100644
--- a/rust/noosphere-cli/src/native/workspace.rs
+++ b/rust/noosphere-cli/src/native/workspace.rs
@@ -5,8 +5,10 @@ use cid::Cid;
use directories::ProjectDirs;
use noosphere::sphere::SphereContextBuilder;
use noosphere_core::authority::Author;
+use noosphere_core::context::{
+ SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL,
+};
use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld};
-use noosphere_sphere::{SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL};
use noosphere_storage::{KeyValueStore, NativeStorage, SphereDb};
use serde_json::Value;
use std::path::{Path, PathBuf};
diff --git a/rust/noosphere-cli/src/web.rs b/rust/noosphere-cli/src/web.rs
index ba5046fda..93210dfbb 100644
--- a/rust/noosphere-cli/src/web.rs
+++ b/rust/noosphere-cli/src/web.rs
@@ -1 +1,2 @@
+#[allow(missing_docs)]
pub async fn main() {}
diff --git a/rust/noosphere-common/Cargo.toml b/rust/noosphere-common/Cargo.toml
new file mode 100644
index 000000000..43c752f95
--- /dev/null
+++ b/rust/noosphere-common/Cargo.toml
@@ -0,0 +1,34 @@
+[package]
+name = "noosphere-common"
+version = "0.1.0"
+edition = "2021"
+description = "Common, generic utilities that are shared across other Noosphere packages"
+keywords = ["noosphere"]
+categories = []
+rust-version = "1.70.0"
+license = "MIT OR Apache-2.0"
+documentation = "https://docs.rs/noosphere-common"
+repository = "https://github.com/subconsciousnetwork/noosphere"
+homepage = "https://github.com/subconsciousnetwork/noosphere"
+readme = "README.md"
+
+[features]
+helpers = ["rand"]
+
+[dependencies]
+anyhow = { workspace = true }
+tracing = { workspace = true }
+rand = { workspace = true, optional = true }
+futures-util = { workspace = true }
+
+[dev-dependencies]
+rand = { workspace = true }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tokio = { workspace = true, features = ["full"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+tokio = { workspace = true, features = ["sync", "macros"] }
+futures = { workspace = true }
+wasm-bindgen = { workspace = true }
+wasm-bindgen-futures = { workspace = true }
\ No newline at end of file
diff --git a/rust/noosphere-common/README.md b/rust/noosphere-common/README.md
new file mode 100644
index 000000000..2220602e9
--- /dev/null
+++ b/rust/noosphere-common/README.md
@@ -0,0 +1,5 @@
+
+
+# Noosphere Common
+
+Common, generic utilities that are shared across other Noosphere packages.
diff --git a/rust/noosphere-common/src/helpers/mod.rs b/rust/noosphere-common/src/helpers/mod.rs
new file mode 100644
index 000000000..3542c5677
--- /dev/null
+++ b/rust/noosphere-common/src/helpers/mod.rs
@@ -0,0 +1,8 @@
+//! Generic helper utilities intended to be used exclusively in tests and hidden
+//! setup for examples
+
+mod random;
+mod wait;
+
+pub use random::*;
+pub use wait::*;
diff --git a/rust/noosphere-cli/tests/helpers/random.rs b/rust/noosphere-common/src/helpers/random.rs
similarity index 51%
rename from rust/noosphere-cli/tests/helpers/random.rs
rename to rust/noosphere-common/src/helpers/random.rs
index 54bf5cc77..23fd74339 100644
--- a/rust/noosphere-cli/tests/helpers/random.rs
+++ b/rust/noosphere-common/src/helpers/random.rs
@@ -6,6 +6,25 @@ use tokio::sync::Mutex;
/// Its primary purpose is to retain and report the seed in use for a
/// random number generator that can be shared across threads in tests.
/// This is probably not suitable for use outside of tests.
+///
+/// ```rust
+/// # #![cfg(feature = "helpers")]
+/// # use anyhow::Result;
+/// # use rand::Rng;
+/// # use noosphere_common::helpers::TestEntropy;
+/// #
+/// # #[tokio::main(flavor = "multi_thread")]
+/// # async fn main() -> Result<()> {
+/// #
+/// let test_entropy = TestEntropy::default();
+/// let random_int = test_entropy.to_rng().lock().await.gen::<u8>();
+///
+/// let seeded_entropy = TestEntropy::from_seed(test_entropy.seed().clone());
+/// assert_eq!(random_int, seeded_entropy.to_rng().lock().await.gen::<u8>());
+/// #
+/// # Ok(())
+/// # }
+/// ```
pub struct TestEntropy {
seed: [u8; 32],
rng: Arc<Mutex<StdRng>>,
@@ -18,18 +37,22 @@ impl Default for TestEntropy {
}
impl TestEntropy {
+ /// Initialize the [TestEntropy] with an explicit seed
pub fn from_seed(seed: [u8; 32]) -> Self {
tracing::info!(?seed, "Initializing test entropy...");
- let rng = Arc::new(Mutex::new(SeedableRng::from_seed(seed.clone())));
+ let rng = Arc::new(Mutex::new(SeedableRng::from_seed(seed)));
Self { seed, rng }
}
+ /// Get an owned instance of the internal [Rng] initialized by this
+ /// [TestEntropy]
pub fn to_rng(&self) -> Arc<Mutex<StdRng>> {
self.rng.clone()
}
- pub fn seed(&self) -> &[u8] {
+ /// Get the seed used to initialize the internal [Rng] of this [TestEntropy]
+ pub fn seed(&self) -> &[u8; 32] {
&self.seed
}
}
diff --git a/rust/noosphere-common/src/helpers/wait.rs b/rust/noosphere-common/src/helpers/wait.rs
new file mode 100644
index 000000000..2604fa849
--- /dev/null
+++ b/rust/noosphere-common/src/helpers/wait.rs
@@ -0,0 +1,8 @@
+use std::time::Duration;
+
+/// Wait for the specified number of seconds; uses [tokio::time::sleep], so this
+/// will yield to the async runtime rather than block until the sleep time is
+/// elapsed.
+pub async fn wait(seconds: u64) {
+ tokio::time::sleep(Duration::from_secs(seconds)).await;
+}
diff --git a/rust/noosphere-common/src/lib.rs b/rust/noosphere-common/src/lib.rs
new file mode 100644
index 000000000..51b7cebae
--- /dev/null
+++ b/rust/noosphere-common/src/lib.rs
@@ -0,0 +1,16 @@
+//! Common, generic utilities that are shared across other Noosphere packages.
+#![warn(missing_docs)]
+
+#[macro_use]
+extern crate tracing;
+
+mod sync;
+mod task;
+mod unshared;
+
+pub use sync::*;
+pub use task::*;
+pub use unshared::*;
+
+#[cfg(any(test, feature = "helpers"))]
+pub mod helpers;
diff --git a/rust/noosphere-common/src/sync.rs b/rust/noosphere-common/src/sync.rs
new file mode 100644
index 000000000..af6b65bc1
--- /dev/null
+++ b/rust/noosphere-common/src/sync.rs
@@ -0,0 +1,27 @@
+#[allow(missing_docs)]
+#[cfg(not(target_arch = "wasm32"))]
+pub trait ConditionalSend: Send {}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<S> ConditionalSend for S where S: Send {}
+
+#[allow(missing_docs)]
+#[cfg(not(target_arch = "wasm32"))]
+pub trait ConditionalSync: Send + Sync {}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<S> ConditionalSync for S where S: Send + Sync {}
+
+#[allow(missing_docs)]
+#[cfg(target_arch = "wasm32")]
+pub trait ConditionalSend {}
+
+#[cfg(target_arch = "wasm32")]
+impl<S> ConditionalSend for S {}
+
+#[allow(missing_docs)]
+#[cfg(target_arch = "wasm32")]
+pub trait ConditionalSync {}
+
+#[cfg(target_arch = "wasm32")]
+impl<S> ConditionalSync for S {}
diff --git a/rust/noosphere-common/src/task.rs b/rust/noosphere-common/src/task.rs
new file mode 100644
index 000000000..6fc481f4e
--- /dev/null
+++ b/rust/noosphere-common/src/task.rs
@@ -0,0 +1,128 @@
+use anyhow::Result;
+use std::future::Future;
+
+#[cfg(target_arch = "wasm32")]
+use std::pin::Pin;
+
+#[cfg(target_arch = "wasm32")]
+use tokio::sync::oneshot::channel;
+
+#[cfg(target_arch = "wasm32")]
+use futures::future::join_all;
+
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::task::JoinSet;
+
+#[cfg(target_arch = "wasm32")]
+/// Spawn a future by scheduling it with the local executor. The returned
+/// future will be pending until the spawned future completes.
+pub async fn spawn<F>(future: F) -> Result<F::Output>
+where
+ F: Future + 'static,
+ F::Output: Send + 'static,
+{
+ let (tx, rx) = channel();
+
+ wasm_bindgen_futures::spawn_local(async move {
+ if let Err(_) = tx.send(future.await) {
+ warn!("Receiver dropped before spawned task completed");
+ }
+ });
+
+ Ok(rx.await?)
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+/// Spawn a future by scheduling it with the local executor. The returned
+/// future will be pending until the spawned future completes.
+pub async fn spawn<F>(future: F) -> Result<F::Output>
+where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+{
+ Ok(tokio::spawn(future).await?)
+}
+
+/// An aggregator of async work that can be used to observe the moment when all
+/// the aggregated work is completed. It is similar to tokio's [JoinSet], but is
+/// relatively constrained and also works on `wasm32-unknown-unknown`. Unlike
+/// [JoinSet], the results can not be observed individually.
+///
+/// ```rust
+/// # use anyhow::Result;
+/// # use noosphere_common::TaskQueue;
+/// #
+/// # #[tokio::main(flavor = "multi_thread")]
+/// # async fn main() -> Result<()> {
+/// #
+/// let mut task_queue = TaskQueue::default();
+/// for i in 0..10 {
+/// task_queue.spawn(async move {
+/// println!("{}", i);
+/// Ok(())
+/// });
+/// }
+/// task_queue.join().await?;
+/// #
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Default)]
+pub struct TaskQueue {
+ #[cfg(not(target_arch = "wasm32"))]
+ tasks: JoinSet<Result<()>>,
+
+ #[cfg(target_arch = "wasm32")]
+ tasks: Vec<Pin<Box<dyn Future<Output = ()>>>>,
+}
+
+impl TaskQueue {
+ #[cfg(not(target_arch = "wasm32"))]
+ /// Queue a future to be spawned in the local executor. All queued futures will be polled
+ /// to completion before the [TaskQueue] can be joined.
+ pub fn spawn<F>(&mut self, future: F)
+ where
+ F: Future<Output = Result<()>> + Send + 'static,
+ {
+ self.tasks.spawn(future);
+ }
+
+ #[cfg(not(target_arch = "wasm32"))]
+ /// Returns a future that finishes when all queued futures have finished.
+ pub async fn join(&mut self) -> Result<()> {
+ while let Some(result) = self.tasks.join_next().await {
+ trace!("Task completed, {} remaining in queue...", self.tasks.len());
+ result??;
+ }
+ Ok(())
+ }
+
+ #[cfg(target_arch = "wasm32")]
+ /// Queue a future to be spawned in the local executor. All queued futures will be polled
+ /// to completion before the [TaskQueue] can be joined.
+ pub fn spawn<F>(&mut self, future: F)
+ where
+ F: Future<Output = Result<()>> + 'static,
+ {
+ let task_count = self.tasks.len();
+
+ self.tasks.push(Box::pin(async move {
+ if let Err(error) = spawn(future).await {
+ error!("Queued task failed: {:?}", error);
+ }
+ trace!("Task {} completed...", task_count + 1);
+ }));
+ }
+
+ #[cfg(target_arch = "wasm32")]
+ /// Returns a future that finishes when all queued futures have finished.
+ pub async fn join(&mut self) -> Result<()> {
+ let tasks = std::mem::replace(&mut self.tasks, Vec::new());
+
+ debug!("Joining {} queued tasks...", tasks.len());
+
+ join_all(tasks).await;
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-common/src/unshared.rs b/rust/noosphere-common/src/unshared.rs
new file mode 100644
index 000000000..000b9a1c3
--- /dev/null
+++ b/rust/noosphere-common/src/unshared.rs
@@ -0,0 +1,83 @@
+use crate::ConditionalSend;
+use futures_util::Stream;
+
+/// NOTE: This type was adapted from https://github.com/Nullus157/async-compression/blob/main/src/unshared.rs
+/// Original implementation licensed MIT/Apache 2
+///
+/// Wraps a type and only allows unique borrowing, the main usecase is to wrap a `!Sync` type and
+/// implement `Sync` for it as this type blocks having multiple shared references to the inner
+/// value.
+///
+/// # Safety
+///
+/// We must be careful when accessing `inner`, there must be no way to create a shared reference to
+/// it from a shared reference to an `Unshared`, as that would allow creating shared references on
+/// multiple threads.
+///
+/// As an example deriving or implementing `Clone` is impossible, two threads could attempt to
+/// clone a shared `Unshared<T>` reference which would result in accessing the same inner value
+/// concurrently.
+#[repr(transparent)]
+pub struct Unshared<T>(T);
+
+impl<T> Unshared<T> {
+ /// Initialize a new [Unshared], wrapping the provided inner value
+ pub fn new(inner: T) -> Self {
+ Unshared(inner)
+ }
+
+ /// Get a mutable (unique) reference to the inner value
+ pub fn get_mut(&mut self) -> &mut T {
+ &mut self.0
+ }
+}
+
+/// Safety: See comments on main docs for `Unshared`
+unsafe impl<T> Sync for Unshared<T> {}
+
+impl<T> std::fmt::Debug for Unshared<T> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct(core::any::type_name::<T>()).finish()
+ }
+}
+
+/// Wrapper that implements [Stream] for any [Unshared] that happens to wrap an
+/// appropriately bounded [Stream]. This is useful for making a `!Sync` stream
+/// into a `Sync` one in cases where we know it will not be shared by concurrent
+/// actors.
+///
+/// Implementation note: we do not implement [Stream] directly on [Unshared] as
+/// an expression of hygiene; only mutable borrows of the inner value should be
+/// possible in order to preserve the soundness of [Unshared].
+#[repr(transparent)]
+pub struct UnsharedStream<T>(Unshared<T>)
+where
+ T: Stream + Unpin,
+ T::Item: ConditionalSend + 'static;
+
+impl<T> UnsharedStream<T>
+where
+ T: Stream + Unpin,
+ T::Item: ConditionalSend + 'static,
+{
+ /// Initialize a new [UnsharedStream] wrapping a provided (presumably `!Sync`)
+ /// [Stream]
+ pub fn new(inner: T) -> Self {
+ UnsharedStream(Unshared::new(inner))
+ }
+}
+
+impl<T> Stream for UnsharedStream<T>
+where
+ T: Stream + Unpin,
+ T::Item: ConditionalSend + 'static,
+{
+ type Item = T::Item;
+
+ fn poll_next(
+ self: std::pin::Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
+ std::pin::pin!(self.get_mut().0.get_mut()).poll_next(cx)
+ }
+}
diff --git a/rust/noosphere-core/Cargo.toml b/rust/noosphere-core/Cargo.toml
index e9f6c10ff..03a90473b 100644
--- a/rust/noosphere-core/Cargo.toml
+++ b/rust/noosphere-core/Cargo.toml
@@ -19,13 +19,14 @@ homepage = "https://github.com/subconsciousnetwork/noosphere"
readme = "README.md"
[features]
+default = []
sentry = ["dep:sentry-tracing"]
helpers = []
[dependencies]
tracing = { workspace = true }
cid = { workspace = true }
-url = { workspace = true }
+url = { workspace = true, features = ["serde"] }
async-trait = "~0.1"
async-recursion = "^1"
async-stream = { workspace = true }
@@ -33,23 +34,30 @@ async-stream = { workspace = true }
# NOTE: async-once-cell 0.4.0 shipped unstable feature usage
async-once-cell = "~0.4"
anyhow = { workspace = true }
+bytes = { workspace = true }
+iroh-car = { workspace = true }
thiserror = { workspace = true }
fastcdc = { workspace = true }
-futures = "~0.3"
+futures = { workspace = true }
+futures-util = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
+serde_urlencoded = { workspace = true }
byteorder = "^1.4"
base64 = "0.21"
ed25519-zebra = "^3"
-rand = "~0.8"
+rand = { workspace = true }
once_cell = "^1"
tiny-bip39 = "^1"
tokio-stream = { workspace = true }
libipld-core = { workspace = true }
libipld-cbor = { workspace = true }
+reqwest = { workspace = true }
strum = { workspace = true }
strum_macros = { workspace = true }
+tokio-util = { workspace = true, features = ["io"] }
+noosphere-common = { version = "0.1.0", path = "../noosphere-common" }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
noosphere-collections = { version = "0.6.3", path = "../noosphere-collections" }
@@ -60,14 +68,26 @@ sentry-tracing = { workspace = true, optional = true }
[dev-dependencies]
wasm-bindgen-test = { workspace = true }
serde_bytes = "~0.11"
+noosphere-common = { version = "0.1.0", path = "../noosphere-common", features = ["helpers"] }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
tokio = { workspace = true, features = ["full"] }
tracing-subscriber = { workspace = true }
+axum = { workspace = true, features = ["headers", "macros"] }
[target.'cfg(target_arch = "wasm32")'.dependencies]
# NOTE: This is needed so that rand can be included in WASM builds
getrandom = { version = "~0.2", features = ["js"] }
+gloo-net = { workspace = true }
+wasm-streams = { workspace = true }
+wasm-bindgen = { workspace = true }
+js-sys = { workspace = true }
tokio = { workspace = true, features = ["sync", "macros"] }
console_error_panic_hook = "0.1"
tracing-wasm = "~0.2"
+
+[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
+workspace = true
+features = [
+ "ReadableStream"
+]
\ No newline at end of file
diff --git a/rust/noosphere-core/src/api/client.rs b/rust/noosphere-core/src/api/client.rs
new file mode 100644
index 000000000..fe2f929af
--- /dev/null
+++ b/rust/noosphere-core/src/api/client.rs
@@ -0,0 +1,498 @@
+use std::str::FromStr;
+
+use crate::{
+ api::{route::RouteUrl, v0alpha1, v0alpha2},
+ stream::{from_car_stream, memo_history_stream, put_block_stream, to_car_stream},
+};
+
+use anyhow::{anyhow, Result};
+use async_stream::try_stream;
+use bytes::Bytes;
+use cid::Cid;
+use iroh_car::CarReader;
+use libipld_cbor::DagCborCodec;
+use noosphere_common::{ConditionalSend, ConditionalSync, UnsharedStream};
+
+use crate::{
+ authority::{generate_capability, Author, SphereAbility, SphereReference},
+ data::{Link, MemoIpld},
+};
+use noosphere_storage::{block_deserialize, block_serialize, BlockStore};
+use reqwest::{header::HeaderMap, StatusCode};
+use tokio_stream::{Stream, StreamExt};
+use tokio_util::io::StreamReader;
+use ucan::{
+ builder::UcanBuilder,
+ capability::CapabilityView,
+ crypto::{did::DidParser, KeyMaterial},
+ store::{UcanJwtStore, UcanStore},
+ ucan::Ucan,
+};
+use url::Url;
+
+#[cfg(doc)]
+use crate::data::Did;
+
+/// A [Client] is a simple, portable HTTP client for the Noosphere gateway REST
+/// API. It embodies the intended usage of the REST API, which includes an
+/// opening handshake (with associated key verification) and various
+/// UCAN-authorized verbs over sphere data.
+pub struct Client<K, S>
+where
+ K: KeyMaterial + Clone + 'static,
+ S: UcanStore + BlockStore + 'static,
+{
+ /// The [v0alpha1::IdentifyResponse] that was received from the gateway when
+ /// the [Client] was initialized
+ pub session: v0alpha1::IdentifyResponse,
+
+ /// The [Did] of the sphere represented by this [Client]
+ pub sphere_identity: String,
+
+ /// The [Url] for the gateway API being used by this [Client]
+ pub api_base: Url,
+
+ /// The [Author] that is wielding this [Client] to interact with the gateway API
+ pub author: Author<K>,
+
+ /// The backing [BlockStore] (also used as a [UcanStore]) for this [Client]
+ pub store: S,
+
+ client: reqwest::Client,
+}
+
+impl<K, S> Client<K, S>
+where
+ K: KeyMaterial + Clone + 'static,
+ S: UcanStore + BlockStore + 'static,
+{
+ /// Initialize the [Client] by perfoming an "identification" handshake with
+ /// a gateway whose API presumably lives at the specified URL. The request
+ /// is authorized (so the provided [Author] must have the appropriate
+ /// credentials), and the gateway responds with a
+ /// [v0alpha1::IdentifyResponse] to verify its own credentials for the
+ /// client.
+ pub async fn identify(
+ sphere_identity: &str,
+ api_base: &Url,
+ author: &Author<K>,
+ did_parser: &mut DidParser,
+ store: S,
+ ) -> Result<Client<K, S>> {
+ debug!("Initializing Noosphere API client");
+ debug!("Client represents sphere {}", sphere_identity);
+ debug!("Client targetting API at {}", api_base);
+
+ let client = reqwest::Client::new();
+
+ let mut url = api_base.clone();
+ url.set_path(&v0alpha1::Route::Did.to_string());
+
+ let did_response = client.get(url).send().await?;
+
+ match did_response.status() {
+ StatusCode::OK => (),
+ _ => return Err(anyhow!("Unable to look up gateway identity")),
+ };
+
+ let gateway_identity = did_response.text().await?;
+
+ let mut url = api_base.clone();
+ url.set_path(&v0alpha1::Route::Identify.to_string());
+
+ let (jwt, ucan_headers) = Self::make_bearer_token(
+ &gateway_identity,
+ author,
+ &generate_capability(sphere_identity, SphereAbility::Fetch),
+ &store,
+ )
+ .await?;
+
+ let identify_response: v0alpha1::IdentifyResponse = client
+ .get(url)
+ .bearer_auth(jwt)
+ .headers(ucan_headers)
+ .send()
+ .await?
+ .json()
+ .await?;
+
+ identify_response.verify(did_parser, &store).await?;
+
+ debug!(
+ "Handshake succeeded with gateway {}",
+ identify_response.gateway_identity
+ );
+
+ Ok(Client {
+ session: identify_response,
+ sphere_identity: sphere_identity.into(),
+ api_base: api_base.clone(),
+ author: author.clone(),
+ store,
+ client,
+ })
+ }
+
+ async fn make_bearer_token(
+ gateway_identity: &str,
+ author: &Author<K>,
+ capability: &CapabilityView<SphereReference, SphereAbility>,
+ store: &S,
+ ) -> Result<(String, HeaderMap)> {
+ let mut signable = UcanBuilder::default()
+ .issued_by(&author.key)
+ .for_audience(gateway_identity)
+ .with_lifetime(120)
+ .claiming_capability(capability)
+ .with_nonce()
+ .build()?;
+
+ let mut ucan_headers = HeaderMap::new();
+
+ let authorization = author.require_authorization()?;
+ let authorization_cid = Cid::try_from(authorization)?;
+
+ match authorization.as_ucan(store).await {
+ Ok(ucan) => {
+ if let Some(ucan_proofs) = ucan.proofs() {
+ // TODO(ucan-wg/rs-ucan#37): We should integrate a helper for this kind of stuff into rs-ucan
+ let mut proofs_to_search: Vec<String> = ucan_proofs.clone();
+
+ debug!("Making bearer token... {:?}", proofs_to_search);
+
+ while let Some(cid_string) = proofs_to_search.pop() {
+ let cid = Cid::from_str(cid_string.as_str())?;
+ let jwt = store.require_token(&cid).await?;
+ let ucan = Ucan::from_str(&jwt)?;
+
+ debug!("Adding UCAN header for {}", cid);
+
+ if let Some(ucan_proofs) = ucan.proofs() {
+ proofs_to_search.extend(ucan_proofs.clone().into_iter());
+ }
+
+ ucan_headers.append("ucan", format!("{cid} {jwt}").parse()?);
+ }
+ }
+
+ ucan_headers.append(
+ "ucan",
+ format!("{} {}", authorization_cid, ucan.encode()?).parse()?,
+ );
+ }
+ _ => {
+ debug!(
+ "Unable to resolve authorization to a UCAN; it will be used as a blind proof"
+ )
+ }
+ };
+
+ // TODO(ucan-wg/rs-ucan#32): This is kind of a hack until we can add proofs by CID
+ signable
+ .proofs
+ .push(Cid::try_from(authorization)?.to_string());
+
+ let jwt = signable.sign().await?.encode()?;
+
+ // TODO: It is inefficient to send the same UCANs with every request,
+ // we should probably establish a conventional flow for syncing UCANs
+ // this way only once when pairing a gateway. For now, this is about the
+ // same efficiency as what we had before when UCANs were all inlined to
+ // a single token.
+ Ok((jwt, ucan_headers))
+ }
+
+ /// Replicate content from Noosphere, streaming its blocks from the
+ /// configured gateway. If the gateway doesn't have the desired content, it
+ /// will look it up from other sources such as IPFS if they are available.
+ /// Note that this means this call can potentially block on upstream
+ /// access to an IPFS node (which, depending on the node's network
+ /// configuration and peering status, can be quite slow).
+ pub async fn replicate(
+ &self,
+ memo_version: &Cid,
+ params: Option<&v0alpha1::ReplicateParameters>,
+ ) -> Result<impl Stream<Item = Result<(Cid, Vec<u8>)>>> {
+ let url = Url::try_from(RouteUrl(
+ &self.api_base,
+ v0alpha1::Route::Replicate(Some(*memo_version)),
+ params,
+ ))?;
+
+ debug!("Client replicating {} from {}", memo_version, url);
+
+ let capability = generate_capability(&self.sphere_identity, SphereAbility::Fetch);
+
+ let (token, ucan_headers) = Self::make_bearer_token(
+ &self.session.gateway_identity,
+ &self.author,
+ &capability,
+ &self.store,
+ )
+ .await?;
+
+ let response = self
+ .client
+ .get(url)
+ .bearer_auth(token)
+ .headers(ucan_headers)
+ .send()
+ .await?;
+
+ Ok(
+ CarReader::new(StreamReader::new(response.bytes_stream().map(
+ |item| match item {
+ Ok(item) => Ok(item),
+ Err(error) => {
+ error!("Failed to read CAR stream: {}", error);
+ Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe))
+ }
+ },
+ )))
+ .await?
+ .stream()
+ .map(|block| match block {
+ Ok(block) => Ok(block),
+ Err(error) => Err(anyhow!(error)),
+ }),
+ )
+ }
+
+ /// Fetch the latest, canonical history of the client's sphere from the
+ /// gateway, which serves as the aggregation point for history across many
+ /// clients.
+ pub async fn fetch(
+ &self,
+ params: &v0alpha1::FetchParameters,
+ ) -> Result<Option<(Link<MemoIpld>, impl Stream<Item = Result<(Cid, Vec<u8>)>>)>> {
+ let url = Url::try_from(RouteUrl(
+ &self.api_base,
+ v0alpha1::Route::Fetch,
+ Some(params),
+ ))?;
+
+ debug!("Client fetching blocks from {}", url);
+
+ let capability = generate_capability(&self.sphere_identity, SphereAbility::Fetch);
+ let (token, ucan_headers) = Self::make_bearer_token(
+ &self.session.gateway_identity,
+ &self.author,
+ &capability,
+ &self.store,
+ )
+ .await?;
+
+ let response = self
+ .client
+ .get(url)
+ .bearer_auth(token)
+ .headers(ucan_headers)
+ .send()
+ .await?;
+
+ let reader = CarReader::new(StreamReader::new(response.bytes_stream().map(
+ |item| match item {
+ Ok(item) => Ok(item),
+ Err(error) => {
+ error!("Failed to read CAR stream: {}", error);
+ Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe))
+ }
+ },
+ )))
+ .await?;
+
+ let tip = reader.header().roots().first().cloned();
+
+ if let Some(tip) = tip {
+ Ok(match tip.codec() {
+ // Identity codec = no changes
+ 0 => None,
+ _ => Some((
+ tip.into(),
+ reader.stream().map(|block| match block {
+ Ok(block) => Ok(block),
+ Err(error) => Err(anyhow!(error)),
+ }),
+ )),
+ })
+ } else {
+ Ok(None)
+ }
+ }
+
+ fn make_push_request_stream(
+ store: S,
+ push_body: v0alpha2::PushBody,
+ ) -> impl Stream<Item = Result<Bytes, std::io::Error>> + ConditionalSync + 'static {
+ let root = push_body.local_tip.clone().into();
+ trace!("Creating push stream...");
+
+ let block_stream = try_stream! {
+
+ let history_stream = memo_history_stream(
+ store,
+ &push_body.local_tip,
+ push_body.local_base.as_ref(),
+ true
+ );
+
+ yield block_serialize::<DagCborCodec, _>(push_body)?;
+
+ for await item in history_stream {
+ yield item?;
+ };
+ };
+
+ // Safety: this stream is not shared by us, or by its consumer (reqwest
+ // on native targets, gloo-net on web) to others; the [Unshared] is required
+ // in order for the wrapped [Stream] to satisfy a `Sync` bound.
+ // See: https://github.com/seanmonstar/reqwest/issues/1969
+ UnsharedStream::new(Box::pin(to_car_stream(vec![root], block_stream)))
+ }
+
+ #[cfg(target_arch = "wasm32")]
+ async fn make_push_request(
+ &self,
+ url: Url,
+ ucan_headers: HeaderMap,
+ token: &str,
+ push_body: &v0alpha2::PushBody,
+ ) -> Result<impl Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend, v0alpha2::PushError>
+ {
+ // Implementation note: currently reqwest does not support streaming
+ // request bodies under wasm32 targets even though it is technically
+ // feasiable via [ReadableStream]. So, we jury rig a one-off streaming
+ // request here using wasm-bindgen and wasm-streams:
+
+ use gloo_net::http::Headers;
+ use gloo_net::http::Method;
+ use gloo_net::http::RequestBuilder;
+ use js_sys::{JsString, Uint8Array};
+ use wasm_bindgen::JsValue;
+ use wasm_streams::ReadableStream;
+
+ let headers = Headers::new();
+ headers.append("Authorization", &format!("Bearer {}", token));
+
+ for (name, value) in ucan_headers {
+ if let (Some(name), Ok(value)) = (name, value.to_str()) {
+ headers.append(name.as_str(), value);
+ }
+ }
+
+ let stream = Self::make_push_request_stream(self.store.clone(), push_body.clone());
+
+ let readable_stream = ReadableStream::from_stream(stream.map(|result| match result {
+ Ok(bytes) => Ok(JsValue::from(Uint8Array::from(bytes.as_ref()))),
+ Err(error) => Err(JsValue::from(JsString::from(error.to_string()))),
+ }));
+
+ let request = RequestBuilder::new(url.as_str())
+ .method(Method::PUT)
+ .headers(headers)
+ .body(JsValue::from(readable_stream.as_raw()))
+ .map_err(|error| v0alpha2::PushError::Internal(Some(error.to_string())))?;
+
+ let response = request
+ .send()
+ .await
+ .map_err(|error| v0alpha2::PushError::Internal(Some(error.to_string())))?;
+
+ let body_stream = response
+ .body()
+ .ok_or_else(|| v0alpha2::PushError::UnexpectedBody)?;
+ let body_stream = ReadableStream::from_raw(wasm_bindgen::JsCast::unchecked_into::<
+ wasm_streams::readable::sys::ReadableStream,
+ >(JsValue::from(body_stream)));
+
+ let car_stream = body_stream.into_stream().map(|result| match result {
+ Ok(value) => match Uint8Array::try_from(value) {
+ Ok(array) => Ok(Bytes::from(array.to_vec())),
+ Err(_) => Err(std::io::Error::new(
+ std::io::ErrorKind::Other,
+ v0alpha2::PushError::Internal(None),
+ )),
+ },
+ Err(error) => Err(std::io::Error::new(
+ std::io::ErrorKind::Other,
+ error.as_string().unwrap_or_default(),
+ )),
+ });
+
+ Ok(from_car_stream(car_stream))
+ }
+
+ #[cfg(not(target_arch = "wasm32"))]
+ async fn make_push_request(
+ &self,
+ url: Url,
+ ucan_headers: HeaderMap,
+ token: &str,
+ push_body: &v0alpha2::PushBody,
+ ) -> Result<impl Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend, v0alpha2::PushError>
+ {
+ use reqwest::Body;
+
+ let stream = Self::make_push_request_stream(self.store.clone(), push_body.clone());
+
+ let response = self
+ .client
+ .put(url)
+ .bearer_auth(token)
+ .headers(ucan_headers)
+ .header("Content-Type", "application/octet-stream")
+ .body(Body::wrap_stream(stream))
+ .send()
+ .await
+ .map_err(|err| v0alpha2::PushError::from(anyhow!(err)))?;
+
+ trace!("Checking response...");
+ if response.status() == StatusCode::CONFLICT {
+ return Err(v0alpha2::PushError::Conflict);
+ }
+
+ trace!("Fielding response...");
+
+ Ok(from_car_stream(response.bytes_stream()))
+ }
+
+ /// Push the latest local history of this client to the gateway
+ pub async fn push(
+ &self,
+ push_body: &v0alpha2::PushBody,
+ ) -> Result<v0alpha2::PushResponse, v0alpha2::PushError> {
+ let url = Url::try_from(RouteUrl::<_, ()>(
+ &self.api_base,
+ v0alpha2::Route::Push,
+ None,
+ ))?;
+ debug!(
+ "Client pushing changes for sphere {} to {}",
+ push_body.sphere, url
+ );
+ let capability = generate_capability(&self.sphere_identity, SphereAbility::Push);
+ let (token, ucan_headers) = Self::make_bearer_token(
+ &self.session.gateway_identity,
+ &self.author,
+ &capability,
+ &self.store,
+ )
+ .await?;
+
+ let block_stream = self
+ .make_push_request(url, ucan_headers, &token, push_body)
+ .await?;
+
+ tokio::pin!(block_stream);
+
+ let push_response = match block_stream.try_next().await? {
+ Some((_, bytes)) => block_deserialize::<DagCborCodec, v0alpha2::PushResponse>(&bytes)?,
+ _ => return Err(v0alpha2::PushError::UnexpectedBody),
+ };
+
+ put_block_stream(self.store.clone(), block_stream).await?;
+
+ Ok(push_response)
+ }
+}
diff --git a/rust/noosphere-core/src/api/data.rs b/rust/noosphere-core/src/api/data.rs
new file mode 100644
index 000000000..0f845cbb8
--- /dev/null
+++ b/rust/noosphere-core/src/api/data.rs
@@ -0,0 +1,35 @@
+use std::str::FromStr;
+
+use anyhow::Result;
+
+use serde::{Deserialize, Deserializer};
+
+/// A helper to express the deserialization of a query string to some
+/// consistent result type
+pub trait AsQuery {
+ /// Get the value of this trait implementor as a [Result<Option<String>>]
+ fn as_query(&self) -> Result<Option<String>>;
+}
+
+impl AsQuery for () {
+ fn as_query(&self) -> Result<Option<String>> {
+ Ok(None)
+ }
+}
+
+// NOTE: Adapted from https://github.com/tokio-rs/axum/blob/7caa4a3a47a31c211d301f3afbc518ea2c07b4de/examples/query-params-with-empty-strings/src/main.rs#L42-L54
+/// Serde deserialization decorator to map empty Strings to None,
+pub(crate) fn empty_string_as_none<'de, D, T>(de: D) -> Result<Option<T>, D::Error>
+where
+ D: Deserializer<'de>,
+ T: FromStr,
+ T::Err: std::fmt::Display,
+{
+ let opt = Option::<String>::deserialize(de)?;
+ match opt.as_deref() {
+ None | Some("") => Ok(None),
+ Some(s) => FromStr::from_str(s)
+ .map_err(serde::de::Error::custom)
+ .map(Some),
+ }
+}
diff --git a/rust/noosphere-core/src/api/mod.rs b/rust/noosphere-core/src/api/mod.rs
new file mode 100644
index 000000000..2973c1d19
--- /dev/null
+++ b/rust/noosphere-core/src/api/mod.rs
@@ -0,0 +1,13 @@
+//! This module contains data structures and client implementation for working
+//! with the REST API exposed by Noosphere Gateways.
+
+mod client;
+mod data;
+mod route;
+
+pub use client::*;
+pub use data::*;
+pub use route::*;
+
+pub mod v0alpha1;
+pub mod v0alpha2;
diff --git a/rust/noosphere-core/src/api/route.rs b/rust/noosphere-core/src/api/route.rs
new file mode 100644
index 000000000..9125b3113
--- /dev/null
+++ b/rust/noosphere-core/src/api/route.rs
@@ -0,0 +1,54 @@
+use anyhow::Result;
+use std::fmt::Display;
+
+use url::Url;
+
+use crate::api::data::AsQuery;
+
+/// A helper macro to quickly implement a common [Display] format for
+/// Noosphere Gateway REST API routes
+#[macro_export]
+macro_rules! route_display {
+ ($routes:ty) => {
+ impl std::fmt::Display for $routes {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "/api/{}/{}", self.api_version(), self.to_fragment())
+ }
+ }
+ };
+}
+
+/// A helper trait implemented by any route enum that enables it to be easily
+/// serialized as part of a URL
+pub trait RouteSignature: Display {
+ /// Produces the path fragment for a given route
+ fn to_fragment(&self) -> String;
+ /// Gets the API version for the given route
+ fn api_version(&self) -> &str;
+}
+
+/// The [RouteUrl] is a helper to produce a [Url] from any implementor of [RouteSignature], but specifically
+/// the enums found in the [crate::v0alpha1] and [crate::v0alpha2] modules.
+pub struct RouteUrl<'a, 'b, Route: RouteSignature, Params: AsQuery = ()>(
+ pub &'a Url,
+ pub Route,
+ pub Option<&'b Params>,
+);
+
+impl<'a, 'b, Route: RouteSignature, Params: AsQuery> TryFrom<RouteUrl<'a, 'b, Route, Params>>
+ for Url
+{
+ type Error = anyhow::Error;
+
+ fn try_from(value: RouteUrl<'a, 'b, Route, Params>) -> Result<Self, Self::Error> {
+ let RouteUrl(api_base, route, params) = value;
+ let mut url = api_base.clone();
+ url.set_path(&route.to_string());
+ if let Some(params) = params {
+ url.set_query(params.as_query()?.as_deref());
+ } else {
+ url.set_query(None);
+ }
+ Ok(url)
+ }
+}
diff --git a/rust/noosphere-core/src/api/v0alpha1/data.rs b/rust/noosphere-core/src/api/v0alpha1/data.rs
new file mode 100644
index 000000000..6ae446a2b
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha1/data.rs
@@ -0,0 +1,243 @@
+use std::fmt::Display;
+
+use crate::api::data::{empty_string_as_none, AsQuery};
+use crate::{
+ authority::{generate_capability, SphereAbility, SPHERE_SEMANTICS},
+ data::{Bundle, Did, Jwt, Link, MemoIpld},
+ error::NoosphereError,
+};
+use anyhow::{anyhow, Result};
+use cid::Cid;
+use noosphere_storage::{base64_decode, base64_encode};
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use ucan::{
+ chain::ProofChain,
+ crypto::{did::DidParser, KeyMaterial},
+ store::UcanStore,
+ Ucan,
+};
+
+/// The query parameters expected for the "replicate" API route
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ReplicateParameters {
+ /// This is the last revision of the content that is being fetched that is
+ /// already fully available to the caller of the API
+ #[serde(default, deserialize_with = "empty_string_as_none")]
+ pub since: Option<Link<MemoIpld>>,
+}
+
+impl AsQuery for ReplicateParameters {
+ fn as_query(&self) -> Result<Option<String>> {
+ Ok(self.since.as_ref().map(|since| format!("since={since}")))
+ }
+}
+
+/// The query parameters expected for the "fetch" API route
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FetchParameters {
+ /// This is the last revision of the "counterpart" sphere that is managed
+ /// by the API host that the client is fetching from
+ #[serde(default, deserialize_with = "empty_string_as_none")]
+ pub since: Option<Link<MemoIpld>>,
+}
+
+impl AsQuery for FetchParameters {
+ fn as_query(&self) -> Result<Option<String>> {
+ Ok(self.since.as_ref().map(|since| format!("since={since}")))
+ }
+}
+
+/// The possible responses from the "fetch" API route
+#[derive(Debug, Serialize, Deserialize)]
+pub enum FetchResponse {
+ /// There are new revisions to the local and "counterpart" spheres to sync
+ /// with local history
+ NewChanges {
+ /// The tip of the "counterpart" sphere that is managed by the API host
+ /// that the client is fetching from
+ tip: Cid,
+ },
+ /// There are no new revisions since the revision specified in the initial
+ /// fetch request
+ UpToDate,
+}
+
+/// The body payload expected by the "push" API route
+#[derive(Debug, Serialize, Deserialize)]
+pub struct PushBody {
+ /// The DID of the local sphere whose revisions are being pushed
+ pub sphere: Did,
+ /// The base revision represented by the payload being pushed; if the
+ /// entire history is being pushed, then this should be None
+ pub local_base: Option<Link<MemoIpld>>,
+ /// The tip of the history represented by the payload being pushed
+ pub local_tip: Link<MemoIpld>,
+ /// The last received tip of the counterpart sphere
+ pub counterpart_tip: Option<Link<MemoIpld>>,
+ /// A bundle of all the blocks needed to hydrate the revisions from the
+ /// base to the tip of history as represented by this payload
+ pub blocks: Bundle,
+ /// An optional name record to publish to the Noosphere Name System
+ pub name_record: Option<Jwt>,
+}
+
+/// The possible responses from the "push" API route
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum PushResponse {
+ /// The new history was accepted
+ Accepted {
+ /// This is the new tip of the "counterpart" sphere after accepting
+ /// the latest history from the local sphere. This is guaranteed to be
+ /// at least one revision ahead of the latest revision being tracked
+ /// by the client (because it points to the newly received tip of the
+ /// local sphere's history)
+ new_tip: Link<MemoIpld>,
+ /// The blocks needed to hydrate the revisions of the "counterpart"
+ /// sphere history to the tip represented in this response
+ blocks: Bundle,
+ },
+ /// The history was already known by the API host, so no changes were made
+ NoChange,
+}
+
+/// Error types for typical "push" API failure conditions
+#[derive(Error, Debug)]
+pub enum PushError {
+ #[allow(missing_docs)]
+ #[error("Pushed history conflicts with canonical history")]
+ Conflict,
+ #[allow(missing_docs)]
+ #[error("Missing some implied history")]
+ MissingHistory,
+ #[allow(missing_docs)]
+ #[error("Replica is up to date")]
+ UpToDate,
+ #[allow(missing_docs)]
+ #[error("Internal error")]
+ Internal(anyhow::Error),
+}
+
+impl From<NoosphereError> for PushError {
+ fn from(error: NoosphereError) -> Self {
+ error.into()
+ }
+}
+
+impl From<anyhow::Error> for PushError {
+ fn from(value: anyhow::Error) -> Self {
+ PushError::Internal(value)
+ }
+}
+
+impl From<PushError> for StatusCode {
+ fn from(error: PushError) -> Self {
+ match error {
+ PushError::Conflict => StatusCode::CONFLICT,
+ PushError::MissingHistory => StatusCode::UNPROCESSABLE_ENTITY,
+ PushError::UpToDate => StatusCode::BAD_REQUEST,
+ PushError::Internal(error) => {
+ error!("Internal: {:?}", error);
+ StatusCode::INTERNAL_SERVER_ERROR
+ }
+ }
+ }
+}
+
+/// The response from the "identify" API route; this is a signed response that
+/// allows the client to verify the authority of the API host
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IdentifyResponse {
+ /// The DID of the API host
+ pub gateway_identity: Did,
+ /// The DID of the "counterpart" sphere
+ pub sphere_identity: Did,
+ /// The signature of the API host over this payload, as base64-encoded bytes
+ pub signature: String,
+ /// The proof that the API host was authorized by the "counterpart" sphere
+ /// in the form of a UCAN JWT
+ pub proof: String,
+}
+
+impl IdentifyResponse {
+ /// Create and sign an [IdentifyResponse] with the provided credential
+ pub async fn sign<K>(sphere_identity: &str, key: &K, proof: &Ucan) -> Result<Self>
+ where
+ K: KeyMaterial,
+ {
+ let gateway_identity = Did(key.get_did().await?);
+ let signature = base64_encode(
+ &key.sign(&[gateway_identity.as_bytes(), sphere_identity.as_bytes()].concat())
+ .await?,
+ )?;
+ Ok(IdentifyResponse {
+ gateway_identity,
+ sphere_identity: sphere_identity.into(),
+ signature,
+ proof: proof.encode()?,
+ })
+ }
+
+ /// Compare one [IdentifyResponse] with another to verify that they refer to
+ /// the same gateway
+ pub fn shares_identity_with(&self, other: &IdentifyResponse) -> bool {
+ self.gateway_identity == other.gateway_identity
+ && self.sphere_identity == other.sphere_identity
+ }
+
+ /// Verifies that the signature scheme on the payload. The signature is made
+ /// by signing the bytes of the gateway's key DID plus the bytes of the
+ /// sphere DID that the gateway claims to manage. Remember: this sphere is
+ /// not the user's sphere, but rather the "counterpart" sphere created and
+ /// modified by the gateway. Additionally, a proof is given that the gateway
+ /// has been authorized to modify its own sphere.
+ ///
+ /// This verification is intended to check two things:
+ ///
+ /// 1. The gateway has control of the key that it represents with its DID
+ /// 2. The gateway is authorized to modify the sphere it claims to manage
+ pub async fn verify<S: UcanStore>(&self, did_parser: &mut DidParser, store: &S) -> Result<()> {
+ let gateway_key = did_parser.parse(&self.gateway_identity)?;
+ let payload_bytes = [
+ self.gateway_identity.as_bytes(),
+ self.sphere_identity.as_bytes(),
+ ]
+ .concat();
+ let signature_bytes = base64_decode(&self.signature)?;
+
+ // Verify that the signature is valid
+ gateway_key.verify(&payload_bytes, &signature_bytes).await?;
+
+ let proof = ProofChain::try_from_token_string(&self.proof, None, did_parser, store).await?;
+
+ if proof.ucan().audience() != self.gateway_identity.as_str() {
+ return Err(anyhow!("Wrong audience!"));
+ }
+
+ let capability = generate_capability(&self.sphere_identity, SphereAbility::Push);
+ let capability_infos = proof.reduce_capabilities(&SPHERE_SEMANTICS);
+
+ for capability_info in capability_infos {
+ if capability_info.capability.enables(&capability)
+ && capability_info
+ .originators
+ .contains(self.sphere_identity.as_str())
+ {
+ return Ok(());
+ }
+ }
+
+ Err(anyhow!("Not authorized!"))
+ }
+}
+
+impl Display for IdentifyResponse {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "((Gateway {}), (Sphere {}))",
+ self.gateway_identity, self.sphere_identity
+ )
+ }
+}
diff --git a/rust/noosphere-core/src/api/v0alpha1/mod.rs b/rust/noosphere-core/src/api/v0alpha1/mod.rs
new file mode 100644
index 000000000..030aacfc0
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha1/mod.rs
@@ -0,0 +1,6 @@
+//! The original, alpha version of the Noosphere Gateway REST API
+mod data;
+mod route;
+
+pub use data::*;
+pub use route::*;
diff --git a/rust/noosphere-core/src/api/v0alpha1/route.rs b/rust/noosphere-core/src/api/v0alpha1/route.rs
new file mode 100644
index 000000000..2513d4008
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha1/route.rs
@@ -0,0 +1,41 @@
+use crate::api::route::RouteSignature;
+use crate::route_display;
+use cid::Cid;
+
+/// The version of the API represented by this module
+pub const API_VERSION: &str = "v0alpha1";
+
+/// An enum whose variants represent all of the routes in this version of the API
+pub enum Route {
+ /// Fetch the latest canonical history of a sphere from the gateway
+ Fetch,
+ /// Push the latest local history of a sphere from a client to the gateway
+ Push,
+ /// Get the DID of the gateway
+ Did,
+ /// Get a signed verification of the gateway's credentials
+ Identify,
+ /// Replicate content from the broader Noosphere network
+ Replicate(Option<Cid>),
+}
+
+route_display!(Route);
+
+impl RouteSignature for Route {
+ fn to_fragment(&self) -> String {
+ match self {
+ Route::Fetch => "fetch".into(),
+ Route::Push => "push".into(),
+ Route::Did => "did".into(),
+ Route::Identify => "identify".into(),
+ Route::Replicate(cid) => match cid {
+ Some(cid) => format!("replicate/{cid}"),
+ None => "replicate/:memo".into(),
+ },
+ }
+ }
+
+ fn api_version(&self) -> &str {
+ API_VERSION
+ }
+}
diff --git a/rust/noosphere-core/src/api/v0alpha2/data.rs b/rust/noosphere-core/src/api/v0alpha2/data.rs
new file mode 100644
index 000000000..2d5aa0fd1
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha2/data.rs
@@ -0,0 +1,83 @@
+use crate::{
+ data::{Did, Jwt, Link, MemoIpld},
+ error::NoosphereError,
+};
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+/// The body payload expected by the "push" API route
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PushBody {
+ /// The DID of the local sphere whose revisions are being pushed
+ pub sphere: Did,
+ /// The base revision represented by the payload being pushed; if the
+ /// entire history is being pushed, then this should be None
+ pub local_base: Option<Link<MemoIpld>>,
+ /// The tip of the history represented by the payload being pushed
+ pub local_tip: Link<MemoIpld>,
+ /// The last received tip of the counterpart sphere
+ pub counterpart_tip: Option<Link<MemoIpld>>,
+ /// An optional name record to publish to the Noosphere Name System
+ pub name_record: Option<Jwt>,
+}
+
+/// The possible responses from the "push" API route
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum PushResponse {
+ /// The new history was accepted
+ Accepted {
+ /// This is the new tip of the "counterpart" sphere after accepting
+ /// the latest history from the local sphere. This is guaranteed to be
+ /// at least one revision ahead of the latest revision being tracked
+ /// by the client (because it points to the newly received tip of the
+ /// local sphere's history)
+ new_tip: Link<MemoIpld>,
+ },
+ /// The history was already known by the API host, so no changes were made
+ NoChange,
+}
+
+/// Error types for typical "push" API failure conditions
+#[derive(Serialize, Deserialize, Error, Debug)]
+pub enum PushError {
+ #[allow(missing_docs)]
+ #[error("First block in upstream was missing or unexpected type")]
+ UnexpectedBody,
+ #[allow(missing_docs)]
+ #[error("Pushed history conflicts with canonical history")]
+ Conflict,
+ #[allow(missing_docs)]
+ #[error("Missing some implied history")]
+ MissingHistory,
+ #[allow(missing_docs)]
+ #[error("Replica is up to date")]
+ UpToDate,
+ #[allow(missing_docs)]
+ #[error("Internal error: {0:?}")]
+ Internal(Option<String>),
+}
+
+impl From<&PushError> for StatusCode {
+ fn from(value: &PushError) -> Self {
+ match value {
+ PushError::UnexpectedBody => StatusCode::UNPROCESSABLE_ENTITY,
+ PushError::Conflict => StatusCode::CONFLICT,
+ PushError::MissingHistory => StatusCode::FAILED_DEPENDENCY,
+ PushError::UpToDate => StatusCode::NOT_MODIFIED,
+ PushError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
+ }
+ }
+}
+
+impl From<NoosphereError> for PushError {
+ fn from(error: NoosphereError) -> Self {
+ error.into()
+ }
+}
+
+impl From<anyhow::Error> for PushError {
+ fn from(value: anyhow::Error) -> Self {
+ PushError::Internal(Some(format!("{value}")))
+ }
+}
diff --git a/rust/noosphere-core/src/api/v0alpha2/mod.rs b/rust/noosphere-core/src/api/v0alpha2/mod.rs
new file mode 100644
index 000000000..dcd1ff84b
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha2/mod.rs
@@ -0,0 +1,6 @@
+//! The first major revision of the alpha Noosphere Gateway REST API
+mod data;
+mod route;
+
+pub use data::*;
+pub use route::*;
diff --git a/rust/noosphere-core/src/api/v0alpha2/route.rs b/rust/noosphere-core/src/api/v0alpha2/route.rs
new file mode 100644
index 000000000..43be7acbc
--- /dev/null
+++ b/rust/noosphere-core/src/api/v0alpha2/route.rs
@@ -0,0 +1,25 @@
+use crate::api::route::RouteSignature;
+use crate::route_display;
+
+/// The version of the API represented by this module
+pub const API_VERSION: &str = "v0alpha2";
+
+/// An enum whose variants represent all of the routes in this version of the API
+pub enum Route {
+ /// Push the latest local history of a sphere from a client to the gateway
+ Push,
+}
+
+route_display!(Route);
+
+impl RouteSignature for Route {
+ fn to_fragment(&self) -> String {
+ match self {
+ Route::Push => "push".to_owned(),
+ }
+ }
+
+ fn api_version(&self) -> &str {
+ API_VERSION
+ }
+}
diff --git a/rust/noosphere-core/src/authority/author.rs b/rust/noosphere-core/src/authority/author.rs
index e65ad9d8b..c307caa3c 100644
--- a/rust/noosphere-core/src/authority/author.rs
+++ b/rust/noosphere-core/src/authority/author.rs
@@ -20,7 +20,9 @@ use super::generate_capability;
/// else read-only access (to all other spheres).
#[derive(PartialEq, Eq, Debug, Clone)]
pub enum Access {
+ /// Read/write access to a sphere
ReadWrite,
+ /// Read-only access to a sphere
ReadOnly,
}
@@ -33,7 +35,9 @@ pub struct Author<K>
where
K: KeyMaterial + Clone + 'static,
{
+ /// [KeyMaterial] that the [Author] has access to
pub key: K,
+ /// Optional proof of [Authorization] for the associated key
pub authorization: Option<Authorization>,
}
diff --git a/rust/noosphere-core/src/authority/authorization.rs b/rust/noosphere-core/src/authority/authorization.rs
index 2cd6d4c2b..bd9d7198d 100644
--- a/rust/noosphere-core/src/authority/authorization.rs
+++ b/rust/noosphere-core/src/authority/authorization.rs
@@ -23,14 +23,16 @@ use super::SUPPORTED_KEYS;
/// similar construct to land in rs-ucan
#[derive(Clone, Debug)]
pub enum Authorization {
- /// A fully instantiated UCAN
+ /// A fully instantiated [Ucan]
Ucan(Ucan),
- /// A CID that refers to a UCAN that may be looked up in storage at the
+ /// A [Cid] that refers to a [Ucan] that may be looked up in storage at the
/// point of invocation
Cid(Cid),
}
impl Authorization {
+ /// Attempt to resolve the [Authorization] as a fully deserialized [Ucan]
+ /// (if it is not one already.
pub async fn as_ucan<S: UcanJwtStore>(&self, store: &S) -> Result<Ucan> {
match self {
Authorization::Ucan(ucan) => Ok(ucan.clone()),
@@ -38,6 +40,7 @@ impl Authorization {
}
}
+ /// Attempt to resolve the [Authorization] as a [ProofChain] (via its associated [Ucan])
pub async fn as_proof_chain<S: UcanJwtStore>(&self, store: &S) -> Result<ProofChain> {
let mut did_parser = DidParser::new(SUPPORTED_KEYS);
Ok(match self {
diff --git a/rust/noosphere-core/src/authority/capability.rs b/rust/noosphere-core/src/authority/capability.rs
index 4db0516eb..71624d892 100644
--- a/rust/noosphere-core/src/authority/capability.rs
+++ b/rust/noosphere-core/src/authority/capability.rs
@@ -5,6 +5,7 @@ use ucan::capability::{
};
use url::Url;
+/// The ordinal levels of authority allowed within Noosphere
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug)]
pub enum SphereAbility {
/// May read information about a sphere from a counterpart
@@ -45,8 +46,13 @@ impl TryFrom<String> for SphereAbility {
}
}
+#[cfg(doc)]
+use crate::data::Did;
+
+/// A reference to a sphere, by its [Did]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SphereReference {
+ /// The [Did] of the sphere
pub did: String,
}
@@ -78,10 +84,12 @@ impl TryFrom<Url> for SphereReference {
}
}
+/// A struct that implements [CapabilitySemantics] for spheres
pub struct SphereSemantics {}
impl CapabilitySemantics<SphereReference, SphereAbility> for SphereSemantics {}
+/// A shared instance of [SphereSemantics]
pub const SPHERE_SEMANTICS: SphereSemantics = SphereSemantics {};
/// Generates a [Capability] struct representing permissions in a [LinkRecord].
diff --git a/rust/noosphere-core/src/authority/key_material.rs b/rust/noosphere-core/src/authority/key_material.rs
index b25a4da6f..c28e2dc6d 100644
--- a/rust/noosphere-core/src/authority/key_material.rs
+++ b/rust/noosphere-core/src/authority/key_material.rs
@@ -8,18 +8,21 @@ use ucan_key_support::{
rsa::{bytes_to_rsa_key, RSA_MAGIC_BYTES},
};
+/// A common set of DID Key formats that are supported by this crate
// TODO: Conditional web crypto support
pub const SUPPORTED_KEYS: &KeyConstructorSlice = &[
(ED25519_MAGIC_BYTES, bytes_to_ed25519_key),
(RSA_MAGIC_BYTES, bytes_to_rsa_key),
];
+/// Produce a unique [Ed25519KeyMaterial] for general purpose use cases
pub fn generate_ed25519_key() -> Ed25519KeyMaterial {
let private_key = Ed25519PrivateKey::new(rand::thread_rng());
let public_key = Ed25519PublicKey::from(&private_key);
Ed25519KeyMaterial(public_key, Some(private_key))
}
+/// Restore an [Ed25519KeyMaterial] from a [Mnemonic]
pub fn restore_ed25519_key(mnemonic: &str) -> Result<Ed25519KeyMaterial> {
let mnemonic = BipMnemonic::from_phrase(mnemonic, Language::English)?;
let private_key = Ed25519PrivateKey::try_from(mnemonic.entropy())?;
@@ -28,6 +31,9 @@ pub fn restore_ed25519_key(mnemonic: &str) -> Result<Ed25519KeyMaterial> {
Ok(Ed25519KeyMaterial(public_key, Some(private_key)))
}
+/// Produce a [Mnemonic] for a given [Ed25519KeyMaterial]; note that the private
+/// part of the key must be available in the [Ed25519KeyMaterial] in order to
+/// produce the mnemonic.
pub fn ed25519_key_to_mnemonic(key_material: &Ed25519KeyMaterial) -> Result<Mnemonic> {
let private_key = &key_material.1.ok_or_else(|| {
anyhow!(
@@ -38,8 +44,11 @@ pub fn ed25519_key_to_mnemonic(key_material: &Ed25519KeyMaterial) -> Result<Mnem
Ok(Mnemonic(mnemonic.into_phrase()))
}
-pub const ED25519_KEYPAIR_LENGTH: usize = 64;
-pub const ED25519_KEY_LENGTH: usize = 32;
+const ED25519_KEYPAIR_LENGTH: usize = 64;
+const ED25519_KEY_LENGTH: usize = 32;
+
+/// For a given [Ed25519KeyMaterial] produce a serialized byte array of the
+/// key that is suitable for persisting to secure storage.
pub fn ed25519_key_to_bytes(
key_material: &Ed25519KeyMaterial,
) -> Result<[u8; ED25519_KEYPAIR_LENGTH]> {
diff --git a/rust/noosphere-core/src/authority/mod.rs b/rust/noosphere-core/src/authority/mod.rs
index 7647bf004..b6ed6f474 100644
--- a/rust/noosphere-core/src/authority/mod.rs
+++ b/rust/noosphere-core/src/authority/mod.rs
@@ -1,3 +1,9 @@
+//! Data types and helper routines related to general Noosphere authority
+//! concepts.
+//!
+//! This includes key material generation, expressing capabilities and passing
+//! around proof of authorization within the other corners of the API.
+
mod author;
mod authorization;
mod capability;
diff --git a/rust/noosphere-core/src/context/authority/mod.rs b/rust/noosphere-core/src/context/authority/mod.rs
new file mode 100644
index 000000000..4c768903e
--- /dev/null
+++ b/rust/noosphere-core/src/context/authority/mod.rs
@@ -0,0 +1,5 @@
+mod read;
+pub use read::*;
+
+mod write;
+pub use write::*;
diff --git a/rust/noosphere-core/src/context/authority/read.rs b/rust/noosphere-core/src/context/authority/read.rs
new file mode 100644
index 000000000..01cd92f4a
--- /dev/null
+++ b/rust/noosphere-core/src/context/authority/read.rs
@@ -0,0 +1,192 @@
+use std::sync::Arc;
+
+use crate::{
+ authority::{Author, Authorization},
+ context::{HasSphereContext, SphereContextKey},
+ data::{Did, Link, Mnemonic},
+ error::NoosphereError,
+};
+use anyhow::{anyhow, Result};
+use noosphere_storage::Storage;
+
+use tokio_stream::StreamExt;
+
+use async_trait::async_trait;
+
+/// Anything that can read the authority section from a sphere should implement
+/// [SphereAuthorityRead]. A blanket implementation is provided for anything
+/// that implements [HasSphereContext].
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereAuthorityRead<S>
+where
+ S: Storage + 'static,
+ Self: Sized,
+{
+ /// For a given [Authorization], checks that the authorization and all of its
+ /// ancester proofs are valid and have not been revoked
+ async fn verify_authorization(
+ &self,
+ authorization: &Authorization,
+ ) -> Result<(), NoosphereError>;
+
+ /// Look up an [Authorization] by a [Did].
+ async fn get_authorization(&self, did: &Did) -> Result<Option<Authorization>>;
+
+ /// Look up all [Authorization]s with the specified name
+ async fn get_authorizations_by_name(&self, name: &str) -> Result<Vec<Authorization>>;
+
+ /// Derive a root sphere key from a mnemonic and return a version of this
+ /// [SphereAuthorityRead] whose inner [SphereContext]'s [Author] is using
+ /// that root sphere key.
+ async fn escalate_authority(&self, mnemonic: &Mnemonic) -> Result<Self>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereAuthorityRead<S> for C
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ async fn verify_authorization(
+ &self,
+ authorization: &Authorization,
+ ) -> Result<(), NoosphereError> {
+ self.to_sphere()
+ .await?
+ .verify_authorization(authorization)
+ .await
+ }
+
+ async fn get_authorization(&self, did: &Did) -> Result<Option<Authorization>> {
+ let sphere = self.to_sphere().await?;
+ let authority = sphere.get_authority().await?;
+ let delegations = authority.get_delegations().await?;
+ let delegations_stream = delegations.into_stream().await?;
+
+ tokio::pin!(delegations_stream);
+
+ while let Some((Link { cid, .. }, delegation)) = delegations_stream.try_next().await? {
+ let ucan = delegation.resolve_ucan(sphere.store()).await?;
+ let authorized_did = ucan.audience();
+
+ if authorized_did == did {
+ return Ok(Some(Authorization::Cid(cid)));
+ }
+ }
+
+ Ok(None)
+ }
+
+ async fn get_authorizations_by_name(&self, name: &str) -> Result<Vec<Authorization>> {
+ let sphere = self.to_sphere().await?;
+ let authority = sphere.get_authority().await?;
+ let delegations = authority.get_delegations().await?;
+ let delegations_stream = delegations.into_stream().await?;
+ let mut authorizations = Vec::new();
+
+ tokio::pin!(delegations_stream);
+
+ while let Some((link, delegation)) = delegations_stream.try_next().await? {
+ if delegation.name == name {
+ authorizations.push(Authorization::Cid(link.into()));
+ }
+ }
+
+ Ok(authorizations)
+ }
+
+ async fn escalate_authority(&self, mnemonic: &Mnemonic) -> Result<Self> {
+ let root_key: SphereContextKey = Arc::new(Box::new(mnemonic.to_credential()?));
+ let root_author = Author {
+ key: root_key,
+ authorization: None,
+ };
+
+ let root_identity = root_author.did().await?;
+ let sphere_identity = self.identity().await?;
+
+ if sphere_identity != root_identity {
+ return Err(anyhow!(
+ "Provided mnemonic did not produce the expected credential"
+ ));
+ }
+
+ Ok(Self::wrap(
+ self.sphere_context()
+ .await?
+ .with_author(&root_author)
+ .await?,
+ )
+ .await)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::data::Did;
+ use anyhow::Result;
+
+ use ucan::crypto::KeyMaterial;
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ use crate::{
+ context::{HasSphereContext, SphereAuthorityRead},
+ helpers::{simulated_sphere_context, SimulationAccess},
+ };
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_get_an_authorization_by_did() -> Result<()> {
+ let (sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let author_did = Did(sphere_context
+ .sphere_context()
+ .await?
+ .author()
+ .key
+ .get_did()
+ .await?);
+
+ let authorization = sphere_context
+ .get_authorization(&author_did)
+ .await?
+ .unwrap();
+
+ let _ucan = authorization
+ .as_ucan(sphere_context.sphere_context().await?.db())
+ .await?;
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_verify_an_authorization_to_write_to_a_sphere() -> Result<()> {
+ let (sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let author_did = Did(sphere_context
+ .sphere_context()
+ .await?
+ .author()
+ .key
+ .get_did()
+ .await?);
+
+ let authorization = sphere_context
+ .get_authorization(&author_did)
+ .await?
+ .unwrap();
+
+ sphere_context.verify_authorization(&authorization).await?;
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/context/authority/write.rs b/rust/noosphere-core/src/context/authority/write.rs
new file mode 100644
index 000000000..db62725a7
--- /dev/null
+++ b/rust/noosphere-core/src/context/authority/write.rs
@@ -0,0 +1,387 @@
+use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, HasSphereContext};
+
+use super::SphereAuthorityRead;
+use crate::{
+ authority::{generate_capability, Authorization, SphereAbility},
+ data::{DelegationIpld, Did, Jwt, Link, RevocationIpld},
+ view::SPHERE_LIFETIME,
+};
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use cid::Cid;
+use noosphere_storage::{Storage, UcanStore};
+use tokio_stream::StreamExt;
+use ucan::{builder::UcanBuilder, crypto::KeyMaterial};
+
+/// Any type which implements [SphereAuthorityWrite] is able to manipulate the
+/// [AuthorityIpld] section of a sphere. This includes authorizing other keys
+/// and revoking prior authorizations.
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereAuthorityWrite<S>: SphereAuthorityRead<S>
+where
+ S: Storage + 'static,
+{
+ /// Authorize another key by its [Did], associating the authorization with a
+ /// provided display name
+ async fn authorize(&mut self, name: &str, identity: &Did) -> Result<Authorization>;
+
+ /// Revoke a previously granted authorization.
+ ///
+ /// Note that correctly revoking an authorization requires signing with a credential
+ /// that is in the chain of authority that ultimately granted the authorization being
+ /// revoked. Attempting to revoke a credential with any credential that isn't in that
+ /// chain of authority will fail.
+ async fn revoke_authorization(&mut self, authorization: &Authorization) -> Result<()>;
+
+ /// Recover authority by revoking all previously delegated authorizations
+ /// and creating a new one that delegates authority to the specified key
+ /// (given by its [Did]).
+ ///
+ /// Note that correctly recovering authority requires signing with the root
+ /// sphere credential, so generally can only be performed on a type that
+ /// implements [SphereAuthorityEscalate]
+ async fn recover_authority(&mut self, new_owner: &Did) -> Result<Authorization>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereAuthorityWrite<S> for C
+where
+ C: HasSphereContext<S> + HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ // TODO(#423): We allow optional human-readable names for authorizations,
+ // but this will bear the consequence of leaking personal information about
+ // the user (e.g., a list of their authorized devices). We should encrypt
+ // these names so that they are only readable by the user themselves.
+ // TODO(#560): We should probably enforce that each [Did] only gets one
+ // authorization, from a hygeine perspective; elsewhere we need to assume
+ // multiple authorizations for the same [Did] are possible.
+ async fn authorize(&mut self, name: &str, identity: &Did) -> Result<Authorization> {
+ self.assert_write_access().await?;
+
+ let author = self.sphere_context().await?.author().clone();
+ let mut sphere = self.to_sphere().await?;
+ let authorization = author.require_authorization()?;
+
+ self.verify_authorization(authorization).await?;
+
+ let authorization_expiry: Option<u64> = {
+ let ucan = authorization
+ .as_ucan(&UcanStore(sphere.store().clone()))
+ .await?;
+ *ucan.expires_at()
+ };
+
+ let mut builder = UcanBuilder::default()
+ .issued_by(&author.key)
+ .for_audience(identity)
+ .claiming_capability(&generate_capability(
+ &sphere.get_identity().await?,
+ SphereAbility::Authorize,
+ ))
+ .with_nonce();
+
+ // TODO(ucan-wg/rs-ucan#114): Clean this up when
+ // `UcanBuilder::with_expiration` accepts `Option<u64>`
+ if let Some(expiry) = authorization_expiry {
+ builder = builder.with_expiration(expiry);
+ }
+
+ // TODO(ucan-wg/rs-ucan#32): Clean this up when we can use a CID as an authorization
+ let mut signable = builder.build()?;
+
+ signable
+ .proofs
+ .push(Cid::try_from(authorization)?.to_string());
+
+ let jwt = signable.sign().await?.encode()?;
+
+ let delegation = DelegationIpld::register(name, &jwt, sphere.store_mut()).await?;
+
+ self.sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .delegations_mut()
+ .set(&Link::new(delegation.jwt), &delegation);
+
+ Ok(Authorization::Cid(delegation.jwt))
+ }
+
+ async fn revoke_authorization(&mut self, authorization: &Authorization) -> Result<()> {
+ self.assert_write_access().await?;
+
+ let mut sphere_context = self.sphere_context_mut().await?;
+
+ if !sphere_context
+ .author()
+ .is_authorizer_of(authorization, sphere_context.db())
+ .await?
+ {
+ let author_did = sphere_context.author().did().await?;
+
+ return Err(anyhow!(
+ "{} cannot revoke authorization {} (not a delegating authority)",
+ author_did,
+ authorization
+ ));
+ }
+
+ let authorization_cid = Link::<Jwt>::from(Cid::try_from(authorization)?);
+ let delegations = sphere_context
+ .sphere()
+ .await?
+ .get_authority()
+ .await?
+ .get_delegations()
+ .await?;
+
+ if delegations.get(&authorization_cid).await?.is_none() {
+ return Err(anyhow!(
+ "No authority has been delegated to the authorization being revoked"
+ ));
+ }
+
+ let revocation =
+ RevocationIpld::revoke(&authorization_cid, &sphere_context.author().key).await?;
+
+ sphere_context
+ .mutation_mut()
+ .delegations_mut()
+ .remove(&authorization_cid);
+
+ sphere_context
+ .mutation_mut()
+ .revocations_mut()
+ .set(&authorization_cid, &revocation);
+
+ // TODO(#424): Recursively remove any sub-delegations here (and revoke them?)
+
+ Ok(())
+ }
+
+ async fn recover_authority(&mut self, new_owner: &Did) -> Result<Authorization> {
+ self.assert_write_access().await?;
+
+ let mut sphere_context = self.sphere_context_mut().await?;
+ let author_did = Did(sphere_context.author().key.get_did().await?);
+ let sphere_identity = sphere_context.identity().clone();
+
+ if author_did != sphere_identity {
+ return Err(anyhow!(
+ "Only the root sphere credential can be used to recover authority"
+ ));
+ }
+
+ let sphere = sphere_context.sphere().await?;
+ let authority = sphere.get_authority().await?;
+ let delegations = authority.get_delegations().await?;
+ let delegation_stream = delegations.into_stream().await?;
+
+ tokio::pin!(delegation_stream);
+
+ // First: revoke all current authority
+ while let Some((link, _)) = delegation_stream.try_next().await? {
+ let revocation = RevocationIpld::revoke(&link, &sphere_context.author().key).await?;
+
+ sphere_context
+ .mutation_mut()
+ .delegations_mut()
+ .remove(&link);
+ sphere_context
+ .mutation_mut()
+ .revocations_mut()
+ .set(&link, &revocation);
+ }
+
+ // Then: bless a new owner
+ let ucan = UcanBuilder::default()
+ .issued_by(&sphere_context.author().key)
+ .for_audience(new_owner)
+ .with_lifetime(SPHERE_LIFETIME)
+ .with_nonce()
+ .claiming_capability(&generate_capability(
+ &sphere_identity,
+ SphereAbility::Authorize,
+ ))
+ .build()?
+ .sign()
+ .await?;
+
+ let jwt = ucan.encode()?;
+ let delegation = DelegationIpld::register("(OWNER)", &jwt, sphere_context.db()).await?;
+ let link = Link::new(delegation.jwt);
+
+ sphere_context
+ .mutation_mut()
+ .delegations_mut()
+ .set(&link, &delegation);
+
+ Ok(Authorization::Cid(link.into()))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use crate::authority::{generate_ed25519_key, Author};
+ use crate::data::Did;
+ use anyhow::Result;
+
+ use tokio::sync::Mutex;
+ use ucan::crypto::KeyMaterial;
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ use crate::{
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite,
+ SphereContextKey,
+ },
+ helpers::{simulated_sphere_context, SimulationAccess},
+ };
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_allows_an_authorized_key_to_authorize_other_keys() -> Result<()> {
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let other_key = generate_ed25519_key();
+ let other_did = Did(other_key.get_did().await?);
+
+ let other_authorization = sphere_context.authorize("other", &other_did).await?;
+ sphere_context.save(None).await?;
+
+ assert!(sphere_context
+ .verify_authorization(&other_authorization)
+ .await
+ .is_ok());
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_implicitly_revokes_transitive_authorizations() -> Result<()> {
+ let (mut sphere_context, mnemonic) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let other_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key()));
+ let other_did = Did(other_key.get_did().await?);
+
+ let other_authorization = sphere_context.authorize("other", &other_did).await?;
+ sphere_context.save(None).await?;
+
+ let mut sphere_context_with_other_credential = Arc::new(Mutex::new(
+ sphere_context
+ .sphere_context()
+ .await?
+ .with_author(&Author {
+ key: other_key.clone(),
+ authorization: Some(other_authorization.clone()),
+ })
+ .await?,
+ ));
+
+ let third_key = generate_ed25519_key();
+ let third_did = Did(third_key.get_did().await?);
+
+ let third_authorization = sphere_context_with_other_credential
+ .authorize("third", &third_did)
+ .await?;
+ sphere_context_with_other_credential.save(None).await?;
+
+ let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?;
+
+ root_sphere_context
+ .revoke_authorization(&other_authorization)
+ .await?;
+ root_sphere_context.save(None).await?;
+
+ assert!(sphere_context
+ .verify_authorization(&third_authorization)
+ .await
+ .is_err());
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_catches_revoked_authorizations_when_verifying() -> Result<()> {
+ let (mut sphere_context, mnemonic) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let other_key = generate_ed25519_key();
+ let other_did = Did(other_key.get_did().await?);
+
+ let other_authorization = sphere_context.authorize("other", &other_did).await?;
+ sphere_context.save(None).await?;
+
+ let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?;
+
+ root_sphere_context
+ .revoke_authorization(&other_authorization)
+ .await?;
+ root_sphere_context.save(None).await?;
+
+ assert!(sphere_context
+ .verify_authorization(&other_authorization)
+ .await
+ .is_err());
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_perform_access_recovery_given_a_mnemonic() -> Result<()> {
+ let (mut sphere_context, mnemonic) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let owner = sphere_context.sphere_context().await?.author().clone();
+
+ let other_key = generate_ed25519_key();
+ let other_did = Did(other_key.get_did().await?);
+
+ let other_authorization = sphere_context.authorize("other", &other_did).await?;
+ sphere_context.save(None).await?;
+
+ let next_owner_key = generate_ed25519_key();
+ let next_owner_did = Did(next_owner_key.get_did().await?);
+
+ let mut root_sphere_context = sphere_context.escalate_authority(&mnemonic).await?;
+
+ root_sphere_context
+ .recover_authority(&next_owner_did)
+ .await?;
+ root_sphere_context.save(None).await?;
+
+ assert!(sphere_context
+ .verify_authorization(&other_authorization)
+ .await
+ .is_err());
+
+ assert!(sphere_context
+ .verify_authorization(&owner.authorization.unwrap())
+ .await
+ .is_err());
+
+ sphere_context
+ .verify_authorization(
+ &sphere_context
+ .get_authorization(&next_owner_did)
+ .await?
+ .unwrap(),
+ )
+ .await?;
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/context/content/decoder.rs b/rust/noosphere-core/src/context/content/decoder.rs
new file mode 100644
index 000000000..cfc13029c
--- /dev/null
+++ b/rust/noosphere-core/src/context/content/decoder.rs
@@ -0,0 +1,29 @@
+use crate::data::BodyChunkIpld;
+use async_stream::try_stream;
+use bytes::Bytes;
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use noosphere_storage::BlockStore;
+use tokio_stream::Stream;
+
+/// Helper to easily decode a linked list of `BodyChunkIpld` as a byte stream
+pub struct BodyChunkDecoder<'a, 'b, S: BlockStore>(pub &'a Cid, pub &'b S);
+
+impl<'a, 'b, S: BlockStore> BodyChunkDecoder<'a, 'b, S> {
+ /// Consume the [BodyChunkDecoder] and return an async [Stream] of bytes
+ /// representing the raw body contents
+ pub fn stream(self) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Unpin {
+ let mut next = Some(*self.0);
+ let store = self.1.clone();
+ Box::pin(try_stream! {
+ while let Some(cid) = next {
+ debug!("Unpacking block {}...", cid);
+ let chunk = store.load::<DagCborCodec, BodyChunkIpld>(&cid).await.map_err(|error| {
+ std::io::Error::new(std::io::ErrorKind::UnexpectedEof, error.to_string())
+ })?;
+ yield Bytes::from(chunk.bytes);
+ next = chunk.next;
+ }
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/context/content/file.rs b/rust/noosphere-core/src/context/content/file.rs
new file mode 100644
index 000000000..780796e52
--- /dev/null
+++ b/rust/noosphere-core/src/context/content/file.rs
@@ -0,0 +1,49 @@
+use std::pin::Pin;
+
+use crate::data::{Did, Link, MemoIpld};
+use tokio::io::AsyncRead;
+
+/// A type that may be used as the contents field in a [SphereFile]
+#[cfg(not(target_arch = "wasm32"))]
+pub trait AsyncFileBody: AsyncRead + Unpin + Send {}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<S> AsyncFileBody for S where S: AsyncRead + Unpin + Send {}
+
+#[cfg(target_arch = "wasm32")]
+/// A type that may be used as the contents field in a [SphereFile]
+pub trait AsyncFileBody: AsyncRead + Unpin {}
+
+#[cfg(target_arch = "wasm32")]
+impl<S> AsyncFileBody for S where S: AsyncRead + Unpin {}
+
+/// A descriptor for contents that is stored in a sphere.
+pub struct SphereFile<C> {
+ /// The identity of the associated sphere from which the file was read
+ pub sphere_identity: Did,
+ /// The version of the associated sphere from which the file was read
+ pub sphere_version: Link<MemoIpld>,
+ /// The version of the memo that wraps the file's body contents
+ pub memo_version: Link<MemoIpld>,
+ /// The memo that wraps the file's body contents
+ pub memo: MemoIpld,
+ /// The body contents of the file
+ pub contents: C,
+}
+
+impl<C> SphereFile<C>
+where
+ C: AsyncFileBody + 'static,
+{
+ /// Consume the file and return a version of it where its body contents have
+ /// been boxed and pinned
+ pub fn boxed(self) -> SphereFile<Pin<Box<dyn AsyncFileBody + 'static>>> {
+ SphereFile {
+ sphere_identity: self.sphere_identity,
+ sphere_version: self.sphere_version,
+ memo_version: self.memo_version,
+ memo: self.memo,
+ contents: Box::pin(self.contents),
+ }
+ }
+}
diff --git a/rust/noosphere-core/src/context/content/mod.rs b/rust/noosphere-core/src/context/content/mod.rs
new file mode 100644
index 000000000..ec6444013
--- /dev/null
+++ b/rust/noosphere-core/src/context/content/mod.rs
@@ -0,0 +1,13 @@
+//! Sphere content is a storage space for any files that the sphere owner wishes to associate
+//! with a public "slug", that is addressable by them or others who have replicated the sphere
+//! data.
+
+mod decoder;
+mod file;
+mod read;
+mod write;
+
+pub use decoder::*;
+pub use file::*;
+pub use read::*;
+pub use write::*;
diff --git a/rust/noosphere-core/src/context/content/read.rs b/rust/noosphere-core/src/context/content/read.rs
new file mode 100644
index 000000000..3a4b72026
--- /dev/null
+++ b/rust/noosphere-core/src/context/content/read.rs
@@ -0,0 +1,49 @@
+use anyhow::Result;
+use noosphere_storage::Storage;
+
+use crate::context::HasSphereContext;
+use async_trait::async_trait;
+
+use crate::context::{internal::SphereContextInternal, AsyncFileBody, SphereFile};
+
+/// Anything that can read content from a sphere should implement [SphereContentRead].
+/// A blanket implementation is provided for anything that implements [HasSphereContext].
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereContentRead<S>
+where
+ S: Storage + 'static,
+{
+ /// Read a file that is associated with a given slug at the revision of the
+ /// sphere that this view is pointing to.
+ /// Note that "contents" are `AsyncRead`, and content bytes won't be read
+ /// until contents is polled.
+ async fn read(&self, slug: &str) -> Result<Option<SphereFile<Box<dyn AsyncFileBody>>>>;
+
+ /// Returns true if the content identitifed by slug exists in the sphere at
+ /// the current revision.
+ async fn exists(&self, slug: &str) -> Result<bool> {
+ Ok(self.read(slug).await?.is_some())
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereContentRead<S> for C
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ async fn read(&self, slug: &str) -> Result<Option<SphereFile<Box<dyn AsyncFileBody>>>> {
+ let revision = self.version().await?;
+ let sphere = self.to_sphere().await?;
+
+ let links = sphere.get_content().await?;
+ let hamt = links.get_hamt().await?;
+
+ Ok(match hamt.get(&slug.to_string()).await? {
+ Some(memo) => Some(self.get_file(&revision, memo.clone()).await?),
+ None => None,
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/context/content/write.rs b/rust/noosphere-core/src/context/content/write.rs
new file mode 100644
index 000000000..e13bf387b
--- /dev/null
+++ b/rust/noosphere-core/src/context/content/write.rs
@@ -0,0 +1,183 @@
+use crate::data::{BodyChunkIpld, Header, Link, MemoIpld};
+use anyhow::{anyhow, Result};
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use noosphere_storage::{BlockStore, Storage};
+
+use tokio::io::AsyncReadExt;
+
+use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, HasSphereContext};
+use async_trait::async_trait;
+
+use crate::context::{AsyncFileBody, SphereContentRead};
+
+fn validate_slug(slug: &str) -> Result<()> {
+ if slug.is_empty() {
+ Err(anyhow!("Slug must not be empty."))
+ } else {
+ Ok(())
+ }
+}
+
+/// Anything that can write content to a sphere should implement
+/// [SphereContentWrite]. A blanket implementation is provided for anything that
+/// implements [HasMutableSphereContext].
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereContentWrite<S>: SphereContentRead<S>
+where
+ S: Storage + 'static,
+{
+ /// Like link, this takes a [Link<MemoIpld>] that should be associated
+ /// directly with a slug, but in this case the [Link<MemoIpld>] is assumed
+ /// to refer to a memo, so no wrapping memo is created.
+ async fn link_raw(&mut self, slug: &str, cid: &Link<MemoIpld>) -> Result<()>;
+
+ /// Similar to write, but instead of generating blocks from some provided
+ /// bytes, the caller provides a CID of an existing DAG in storage. That
+ /// CID is used as the body of a Memo that is written to the specified
+ /// slug, and the CID of the memo is returned.
+ async fn link(
+ &mut self,
+ slug: &str,
+ content_type: &str,
+ body_cid: &Cid,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>>;
+
+ /// Write to a slug in the sphere. In order to commit the change to the
+ /// sphere, you must call save. You can buffer multiple writes before
+ /// saving.
+ ///
+ /// The returned CID is a link to the memo for the newly added content.
+ async fn write<F: AsyncFileBody>(
+ &mut self,
+ slug: &str,
+ content_type: &str,
+ mut value: F,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>>;
+
+ /// Unlinks a slug from the content space. Note that this does not remove
+ /// the blocks that were previously associated with the content found at the
+ /// given slug, because they will still be available at an earlier revision
+ /// of the sphere. In order to commit the change, you must save. Note that
+ /// this call is a no-op if there is no matching slug linked in the sphere.
+ ///
+ /// The returned value is the CID previously associated with the slug, if
+ /// any.
+ async fn remove(&mut self, slug: &str) -> Result<Option<Link<MemoIpld>>>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereContentWrite<S> for C
+where
+ C: HasSphereContext<S> + HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ async fn link_raw(&mut self, slug: &str, cid: &Link<MemoIpld>) -> Result<()> {
+ self.assert_write_access().await?;
+ validate_slug(slug)?;
+
+ self.sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .content_mut()
+ .set(&slug.into(), cid);
+
+ Ok(())
+ }
+
+ async fn link(
+ &mut self,
+ slug: &str,
+ content_type: &str,
+ body_cid: &Cid,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>> {
+ self.assert_write_access().await?;
+ validate_slug(slug)?;
+
+ let memo_cid = {
+ let current_file = self.read(slug).await?;
+ let previous_memo_cid = current_file.map(|file| file.memo_version);
+
+ let mut sphere_context = self.sphere_context_mut().await?;
+
+ let mut new_memo = match previous_memo_cid {
+ Some(cid) => {
+ let mut memo = MemoIpld::branch_from(&cid, sphere_context.db()).await?;
+ memo.body = *body_cid;
+ memo
+ }
+ None => MemoIpld {
+ parent: None,
+ headers: Vec::new(),
+ body: *body_cid,
+ },
+ };
+
+ if let Some(headers) = additional_headers {
+ new_memo.replace_headers(headers)
+ }
+
+ new_memo.replace_first_header(&Header::ContentType, content_type);
+
+ // TODO(#43): Configure default/implicit headers here
+ sphere_context
+ .db_mut()
+ .save::<DagCborCodec, MemoIpld>(new_memo)
+ .await?
+ .into()
+ };
+
+ self.link_raw(slug, &memo_cid).await?;
+
+ Ok(memo_cid)
+ }
+
+ async fn write<F: AsyncFileBody>(
+ &mut self,
+ slug: &str,
+ content_type: &str,
+ mut value: F,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>> {
+ debug!("Writing {}...", slug);
+
+ self.assert_write_access().await?;
+ validate_slug(slug)?;
+
+ let mut bytes = Vec::new();
+ value.read_to_end(&mut bytes).await?;
+
+ // TODO(#38): We imply here that the only content types we care about
+ // amount to byte streams, but in point of fact we can support anything
+ // that may be referenced by CID including arbitrary IPLD structures
+ let body_cid =
+ BodyChunkIpld::store_bytes(&bytes, self.sphere_context_mut().await?.db_mut()).await?;
+
+ self.link(slug, content_type, &body_cid, additional_headers)
+ .await
+ }
+
+ async fn remove(&mut self, slug: &str) -> Result<Option<Link<MemoIpld>>> {
+ self.assert_write_access().await?;
+
+ let current_file = self.read(slug).await?;
+
+ Ok(match current_file {
+ Some(file) => {
+ self.sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .content_mut()
+ .remove(&String::from(slug));
+
+ Some(file.memo_version)
+ }
+ None => None,
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/context/context.rs b/rust/noosphere-core/src/context/context.rs
new file mode 100644
index 000000000..53ade732a
--- /dev/null
+++ b/rust/noosphere-core/src/context/context.rs
@@ -0,0 +1,445 @@
+use std::sync::Arc;
+
+use anyhow::Result;
+
+use crate::{
+ api::Client,
+ authority::{Access, Author, SUPPORTED_KEYS},
+ context::metadata::GATEWAY_URL,
+ data::{Did, Link, MemoIpld},
+ view::{Sphere, SphereMutation},
+};
+use noosphere_storage::{KeyValueStore, SphereDb, Storage};
+use tokio::sync::OnceCell;
+use ucan::crypto::{did::DidParser, KeyMaterial};
+use url::Url;
+
+#[cfg(doc)]
+use crate::context::has::HasSphereContext;
+
+/// The type of any [KeyMaterial] that is used within a [SphereContext]
+pub type SphereContextKey = Arc<Box<dyn KeyMaterial>>;
+
+/// A [SphereContext] is an accessor construct over locally replicated sphere
+/// data. It embodies both the storage layer that contains the sphere's data
+/// as the information needed to verify a user's intended level of access to
+/// it (e.g., local key material and [ucan::Ucan]-based authorization).
+/// Additionally, the [SphereContext] maintains a reference to an API [Client]
+/// that may be initialized as the network becomes available.
+///
+/// All interactions that pertain to a sphere, including reading or writing
+/// its contents and syncing with a gateway, flow through the [SphereContext].
+pub struct SphereContext<S>
+where
+ S: Storage + 'static,
+{
+ sphere_identity: Did,
+ origin_sphere_identity: Did,
+ author: Author<SphereContextKey>,
+ access: OnceCell<Access>,
+ db: SphereDb<S>,
+ did_parser: DidParser,
+ client: OnceCell<Arc<Client<SphereContextKey, SphereDb<S>>>>,
+ mutation: SphereMutation,
+}
+
+impl<S> Clone for SphereContext<S>
+where
+ S: Storage + 'static,
+{
+ fn clone(&self) -> Self {
+ Self {
+ sphere_identity: self.sphere_identity.clone(),
+ origin_sphere_identity: self.origin_sphere_identity.clone(),
+ author: self.author.clone(),
+ access: OnceCell::new(),
+ db: self.db.clone(),
+ did_parser: DidParser::new(SUPPORTED_KEYS),
+ client: self.client.clone(),
+ mutation: SphereMutation::new(self.mutation.author()),
+ }
+ }
+}
+
+impl<S> SphereContext<S>
+where
+ S: Storage,
+{
+ /// Instantiate a new [SphereContext] given a sphere [Did], an [Author], a
+ /// [SphereDb] and an optional origin sphere [Did]. The origin sphere [Did]
+ /// is intended to signify whether the [SphereContext] is a local sphere, or
+ /// a global sphere that is being visited by a local author. In most cases,
+ /// a [SphereContext] with _some_ value set as the origin sphere [Did] will
+ /// be read-only.
+ pub async fn new(
+ sphere_identity: Did,
+ author: Author<SphereContextKey>,
+ db: SphereDb<S>,
+ origin_sphere_identity: Option<Did>,
+ ) -> Result<Self> {
+ let author_did = author.identity().await?;
+ let origin_sphere_identity =
+ origin_sphere_identity.unwrap_or_else(|| sphere_identity.clone());
+
+ Ok(SphereContext {
+ sphere_identity,
+ origin_sphere_identity,
+ access: OnceCell::new(),
+ author,
+ db,
+ did_parser: DidParser::new(SUPPORTED_KEYS),
+ client: OnceCell::new(),
+ mutation: SphereMutation::new(&author_did),
+ })
+ }
+
+ /// Clone this [SphereContext], setting the sphere identity to a peer's [Did]
+ pub async fn to_visitor(&self, peer_identity: &Did) -> Result<Self> {
+ self.db().require_version(peer_identity).await?;
+
+ SphereContext::new(
+ peer_identity.clone(),
+ self.author.clone(),
+ self.db.clone(),
+ Some(self.origin_sphere_identity.clone()),
+ )
+ .await
+ }
+
+ /// Clone this [SphereContext], replacing the [Author] with the provided one
+ pub async fn with_author(&self, author: &Author<SphereContextKey>) -> Result<SphereContext<S>> {
+ SphereContext::new(
+ self.sphere_identity.clone(),
+ author.clone(),
+ self.db.clone(),
+ Some(self.origin_sphere_identity.clone()),
+ )
+ .await
+ }
+
+ /// Given a [Did] of a sphere, produce a [SphereContext] backed by the same credentials and
+ /// storage primitives as this one, but that accesses the sphere referred to by the provided
+ /// [Did].
+ pub async fn traverse_by_identity(&self, _sphere_identity: &Did) -> Result<SphereContext<S>> {
+ unimplemented!()
+ }
+
+ /// The identity of the sphere
+ pub fn identity(&self) -> &Did {
+ &self.sphere_identity
+ }
+
+ /// The identity of the gateway sphere in use during this session, if
+ /// any; note that this will cause a request to be made to a gateway if no
+ /// handshake has yet occurred.
+ pub async fn gateway_identity(&self) -> Result<Did> {
+ Ok(self.client().await?.session.gateway_identity.clone())
+ }
+
+ /// The CID of the most recent local version of this sphere
+ pub async fn version(&self) -> Result<Link<MemoIpld>> {
+ Ok(self.db().require_version(self.identity()).await?.into())
+ }
+
+ /// The [Author] who is currently accessing the sphere
+ pub fn author(&self) -> &Author<SphereContextKey> {
+ &self.author
+ }
+
+ /// The [Access] level that the configured [Author] has relative to the
+ /// sphere that this [SphereContext] refers to.
+ pub async fn access(&self) -> Result<Access> {
+ let access = self
+ .access
+ .get_or_try_init(|| async {
+ self.author.access_to(&self.sphere_identity, &self.db).await
+ })
+ .await?;
+ Ok(access.clone())
+ }
+
+ /// Get a mutable reference to the [DidParser] used in this [SphereContext]
+ pub fn did_parser_mut(&mut self) -> &mut DidParser {
+ &mut self.did_parser
+ }
+
+ /// Sets or unsets the gateway URL that points to the gateway API that the
+ /// sphere will use when it is syncing.
+ pub async fn configure_gateway_url(&mut self, url: Option<&Url>) -> Result<()> {
+ self.client = OnceCell::new();
+
+ match url {
+ Some(url) => {
+ self.db.set_key(GATEWAY_URL, url.to_string()).await?;
+ }
+ None => {
+ self.db.unset_key(GATEWAY_URL).await?;
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Get the [SphereDb] instance that manages the current sphere's block
+ /// space and persisted configuration.
+ pub fn db(&self) -> &SphereDb<S> {
+ &self.db
+ }
+
+ /// Get a mutable reference to the [SphereDb] instance that manages the
+ /// current sphere's block space and persisted configuration.
+ pub fn db_mut(&mut self) -> &mut SphereDb<S> {
+ &mut self.db
+ }
+
+ /// Get a read-only reference to the underlying [SphereMutation] that this
+ /// [SphereContext] is tracking
+ pub fn mutation(&self) -> &SphereMutation {
+ &self.mutation
+ }
+
+ /// Get a mutable reference to the underlying [SphereMutation] that this
+ /// [SphereContext] is tracking
+ pub fn mutation_mut(&mut self) -> &mut SphereMutation {
+ &mut self.mutation
+ }
+
+ /// Get a [Sphere] view over the current sphere's latest revision. This view
+ /// offers lower-level access than [HasSphereContext], but includes affordances to
+ /// help tranversing and manipulating IPLD structures that are more
+ /// convenient than working directly with raw data.
+ pub async fn sphere(&self) -> Result<Sphere<SphereDb<S>>> {
+ Ok(Sphere::at(
+ &self.db.require_version(self.identity()).await?.into(),
+ self.db(),
+ ))
+ }
+
+ /// Get a [Client] that will interact with a configured gateway (if a URL
+ /// for one has been configured). This will initialize a [Client] if one is
+ /// not already intialized, and will fail if the [Client] is unable to
+ /// verify the identity of the gateway or otherwise connect to it.
+ pub async fn client(&self) -> Result<Arc<Client<SphereContextKey, SphereDb<S>>>> {
+ let client = self
+ .client
+ .get_or_try_init::<anyhow::Error, _, _>(|| async {
+ let gateway_url: Url = self.db.require_key(GATEWAY_URL).await?;
+
+ Ok(Arc::new(
+ Client::identify(
+ &self.origin_sphere_identity,
+ &gateway_url,
+ &self.author,
+ // TODO: Kill `DidParser` with fire
+ &mut DidParser::new(SUPPORTED_KEYS),
+ self.db.clone(),
+ )
+ .await?,
+ ))
+ })
+ .await?;
+
+ Ok(client.clone())
+ }
+
+ // Reset access so that it is re-evaluated the next time it is measured
+ // self.access.take();
+ pub(crate) fn reset_access(&mut self) {
+ self.access.take();
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use anyhow::Result;
+
+ use crate::{
+ authority::{generate_capability, generate_ed25519_key, SphereAbility},
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite,
+ },
+ data::{ContentType, LinkRecord, LINK_RECORD_FACT_NAME},
+ helpers::{make_valid_link_record, simulated_sphere_context, SimulationAccess},
+ tracing::initialize_tracing,
+ view::Sphere,
+ };
+
+ use noosphere_storage::{MemoryStorage, SphereDb};
+ use ucan::{builder::UcanBuilder, crypto::KeyMaterial, store::UcanJwtStore};
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_validates_slug_names_when_writing() -> Result<()> {
+ initialize_tracing(None);
+ let valid_names: &[&str] = &["j@__/_大", "/"];
+ let invalid_names: &[&str] = &[""];
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ for invalid_name in invalid_names {
+ assert!(sphere_context
+ .write(invalid_name, &ContentType::Text, "hello".as_ref(), None,)
+ .await
+ .is_err());
+ }
+
+ for valid_name in valid_names {
+ assert!(sphere_context
+ .write(valid_name, &ContentType::Text, "hello".as_ref(), None,)
+ .await
+ .is_ok());
+ }
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_validates_petnames_when_setting() -> Result<()> {
+ initialize_tracing(None);
+ let valid_names: &[&str] = &["j@__/_大"];
+ let invalid_names: &[&str] = &["", "did:key:foo"];
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+ let mut db = sphere_context.sphere_context().await?.db().clone();
+ let (other_identity, link_record, _) = make_valid_link_record(&mut db).await?;
+
+ for invalid_name in invalid_names {
+ assert!(sphere_context
+ .set_petname_record(invalid_name, &link_record)
+ .await
+ .is_err());
+ assert!(sphere_context
+ .set_petname(invalid_name, Some(other_identity.clone()))
+ .await
+ .is_err());
+ }
+
+ for valid_name in valid_names {
+ assert!(sphere_context
+ .set_petname(valid_name, Some(other_identity.clone()))
+ .await
+ .is_ok());
+ sphere_context.save(None).await?;
+ assert!(sphere_context
+ .set_petname_record(valid_name, &link_record)
+ .await
+ .is_ok());
+ }
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_disallows_adding_self_as_petname() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+ let db = sphere_context.sphere_context().await?.db().clone();
+ let sphere_identity = sphere_context.identity().await?;
+
+ let link_record = {
+ let version = sphere_context.version().await?;
+ let author = sphere_context.sphere_context().await?.author().clone();
+ LinkRecord::from(
+ UcanBuilder::default()
+ .issued_by(&author.key)
+ .for_audience(&sphere_identity)
+ .witnessed_by(
+ &author.authorization.as_ref().unwrap().as_ucan(&db).await?,
+ None,
+ )
+ .claiming_capability(&generate_capability(
+ &sphere_identity,
+ SphereAbility::Publish,
+ ))
+ .with_lifetime(120)
+ .with_fact(LINK_RECORD_FACT_NAME, version.to_string())
+ .build()?
+ .sign()
+ .await?,
+ )
+ };
+
+ assert!(sphere_context
+ .set_petname_record("myself", &link_record)
+ .await
+ .is_err());
+ assert!(sphere_context
+ .set_petname("myself", Some(sphere_identity.clone()))
+ .await
+ .is_err());
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_disallows_adding_outdated_records() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+ let mut store = sphere_context.sphere_context().await?.db().clone();
+
+ // Generate two LinkRecords, the first one having a later expiry
+ // than the second.
+ let (records, foo_identity) = {
+ let mut records: Vec<LinkRecord> = vec![];
+ let owner_key = generate_ed25519_key();
+ let owner_did = owner_key.get_did().await?;
+ let mut db = SphereDb::new(&MemoryStorage::default()).await?;
+ let (sphere, proof, _) = Sphere::generate(&owner_did, &mut db).await?;
+ let ucan_proof = proof.as_ucan(&db).await?;
+ let sphere_identity = sphere.get_identity().await?;
+ store.write_token(&ucan_proof.encode()?).await?;
+
+ for lifetime in [500, 100] {
+ let link_record = LinkRecord::from(
+ UcanBuilder::default()
+ .issued_by(&owner_key)
+ .for_audience(&sphere_identity)
+ .witnessed_by(&ucan_proof, None)
+ .claiming_capability(&generate_capability(
+ &sphere_identity,
+ SphereAbility::Publish,
+ ))
+ .with_lifetime(lifetime)
+ .with_fact(LINK_RECORD_FACT_NAME, sphere.cid().to_string())
+ .build()?
+ .sign()
+ .await?,
+ );
+
+ store.write_token(&link_record.encode()?).await?;
+ records.push(link_record);
+ }
+ (records, sphere_identity)
+ };
+
+ sphere_context
+ .set_petname("foo", Some(foo_identity))
+ .await?;
+ sphere_context.save(None).await?;
+
+ assert!(sphere_context
+ .set_petname_record("foo", records.get(0).unwrap())
+ .await
+ .is_ok());
+ sphere_context.save(None).await?;
+ assert!(sphere_context
+ .set_petname_record("foo", records.get(1).unwrap())
+ .await
+ .is_err());
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/context/cursor.rs b/rust/noosphere-core/src/context/cursor.rs
new file mode 100644
index 000000000..4e1aa6528
--- /dev/null
+++ b/rust/noosphere-core/src/context/cursor.rs
@@ -0,0 +1,745 @@
+use crate::api::v0alpha1::ReplicateParameters;
+use crate::stream::put_block_stream;
+use crate::{
+ data::{Link, MemoIpld},
+ view::{Sphere, Timeline},
+};
+use anyhow::Result;
+use async_trait::async_trait;
+use noosphere_storage::Storage;
+
+use crate::context::{HasMutableSphereContext, HasSphereContext, SphereContext, SphereReplicaRead};
+use std::marker::PhantomData;
+
+/// A [SphereCursor] is a structure that enables reading from and writing to a
+/// [SphereContext] at specific versions of the associated sphere's history.
+/// There are times when you may wish to be able to use the convenience
+/// implementation of traits built on [HasSphereContext], but to always be sure
+/// of what version you are using them on (such as when traversing sphere
+/// history). That is when you would use a [SphereCursor], which can wrap any
+/// implementor of [HasSphereContext] and mount it to a specific version of the
+/// sphere.
+#[derive(Clone)]
+pub struct SphereCursor<C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ has_sphere_context: C,
+ storage: PhantomData<S>,
+ sphere_version: Option<Link<MemoIpld>>,
+}
+
+impl<C, S> SphereCursor<C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ /// Consume the [SphereCursor] and return its wrapped [HasSphereContext]
+ pub fn to_inner(self) -> C {
+ self.has_sphere_context
+ }
+
+ /// Same as [SphereCursor::mount], but mounts the [SphereCursor] to a known
+ /// version of the history of the sphere.
+ pub fn mounted_at(has_sphere_context: C, sphere_version: &Link<MemoIpld>) -> Self {
+ SphereCursor {
+ has_sphere_context,
+ storage: PhantomData,
+ sphere_version: Some(sphere_version.clone()),
+ }
+ }
+
+ /// Create the [SphereCursor] at the latest local version of the associated
+ /// sphere, mounted to that version. If the latest version changes due to
+ /// effects in the distance, the cursor will still point to the same version
+ /// it referred to when it was created.
+ pub async fn mounted(has_sphere_context: C) -> Result<Self> {
+ let mut cursor = Self::latest(has_sphere_context);
+ cursor.mount().await?;
+ Ok(cursor)
+ }
+
+ /// "Mount" the [SphereCursor] to the given version of the sphere it refers
+ /// to. If the [SphereCursor] is already mounted, the version it is mounted
+ /// to will be overwritten. A mounted [SphereCursor] will remain at the
+ /// version it is mounted to even when the latest version of the sphere
+ /// changes.
+ pub async fn mount_at(&mut self, sphere_version: &Link<MemoIpld>) -> Result<&Self> {
+ self.sphere_version = Some(sphere_version.clone());
+
+ Ok(self)
+ }
+
+ /// Same as [SphereCursor::mount_at] except that it mounts to the latest
+ /// local version of the sphere.
+ pub async fn mount(&mut self) -> Result<&Self> {
+ let sphere_version = self
+ .has_sphere_context
+ .sphere_context()
+ .await?
+ .version()
+ .await?;
+
+ self.mount_at(&sphere_version).await
+ }
+
+ /// "Unmount" the [SphereCursor] so that it always uses the latest local
+ /// version of the sphere that it refers to.
+ pub fn unmount(mut self) -> Result<Self> {
+ self.sphere_version = None;
+ Ok(self)
+ }
+
+ /// Create this [SphereCursor] at the latest local version of the associated
+ /// sphere. The [SphereCursor] will always point to the latest local
+ /// version, unless subsequently mounted.
+ pub fn latest(has_sphere_context: C) -> Self {
+ SphereCursor {
+ has_sphere_context,
+ storage: PhantomData,
+ sphere_version: None,
+ }
+ }
+
+ /// Rewind the [SphereCursor] to point to the version of the sphere just
+ /// prior to this one in the edit chronology. If there was a previous
+ /// version to rewind to then the returned `Option` has the [Cid] of the
+ /// revision, otherwise if the current version is the oldest one it is
+ /// `None`.
+ pub async fn rewind(&mut self) -> Result<Option<&Link<MemoIpld>>> {
+ let sphere = self.to_sphere().await?;
+
+ match sphere.get_parent().await? {
+ Some(parent) => {
+ self.sphere_version = Some(parent.cid().clone());
+ Ok(self.sphere_version.as_ref())
+ }
+ None => Ok(None),
+ }
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> HasMutableSphereContext<S> for SphereCursor<C, S>
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage,
+{
+ type MutableSphereContext = C::MutableSphereContext;
+
+ async fn sphere_context_mut(&mut self) -> Result<Self::MutableSphereContext> {
+ self.has_sphere_context.sphere_context_mut().await
+ }
+
+ async fn save(
+ &mut self,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>> {
+ let new_version = self.has_sphere_context.save(additional_headers).await?;
+
+ if self.sphere_version.is_some() {
+ self.sphere_version = Some(new_version.clone());
+ }
+
+ Ok(new_version)
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> HasSphereContext<S> for SphereCursor<C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ type SphereContext = C::SphereContext;
+
+ async fn sphere_context(&self) -> Result<Self::SphereContext> {
+ self.has_sphere_context.sphere_context().await
+ }
+
+ async fn version(&self) -> Result<Link<MemoIpld>> {
+ match &self.sphere_version {
+ Some(sphere_version) => Ok(sphere_version.clone()),
+ None => self.has_sphere_context.version().await,
+ }
+ }
+
+ async fn wrap(sphere_context: SphereContext<S>) -> Self {
+ SphereCursor::latest(C::wrap(sphere_context).await)
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereReplicaRead<S> for SphereCursor<C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ #[instrument(level = "debug", skip(self))]
+ async fn traverse_by_petnames(&self, petname_path: &[String]) -> Result<Option<Self>> {
+ debug!("Traversing by petname...");
+
+ let replicate = {
+ let cursor = self.clone();
+
+ move |version: Link<MemoIpld>, since: Option<Link<MemoIpld>>| {
+ let cursor = cursor.clone();
+
+ async move {
+ let replicate_parameters = since.as_ref().map(|since| ReplicateParameters {
+ since: Some(since.clone()),
+ });
+ let (db, client) = {
+ let sphere_context = cursor.sphere_context().await?;
+ (sphere_context.db().clone(), sphere_context.client().await?)
+ };
+ let stream = client
+ .replicate(&version, replicate_parameters.as_ref())
+ .await?;
+ put_block_stream(db.clone(), stream).await?;
+
+ // If this was incremental replication, we have to hydrate...
+ if let Some(since) = since {
+ let since_memo = since.load_from(&db).await?;
+ let latest_memo = version.load_from(&db).await?;
+
+ // Only hydrate if since is a causal antecedent
+ if since_memo.lamport_order() < latest_memo.lamport_order() {
+ let timeline = Timeline::new(&db);
+
+ Sphere::hydrate_timeslice(
+ &timeline.slice(&version, Some(&since)).exclude_past(),
+ )
+ .await?;
+ }
+ }
+
+ Ok(()) as Result<(), anyhow::Error>
+ }
+ }
+ };
+
+ let sphere = self.to_sphere().await?;
+
+ let peer_sphere = match sphere
+ .traverse_by_petnames(petname_path, &replicate)
+ .await?
+ {
+ Some(sphere) => sphere,
+ None => return Ok(None),
+ };
+
+ let mut db = sphere.store().clone();
+ let peer_identity = peer_sphere.get_identity().await?;
+ let local_version = db.get_version(&peer_identity).await?.map(|cid| cid.into());
+
+ let should_update_version = if let Some(since) = local_version {
+ let since_memo = Sphere::at(&since, &db).to_memo().await?;
+ let latest_memo = peer_sphere.to_memo().await?;
+
+ since_memo.lamport_order() < latest_memo.lamport_order()
+ } else {
+ true
+ };
+
+ if should_update_version {
+ debug!(
+ "Updating local version of {} to more recent revision {}",
+ peer_identity,
+ peer_sphere.cid()
+ );
+
+ db.set_version(&peer_identity, peer_sphere.cid()).await?;
+ }
+
+ let peer_sphere_context = self
+ .sphere_context()
+ .await?
+ .to_visitor(&peer_identity)
+ .await?;
+
+ Ok(Some(SphereCursor::mounted_at(
+ C::wrap(peer_sphere_context).await,
+ peer_sphere.cid(),
+ )))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use anyhow::Result;
+ use noosphere_storage::UcanStore;
+ use tokio::io::AsyncReadExt;
+
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ use crate::{
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite,
+ SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead,
+ },
+ data::{ContentType, Header},
+ helpers::{
+ make_sphere_context_with_peer_chain, make_valid_link_record, simulated_sphere_context,
+ SimulationAccess,
+ },
+ tracing::initialize_tracing,
+ };
+
+ use super::SphereCursor;
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_unlink_slugs_from_the_content_space() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are great".as_ref(),
+ None,
+ )
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+
+ assert!(cursor.read("cats").await.unwrap().is_some());
+
+ cursor.remove("cats").await.unwrap();
+ cursor.save(None).await.unwrap();
+
+ assert!(cursor.read("cats").await.unwrap().is_none());
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_flushes_on_every_save() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let initial_stats = {
+ sphere_context
+ .lock()
+ .await
+ .db()
+ .to_block_store()
+ .to_stats()
+ .await
+ };
+ let mut cursor = SphereCursor::latest(sphere_context.clone());
+
+ cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are great".as_ref(),
+ None,
+ )
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+
+ let first_save_stats = {
+ sphere_context
+ .lock()
+ .await
+ .db()
+ .to_block_store()
+ .to_stats()
+ .await
+ };
+
+ assert_eq!(first_save_stats.flushes, initial_stats.flushes + 1);
+
+ cursor.remove("cats").await.unwrap();
+ cursor.save(None).await.unwrap();
+
+ let second_save_stats = {
+ sphere_context
+ .lock()
+ .await
+ .db()
+ .to_block_store()
+ .to_stats()
+ .await
+ };
+
+ assert_eq!(second_save_stats.flushes, first_save_stats.flushes + 1);
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_does_not_allow_writes_when_an_author_has_read_only_access() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::Readonly, None)
+ .await
+ .unwrap();
+
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ let write_result = cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are great".as_ref(),
+ None,
+ )
+ .await;
+
+ assert!(write_result.is_err());
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_write_a_file_and_read_it_back() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are great".as_ref(),
+ None,
+ )
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+
+ let mut file = cursor.read("cats").await.unwrap().unwrap();
+
+ file.memo
+ .expect_header(&Header::ContentType, &ContentType::Subtext)
+ .unwrap();
+
+ let mut value = String::new();
+ file.contents.read_to_string(&mut value).await.unwrap();
+
+ assert_eq!("Cats are great", value.as_str());
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_overwrite_a_file_with_new_contents_and_preserve_history() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are great".as_ref(),
+ None,
+ )
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+
+ cursor
+ .write(
+ "cats",
+ &ContentType::Subtext,
+ b"Cats are better than dogs".as_ref(),
+ None,
+ )
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+
+ let mut file = cursor.read("cats").await.unwrap().unwrap();
+
+ file.memo
+ .expect_header(&Header::ContentType, &ContentType::Subtext)
+ .unwrap();
+
+ let mut value = String::new();
+ file.contents.read_to_string(&mut value).await.unwrap();
+
+ assert_eq!("Cats are better than dogs", value.as_str());
+
+ assert!(cursor.rewind().await.unwrap().is_some());
+
+ file = cursor.read("cats").await.unwrap().unwrap();
+
+ file.memo
+ .expect_header(&Header::ContentType, &ContentType::Subtext)
+ .unwrap();
+
+ value.clear();
+ file.contents.read_to_string(&mut value).await.unwrap();
+
+ assert_eq!("Cats are great", value.as_str());
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_throws_an_error_when_saving_without_changes() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ let result = cursor.save(None).await;
+
+ assert!(result.is_err());
+ assert_eq!(result.unwrap_err().to_string(), "No changes to save");
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_throws_an_error_when_saving_with_empty_mutation_and_empty_headers() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ let result = cursor.save(Some(vec![])).await;
+
+ assert!(result.is_err());
+ assert_eq!(result.unwrap_err().to_string(), "No changes to save");
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_get_all_petnames_assigned_to_an_identity() -> Result<()> {
+ let (sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let mut db = UcanStore(sphere_context.sphere_context().await?.db().clone());
+
+ let (peer_1, link_record_1, _) = make_valid_link_record(&mut db).await?;
+ let (peer_2, link_record_2, _) = make_valid_link_record(&mut db).await?;
+ let (peer_3, link_record_3, _) = make_valid_link_record(&mut db).await?;
+
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ cursor
+ .set_petname("foo1", Some(link_record_1.to_sphere_identity()))
+ .await?;
+ cursor
+ .set_petname("bar1", Some(link_record_1.to_sphere_identity()))
+ .await?;
+ cursor
+ .set_petname("baz1", Some(link_record_1.to_sphere_identity()))
+ .await?;
+
+ cursor
+ .set_petname("foo2", Some(link_record_2.to_sphere_identity()))
+ .await?;
+ cursor.save(None).await?;
+
+ cursor.set_petname_record("foo1", &link_record_1).await?;
+ cursor.set_petname_record("bar1", &link_record_1).await?;
+ cursor.set_petname_record("baz1", &link_record_1).await?;
+
+ cursor.set_petname_record("foo2", &link_record_2).await?;
+
+ cursor.save(None).await?;
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_1).await?,
+ vec![
+ String::from("foo1"),
+ String::from("bar1"),
+ String::from("baz1")
+ ]
+ );
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_2).await?,
+ vec![String::from("foo2")]
+ );
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_3).await?,
+ Vec::<String>::new()
+ );
+
+ // Check one more time for good measure, since results are cached internally
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_1).await?,
+ vec![
+ String::from("foo1"),
+ String::from("bar1"),
+ String::from("baz1")
+ ]
+ );
+
+ cursor
+ .set_petname("bar2", Some(link_record_2.to_sphere_identity()))
+ .await?;
+ cursor
+ .set_petname("foo3", Some(link_record_3.to_sphere_identity()))
+ .await?;
+ cursor.save(None).await?;
+
+ cursor.set_petname_record("bar2", &link_record_2).await?;
+ cursor.set_petname_record("foo3", &link_record_3).await?;
+ cursor.save(None).await?;
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_1).await?,
+ vec![
+ String::from("foo1"),
+ String::from("bar1"),
+ String::from("baz1")
+ ]
+ );
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_2).await?,
+ vec![String::from("bar2"), String::from("foo2")]
+ );
+
+ assert_eq!(
+ cursor.get_assigned_petnames(&peer_3).await?,
+ vec![String::from("foo3")]
+ );
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_traverse_a_sequence_of_petnames() -> Result<()> {
+ initialize_tracing(None);
+
+ let name_seqeuence: Vec<String> = vec!["a".into(), "b".into(), "c".into()];
+ let (origin_sphere_context, _) =
+ make_sphere_context_with_peer_chain(&name_seqeuence).await?;
+
+ let cursor = SphereCursor::latest(Arc::new(
+ origin_sphere_context.sphere_context().await?.clone(),
+ ));
+
+ let target_sphere_context = cursor
+ .traverse_by_petnames(&name_seqeuence.into_iter().rev().collect::<Vec<String>>())
+ .await?
+ .unwrap();
+
+ let mut name = String::new();
+ let mut file = target_sphere_context.read("my-name").await?.unwrap();
+ file.contents.read_to_string(&mut name).await?;
+
+ assert_eq!(name.as_str(), "c");
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_resolves_none_when_a_petname_is_missing_from_the_sequence() -> Result<()> {
+ initialize_tracing(None);
+
+ let name_sequence: Vec<String> = vec!["b".into(), "c".into()];
+ let (origin_sphere_context, _) =
+ make_sphere_context_with_peer_chain(&name_sequence).await?;
+
+ let cursor = SphereCursor::latest(Arc::new(
+ origin_sphere_context.sphere_context().await?.clone(),
+ ));
+
+ let traversed_sequence: Vec<String> = vec!["a".into(), "b".into(), "c".into()];
+
+ let target_sphere_context = cursor
+ .traverse_by_petnames(
+ &traversed_sequence
+ .into_iter()
+ .rev()
+ .collect::<Vec<String>>(),
+ )
+ .await
+ .unwrap();
+
+ assert!(target_sphere_context.is_none());
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_correctly_identifies_a_visited_perer() -> Result<()> {
+ initialize_tracing(None);
+
+ let name_sequence: Vec<String> = vec!["a".into(), "b".into(), "c".into()];
+
+ let (origin_sphere_context, dids) =
+ make_sphere_context_with_peer_chain(&name_sequence).await?;
+
+ let cursor = SphereCursor::latest(Arc::new(
+ origin_sphere_context.sphere_context().await?.clone(),
+ ));
+
+ let mut target_sphere_context = cursor;
+ let mut identities = vec![target_sphere_context.identity().await?];
+
+ for name in name_sequence.iter() {
+ target_sphere_context = target_sphere_context
+ .traverse_by_petnames(&[name.clone()])
+ .await?
+ .unwrap();
+ identities.push(target_sphere_context.identity().await?);
+ }
+
+ assert_eq!(identities.into_iter().rev().collect::<Vec<_>>(), dids);
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_traverse_a_sequence_of_petnames_one_at_a_time() -> Result<()> {
+ initialize_tracing(None);
+
+ let name_sequence: Vec<String> = vec!["a".into(), "b".into(), "c".into()];
+
+ let (origin_sphere_context, _) =
+ make_sphere_context_with_peer_chain(&name_sequence).await?;
+
+ let cursor = SphereCursor::latest(Arc::new(
+ origin_sphere_context.sphere_context().await?.clone(),
+ ));
+
+ let mut target_sphere_context = cursor;
+
+ for name in name_sequence.iter() {
+ target_sphere_context = target_sphere_context
+ .traverse_by_petnames(&[name.clone()])
+ .await?
+ .unwrap();
+ }
+
+ let mut name = String::new();
+ let mut file = target_sphere_context
+ .read("my-name")
+ .await
+ .unwrap()
+ .unwrap();
+ file.contents.read_to_string(&mut name).await.unwrap();
+
+ assert_eq!(name.as_str(), "c");
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/context/has.rs b/rust/noosphere-core/src/context/has.rs
new file mode 100644
index 000000000..b9fca8f5c
--- /dev/null
+++ b/rust/noosphere-core/src/context/has.rs
@@ -0,0 +1,223 @@
+use crate::{
+ authority::Author,
+ data::{Did, Link, MemoIpld},
+ view::Sphere,
+};
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use noosphere_storage::{SphereDb, Storage};
+use std::{
+ ops::{Deref, DerefMut},
+ sync::Arc,
+};
+use tokio::sync::{Mutex, OwnedMutexGuard};
+
+use crate::context::SphereContextKey;
+
+use super::SphereContext;
+
+#[allow(missing_docs)]
+#[cfg(not(target_arch = "wasm32"))]
+pub trait HasConditionalSendSync: Send + Sync {}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<S> HasConditionalSendSync for S where S: Send + Sync {}
+
+#[allow(missing_docs)]
+#[cfg(target_arch = "wasm32")]
+pub trait HasConditionalSendSync {}
+
+#[cfg(target_arch = "wasm32")]
+impl<S> HasConditionalSendSync for S {}
+
+/// Any container that can provide non-mutable access to a [SphereContext]
+/// should implement [HasSphereContext]. The most common example of something
+/// that may implement this trait is an `Arc<SphereContext<_, _>>`. Implementors
+/// of this trait will automatically implement other traits that provide
+/// convience methods for accessing different parts of the sphere, such as
+/// content and petnames.
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait HasSphereContext<S>: Clone + HasConditionalSendSync
+where
+ S: Storage + 'static,
+{
+ /// The type of the internal read-only [SphereContext]
+ type SphereContext: Deref<Target = SphereContext<S>> + HasConditionalSendSync;
+
+ /// Get the [SphereContext] that is made available by this container.
+ async fn sphere_context(&self) -> Result<Self::SphereContext>;
+
+ /// Get the DID identity of the sphere that this FS view is reading from and
+ /// writing to
+ async fn identity(&self) -> Result<Did> {
+ let sphere_context = self.sphere_context().await?;
+
+ Ok(sphere_context.identity().clone())
+ }
+
+ /// The CID of the most recent local version of this sphere
+ async fn version(&self) -> Result<Link<MemoIpld>> {
+ self.sphere_context().await?.version().await
+ }
+
+ /// Get a data view into the sphere at the current revision
+ async fn to_sphere(&self) -> Result<Sphere<SphereDb<S>>> {
+ let version = self.version().await?;
+ Ok(Sphere::at(&version, self.sphere_context().await?.db()))
+ }
+
+ /// Create a new [SphereContext] via [SphereContext::with_author] and wrap it in the same
+ /// [HasSphereContext] implementation, returning the result
+ async fn with_author(&self, author: &Author<SphereContextKey>) -> Result<Self> {
+ Ok(Self::wrap(self.sphere_context().await?.with_author(author).await?).await)
+ }
+
+ /// Wrap a given [SphereContext] in this [HasSphereContext]
+ async fn wrap(sphere_context: SphereContext<S>) -> Self;
+}
+
+/// Any container that can provide mutable access to a [SphereContext] should
+/// implement [HasMutableSphereContext]. The most common example of something
+/// that may implement this trait is `Arc<Mutex<SphereContext<_, _>>>`.
+/// Implementors of this trait will automatically implement other traits that
+/// provide convenience methods for modifying the contents, petnames and other
+/// aspects of a sphere.
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait HasMutableSphereContext<S>: HasSphereContext<S> + HasConditionalSendSync
+where
+ S: Storage + 'static,
+{
+ /// The type of the internal mutable [SphereContext]
+ type MutableSphereContext: Deref<Target = SphereContext<S>>
+ + DerefMut<Target = SphereContext<S>>
+ + HasConditionalSendSync;
+
+ /// Get a mutable reference to the [SphereContext] that is wrapped by this
+ /// container.
+ async fn sphere_context_mut(&mut self) -> Result<Self::MutableSphereContext>;
+
+ /// Returns true if any changes have been made to the underlying
+ /// [SphereContext] that have not been committed to the associated sphere
+ /// yet (according to local history).
+ async fn has_unsaved_changes(&self) -> Result<bool> {
+ let context = self.sphere_context().await?;
+ Ok(!context.mutation().is_empty())
+ }
+
+ /// Commits a series of writes to the sphere and signs the new version. The
+ /// new version [Link<MemoIpld>] of the sphere is returned. This method must
+ /// be invoked in order to update the local history of the sphere with any
+ /// changes that have been made.
+ async fn save(
+ &mut self,
+ additional_headers: Option<Vec<(String, String)>>,
+ ) -> Result<Link<MemoIpld>> {
+ let sphere = self.to_sphere().await?;
+ let mut sphere_context = self.sphere_context_mut().await?;
+ let sphere_identity = sphere_context.identity().clone();
+ let mut revision = sphere.apply_mutation(sphere_context.mutation()).await?;
+
+ match additional_headers {
+ Some(headers) if !headers.is_empty() => revision.memo.replace_headers(headers),
+ _ if sphere_context.mutation().is_empty() => return Err(anyhow!("No changes to save")),
+ _ => (),
+ }
+
+ let new_sphere_version = revision
+ .sign(
+ &sphere_context.author().key,
+ sphere_context.author().authorization.as_ref(),
+ )
+ .await?;
+
+ sphere_context
+ .db_mut()
+ .set_version(&sphere_identity, &new_sphere_version)
+ .await?;
+ sphere_context.db_mut().flush().await?;
+ sphere_context.mutation_mut().reset();
+
+ Ok(new_sphere_version)
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<S> HasSphereContext<S> for Arc<Mutex<SphereContext<S>>>
+where
+ S: Storage + 'static,
+{
+ type SphereContext = OwnedMutexGuard<SphereContext<S>>;
+
+ async fn sphere_context(&self) -> Result<Self::SphereContext> {
+ Ok(self.clone().lock_owned().await)
+ }
+
+ async fn wrap(sphere_context: SphereContext<S>) -> Self {
+ Arc::new(Mutex::new(sphere_context))
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<S, T> HasSphereContext<S> for Box<T>
+where
+ T: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ type SphereContext = T::SphereContext;
+
+ async fn sphere_context(&self) -> Result<Self::SphereContext> {
+ T::sphere_context(self).await
+ }
+
+ async fn wrap(sphere_context: SphereContext<S>) -> Self {
+ Box::new(T::wrap(sphere_context).await)
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<S> HasSphereContext<S> for Arc<SphereContext<S>>
+where
+ S: Storage,
+{
+ type SphereContext = Arc<SphereContext<S>>;
+
+ async fn sphere_context(&self) -> Result<Self::SphereContext> {
+ Ok(self.clone())
+ }
+
+ async fn wrap(sphere_context: SphereContext<S>) -> Self {
+ Arc::new(sphere_context)
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<S> HasMutableSphereContext<S> for Arc<Mutex<SphereContext<S>>>
+where
+ S: Storage + 'static,
+{
+ type MutableSphereContext = OwnedMutexGuard<SphereContext<S>>;
+
+ async fn sphere_context_mut(&mut self) -> Result<Self::MutableSphereContext> {
+ self.sphere_context().await
+ }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<S, T> HasMutableSphereContext<S> for Box<T>
+where
+ T: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ type MutableSphereContext = T::MutableSphereContext;
+
+ async fn sphere_context_mut(&mut self) -> Result<Self::MutableSphereContext> {
+ T::sphere_context_mut(self).await
+ }
+}
diff --git a/rust/noosphere-core/src/context/internal.rs b/rust/noosphere-core/src/context/internal.rs
new file mode 100644
index 000000000..af05fb5c6
--- /dev/null
+++ b/rust/noosphere-core/src/context/internal.rs
@@ -0,0 +1,105 @@
+use super::{BodyChunkDecoder, SphereFile};
+use crate::{
+ context::{AsyncFileBody, HasSphereContext},
+ stream::put_block_stream,
+};
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use noosphere_storage::{BlockStore, Storage};
+use std::str::FromStr;
+use tokio_util::io::StreamReader;
+
+use crate::{
+ authority::Access,
+ data::{ContentType, Header, Link, MemoIpld},
+};
+use cid::Cid;
+
+/// A module-private trait for internal trait methods; this is a workaround for
+/// the fact that all trait methods are implicitly public implementation
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub(crate) trait SphereContextInternal<S>
+where
+ S: Storage + 'static,
+{
+ /// Returns an error result if the configured author of the [SphereContext]
+ /// does not have write access to it (as a matter of cryptographic
+ /// authorization).
+ async fn assert_write_access(&self) -> Result<()>;
+
+ async fn get_file(
+ &self,
+ sphere_revision: &Cid,
+ memo_link: Link<MemoIpld>,
+ ) -> Result<SphereFile<Box<dyn AsyncFileBody>>>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereContextInternal<S> for C
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ async fn assert_write_access(&self) -> Result<()> {
+ let sphere_context = self.sphere_context().await?;
+ match sphere_context.access().await? {
+ Access::ReadOnly => Err(anyhow!(
+ "Cannot mutate sphere; author only has read access to its contents"
+ )),
+ _ => Ok(()),
+ }
+ }
+
+ async fn get_file(
+ &self,
+ sphere_revision: &Cid,
+ memo_link: Link<MemoIpld>,
+ ) -> Result<SphereFile<Box<dyn AsyncFileBody>>> {
+ let db = self.sphere_context().await?.db().clone();
+ let memo = memo_link.load_from(&db).await?;
+
+ // If we have a memo, but not the content it refers to, we should try to
+ // replicate from the gateway
+ if db.get_block(&memo.body).await?.is_none() {
+ let client = self
+ .sphere_context()
+ .await?
+ .client()
+ .await
+ .map_err(|error| {
+ warn!("Unable to initialize API client for replicating missing content");
+ error
+ })?;
+
+ // NOTE: This is kind of a hack, since we may be accessing a
+ // "read-only" context. Technically this should be acceptable
+ // because our mutation here is propagating immutable blocks
+ // into the local DB
+ let stream = client.replicate(&memo_link, None).await?;
+
+ put_block_stream(db.clone(), stream).await?;
+ }
+
+ let content_type = match memo.get_first_header(&Header::ContentType) {
+ Some(content_type) => Some(ContentType::from_str(content_type.as_str())?),
+ None => None,
+ };
+
+ let stream = match content_type {
+ // TODO(#86): Content-type aware decoding of body bytes
+ Some(_) => BodyChunkDecoder(&memo.body, &db).stream(),
+ None => return Err(anyhow!("No content type specified")),
+ };
+
+ Ok(SphereFile {
+ sphere_identity: self.sphere_context().await?.identity().clone(),
+ sphere_version: sphere_revision.into(),
+ memo_version: memo_link,
+ memo,
+ // NOTE: we have to box here because traits don't support `impl` types in return values
+ contents: Box::new(StreamReader::new(stream)),
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/context/metadata.rs b/rust/noosphere-core/src/context/metadata.rs
new file mode 100644
index 000000000..ce30a5a7f
--- /dev/null
+++ b/rust/noosphere-core/src/context/metadata.rs
@@ -0,0 +1,36 @@
+//! These constants represent the metadata keys used when a [SphereContext] is
+//! is initialized. Since these represent somewhat free-form key/values in the
+//! storage layer, we are make a best effort to document them here.
+
+#[cfg(doc)]
+use crate::context::SphereContext;
+
+#[cfg(doc)]
+use crate::data::Did;
+
+#[cfg(doc)]
+use cid::Cid;
+
+#[cfg(doc)]
+use url::Url;
+
+/// A key that corresponds to the sphere's identity, which is represented by a
+/// [Did] when it is set.
+pub const IDENTITY: &str = "identity";
+
+/// A name that corresponds to the locally available key. This name is
+/// represented as a string, and should match a credential ID for a key in
+/// whatever the supported platform key storage is.
+pub const USER_KEY_NAME: &str = "user_key_name";
+
+/// The [Cid] of a UCAN JWT that authorizes the configured user key to access
+/// the sphere.
+pub const AUTHORIZATION: &str = "authorization";
+
+/// The base [Url] of a Noosphere Gateway API that will allow this sphere to
+/// sync with it.
+pub const GATEWAY_URL: &str = "gateway_url";
+
+/// The counterpart sphere [Did] that either tracks or is tracked by this
+/// sphere.
+pub const COUNTERPART: &str = "counterpart";
diff --git a/rust/noosphere-core/src/context/mod.rs b/rust/noosphere-core/src/context/mod.rs
new file mode 100644
index 000000000..c95cd0ab9
--- /dev/null
+++ b/rust/noosphere-core/src/context/mod.rs
@@ -0,0 +1,94 @@
+//! This module implements content, petname and other forms of acccess to
+//! spheres. If you have storage and network primitives on your platform, you
+//! can initialize a [SphereContext] and use it to work with and synchronize
+//! spheres, as well as traverse the broader Noosphere data graph.
+//!
+//! In order to initialize a [SphereContext], you need a [Did] (like an ID) for
+//! a Sphere, a [Storage] primitive and an [Author] (which represents whoever or
+//! whatever is trying to access the Sphere inquestion).
+//!
+//! Once you have a [SphereContext], you can begin reading from, writing to and
+//! traversing the Noosphere content graph.
+//!
+//! ```rust
+//! # use anyhow::Result;
+//! # use noosphere_core::context::{SphereCursor, HasMutableSphereContext, SphereContentWrite};
+//! #
+//! # #[cfg(feature = "helpers")]
+//! # use noosphere_core::helpers::{simulated_sphere_context,SimulationAccess};
+//! #
+//! # #[cfg(feature = "helpers")]
+//! # #[tokio::main(flavor = "multi_thread")]
+//! # async fn main() -> Result<()> {
+//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+//! #
+//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?;
+//! sphere_context.save(None).await?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # #[cfg(not(feature = "helpers"))]
+//! # fn main() {}
+//! ```
+//!
+//! You can also use a [SphereContext] to access petnames in the sphere:
+//!
+//! ```rust
+//! # use anyhow::Result;
+//! # #[cfg(feature = "helpers")]
+//! # use noosphere_core::{
+//! # helpers::{simulated_sphere_context,SimulationAccess},
+//! # data::Did,
+//! # context::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite}
+//! # };
+//! #
+//! # #[cfg(feature = "helpers")]
+//! # #[tokio::main(flavor = "multi_thread")]
+//! # async fn main() -> Result<()> {
+//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+//! #
+//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?;
+//! sphere_context.save(None).await?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # #[cfg(not(feature = "helpers"))]
+//! # fn main() {}
+//! ```
+//!
+//!
+
+#![warn(missing_docs)]
+
+#[cfg(doc)]
+use crate::{authority::Author, data::Did};
+
+#[cfg(doc)]
+use noosphere_storage::Storage;
+
+mod authority;
+mod content;
+#[allow(clippy::module_inception)]
+mod context;
+mod cursor;
+mod has;
+mod replication;
+mod walker;
+
+mod internal;
+pub mod metadata;
+mod petname;
+mod sync;
+
+pub use authority::*;
+pub use content::*;
+pub use context::*;
+pub use cursor::*;
+pub use has::*;
+pub use metadata::*;
+pub use petname::*;
+pub use replication::*;
+pub use sync::*;
+pub use walker::*;
diff --git a/rust/noosphere-core/src/context/petname/mod.rs b/rust/noosphere-core/src/context/petname/mod.rs
new file mode 100644
index 000000000..b1ad86483
--- /dev/null
+++ b/rust/noosphere-core/src/context/petname/mod.rs
@@ -0,0 +1,10 @@
+//! Sphere petnames are shorthand names that are associated with DIDs. A petname
+//! can be any string, and always refers to a DID. The DID, in turn, may be
+//! resolved to a CID that represents the tip of history for the sphere that is
+//! implicitly identified by the provided DID.
+
+mod read;
+mod write;
+
+pub use read::*;
+pub use write::*;
diff --git a/rust/noosphere-core/src/context/petname/read.rs b/rust/noosphere-core/src/context/petname/read.rs
new file mode 100644
index 000000000..baa29e803
--- /dev/null
+++ b/rust/noosphere-core/src/context/petname/read.rs
@@ -0,0 +1,138 @@
+use std::collections::BTreeMap;
+
+use crate::data::{Did, Link, LinkRecord, MemoIpld};
+use anyhow::Result;
+use async_trait::async_trait;
+use cid::Cid;
+use futures_util::TryStreamExt;
+use noosphere_storage::{KeyValueStore, Storage};
+
+use crate::context::{HasSphereContext, SphereWalker};
+
+/// Anything that provides read access to petnames in a sphere should implement
+/// [SpherePetnameRead]. A blanket implementation is provided for any container
+/// that implements [HasSphereContext].
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SpherePetnameRead<S>
+where
+ S: Storage + 'static,
+{
+ /// Get the [Did] that is assigned to a petname, if any
+ async fn get_petname(&self, name: &str) -> Result<Option<Did>>;
+
+ /// Resolve the petname via its assigned [Did] to a [Cid] that refers to a
+ /// point in history of a sphere
+ async fn resolve_petname(&self, name: &str) -> Result<Option<Link<MemoIpld>>>;
+
+ /// Given a [Did], get all the petnames that have been assigned to it
+ /// in this sphere
+ async fn get_assigned_petnames(&self, did: &Did) -> Result<Vec<String>>;
+
+ /// Given a petname, get the raw last known [LinkRecord] for that peer
+ async fn get_petname_record(&self, name: &str) -> Result<Option<LinkRecord>>;
+}
+
+fn assigned_petnames_cache_key(origin: &Did, peer: &Did, origin_version: &Cid) -> String {
+ format!(
+ "noosphere:cache:petname:assigned:{}:{}:{}",
+ origin, peer, origin_version
+ )
+}
+
+fn sphere_checkpoint_cache_key(origin: &Did, origin_version: &Cid) -> String {
+ format!(
+ "noosphere:cache:petname:checkpoint:{}:{}",
+ origin, origin_version
+ )
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SpherePetnameRead<S> for C
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ #[instrument(skip(self))]
+ async fn get_assigned_petnames(&self, peer: &Did) -> Result<Vec<String>> {
+ let version = self.version().await?;
+ let origin = self.identity().await?;
+
+ debug!("Getting petnames assigned in {origin} at version {version}");
+
+ let mut db = self.sphere_context().await?.db().clone();
+ let key = assigned_petnames_cache_key(&origin, peer, &version);
+
+ if let Some(names) = db.get_key::<_, Vec<String>>(key).await? {
+ return Ok(names);
+ }
+
+ let checkpoint_key = sphere_checkpoint_cache_key(&origin, &version);
+
+ if db.get_key::<_, u8>(&checkpoint_key).await?.is_some() {
+ warn!("No names were assigned to {peer}",);
+ return Ok(vec![]);
+ }
+
+ let walker = SphereWalker::from(self);
+ let petname_stream = walker.petname_stream();
+ let mut did_petnames: BTreeMap<Did, Vec<String>> = BTreeMap::new();
+
+ tokio::pin!(petname_stream);
+
+ while let Some((petname, identity)) = petname_stream.try_next().await? {
+ match did_petnames.get_mut(&identity.did) {
+ Some(petnames) => {
+ petnames.push(petname);
+ }
+ None => {
+ did_petnames.insert(identity.did, vec![petname]);
+ }
+ };
+ }
+
+ let mut assigned_petnames = None;
+
+ for (did, petnames) in did_petnames {
+ if &did == peer {
+ assigned_petnames = Some(petnames.clone());
+ }
+
+ let key = assigned_petnames_cache_key(&origin, &did, &version);
+ db.set_key(key, petnames).await?;
+ }
+
+ db.set_key(checkpoint_key, 1u8).await?;
+
+ Ok(assigned_petnames.unwrap_or_default())
+ }
+
+ async fn get_petname(&self, name: &str) -> Result<Option<Did>> {
+ let sphere = self.to_sphere().await?;
+ let identities = sphere.get_address_book().await?.get_identities().await?;
+ let address_ipld = identities.get(&name.to_string()).await?;
+
+ Ok(address_ipld.map(|ipld| ipld.did.clone()))
+ }
+
+ async fn resolve_petname(&self, name: &str) -> Result<Option<Link<MemoIpld>>> {
+ Ok(match self.get_petname_record(name).await? {
+ Some(link_record) => link_record.get_link(),
+ None => None,
+ })
+ }
+
+ async fn get_petname_record(&self, name: &str) -> Result<Option<LinkRecord>> {
+ let sphere = self.to_sphere().await?;
+ let identities = sphere.get_address_book().await?.get_identities().await?;
+ let address_ipld = identities.get(&name.to_string()).await?;
+
+ trace!("Recorded address for {name}: {:?}", address_ipld);
+
+ Ok(match address_ipld {
+ Some(identity) => identity.link_record(sphere.store()).await,
+ None => None,
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/context/petname/write.rs b/rust/noosphere-core/src/context/petname/write.rs
new file mode 100644
index 000000000..a782a936c
--- /dev/null
+++ b/rust/noosphere-core/src/context/petname/write.rs
@@ -0,0 +1,192 @@
+use crate::data::{Did, IdentityIpld, LinkRecord};
+use anyhow::{anyhow, Result};
+use async_trait::async_trait;
+use noosphere_storage::Storage;
+use ucan::store::UcanJwtStore;
+
+use crate::context::{internal::SphereContextInternal, HasMutableSphereContext, SpherePetnameRead};
+
+fn validate_petname(petname: &str) -> Result<()> {
+ if petname.is_empty() {
+ Err(anyhow!("Petname must not be empty."))
+ } else if petname.len() >= 4 && petname.starts_with("did:") {
+ Err(anyhow!("Petname must not be a DID."))
+ } else {
+ Ok(())
+ }
+}
+
+/// Anything that can write petnames to a sphere should implement
+/// [SpherePetnameWrite]. A blanket implementation is provided for anything that
+/// implements [HasMutableSphereContext]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SpherePetnameWrite<S>: SpherePetnameRead<S>
+where
+ S: Storage + 'static,
+{
+ /// Configure a petname, by assigning some [Did] to it or none. By assigning
+ /// none, the petname is implicitly removed from the address space (note:
+ /// this does not erase the name from historical versions of the sphere). If
+ /// a name is set that already exists, the previous name shall be
+ /// overwritten by the new one, and any associated [Jwt] shall be unset.
+ async fn set_petname(&mut self, name: &str, identity: Option<Did>) -> Result<()>;
+
+ /// Set the [LinkRecord] associated with a petname. The [LinkRecord] must
+ /// resolve a valid UCAN that authorizes the corresponding sphere to be
+ /// published and grants sufficient authority from the configured [Did] to
+ /// the publisher. The audience of the UCAN must match the [Did] that was
+ /// most recently assigned the associated petname. Note that a petname
+ /// _must_ be assigned to the audience [Did] in order for the record to be
+ /// set.
+ async fn set_petname_record(&mut self, name: &str, record: &LinkRecord) -> Result<Option<Did>>;
+
+ /// Deprecated; use [SpherePetnameWrite::set_petname_record] instead
+ #[deprecated(note = "Use set_petname_record instead")]
+ async fn adopt_petname(&mut self, name: &str, record: &LinkRecord) -> Result<Option<Did>>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SpherePetnameWrite<S> for C
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ async fn set_petname(&mut self, name: &str, identity: Option<Did>) -> Result<()> {
+ self.assert_write_access().await?;
+ validate_petname(name)?;
+
+ if identity.is_some()
+ && self.sphere_context().await?.identity() == identity.as_ref().unwrap()
+ {
+ return Err(anyhow!("Sphere cannot assign itself to a petname."));
+ }
+
+ let current_address = self.get_petname(name).await?;
+
+ if identity != current_address {
+ let mut context = self.sphere_context_mut().await?;
+ match identity {
+ Some(identity) => {
+ context.mutation_mut().identities_mut().set(
+ &name.to_string(),
+ &IdentityIpld {
+ did: identity,
+ // TODO: We should backfill this if we have already resolved
+ // this address by another name
+ link_record: None,
+ },
+ );
+ }
+ None => context
+ .mutation_mut()
+ .identities_mut()
+ .remove(&name.to_string()),
+ };
+ }
+
+ Ok(())
+ }
+
+ async fn adopt_petname(&mut self, name: &str, record: &LinkRecord) -> Result<Option<Did>> {
+ self.set_petname_record(name, record).await
+ }
+
+ async fn set_petname_record(&mut self, name: &str, record: &LinkRecord) -> Result<Option<Did>> {
+ // NOTE: it is not safe for us to blindly adopt link records that don't
+ // match up with the petname we are adopting them against. For example,
+ // consider the following sequence of events:
+ //
+ // 1. A petname is assigned to a DID
+ // 2. During sync, the gateway kicks off a parallel job to resolve the
+ // petname
+ // 3. Meanwhile, we unassign the petname
+ // 4. We sync, the gateway takes no action (no new names to resolve)
+ // 5. Then, the original resolve job finishes and comes back with a
+ // record
+ //
+ // Record adoption is not able to disambiguate between between a new
+ // record being added and a race condition like the one described above.
+ self.assert_write_access().await?;
+ validate_petname(name)?;
+
+ let identity = record.to_sphere_identity();
+ let expected_identity = self.get_petname(name).await?;
+
+ match expected_identity {
+ Some(expected_identity) => {
+ if expected_identity != identity {
+ return Err(anyhow!(
+ "Cannot adopt petname record for '{}'; expected record for {} but got record for {}",
+ name,
+ expected_identity,
+ identity
+ ));
+ }
+ }
+ None => {
+ return Err(anyhow!(
+ "Cannot adopt petname record for '{}' (not assigned to a sphere identity)",
+ name
+ ));
+ }
+ };
+
+ if self.sphere_context().await?.identity() == &identity {
+ return Err(anyhow!("Sphere cannot assign itself to a petname."));
+ }
+
+ if let Some(existing_record) = self.get_petname_record(name).await? {
+ if !existing_record.superceded_by(record) {
+ return Err(anyhow!(
+ "Previously stored record supercedes provided record."
+ ));
+ }
+ }
+
+ let cid = self
+ .sphere_context_mut()
+ .await?
+ .db_mut()
+ .write_token(&record.encode()?)
+ .await?;
+
+ // TODO: Validate the record as a UCAN
+
+ debug!(
+ "Adopting '{}' ({}), resolving to {}...",
+ name, identity, record
+ );
+
+ let new_address = IdentityIpld {
+ did: identity.clone(),
+ link_record: Some(cid.into()),
+ };
+
+ let identities = self
+ .sphere_context()
+ .await?
+ .sphere()
+ .await?
+ .get_address_book()
+ .await?
+ .get_identities()
+ .await?;
+ let previous_identity = identities.get(&name.into()).await?;
+
+ self.sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .identities_mut()
+ .set(&name.into(), &new_address);
+
+ if let Some(previous_identity) = previous_identity {
+ if identity != previous_identity.did {
+ return Ok(Some(previous_identity.did.to_owned()));
+ }
+ }
+
+ Ok(None)
+ }
+}
diff --git a/rust/noosphere-core/src/context/replication/mod.rs b/rust/noosphere-core/src/context/replication/mod.rs
new file mode 100644
index 000000000..cd2e912fe
--- /dev/null
+++ b/rust/noosphere-core/src/context/replication/mod.rs
@@ -0,0 +1,3 @@
+mod read;
+
+pub use read::*;
diff --git a/rust/noosphere-core/src/context/replication/read.rs b/rust/noosphere-core/src/context/replication/read.rs
new file mode 100644
index 000000000..3318b9d3c
--- /dev/null
+++ b/rust/noosphere-core/src/context/replication/read.rs
@@ -0,0 +1,18 @@
+use anyhow::Result;
+use async_trait::async_trait;
+use noosphere_storage::Storage;
+
+/// Implementors are able to traverse from one sphere to the next via
+/// the address book entries found in those spheres
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereReplicaRead<S>: Sized
+where
+ S: Storage + 'static,
+{
+ /// Accepts a linear sequence of petnames and attempts to recursively
+ /// traverse through spheres using that sequence. The sequence is traversed
+ /// from back to front. So, if the sequence is "gold", "cat", "bob", it will
+ /// traverse to bob, then to bob's cat, then to bob's cat's gold.
+ async fn traverse_by_petnames(&self, petnames: &[String]) -> Result<Option<Self>>;
+}
diff --git a/rust/noosphere-core/src/context/sync/error.rs b/rust/noosphere-core/src/context/sync/error.rs
new file mode 100644
index 000000000..5295155cc
--- /dev/null
+++ b/rust/noosphere-core/src/context/sync/error.rs
@@ -0,0 +1,29 @@
+use crate::api::v0alpha2::PushError;
+use thiserror::Error;
+
+/// Different classes of error that may occur during synchronization with a
+/// gateway
+#[derive(Error, Debug)]
+pub enum SyncError {
+ /// The error was a conflict; this is possibly recoverable
+ #[error("There was a conflict during sync")]
+ Conflict,
+ /// The error was some other, non-specific error
+ #[error("{0}")]
+ Other(anyhow::Error),
+}
+
+impl From<anyhow::Error> for SyncError {
+ fn from(value: anyhow::Error) -> Self {
+ SyncError::Other(value)
+ }
+}
+
+impl From<PushError> for SyncError {
+ fn from(value: PushError) -> Self {
+ match value {
+ PushError::Conflict => SyncError::Conflict,
+ any => SyncError::Other(any.into()),
+ }
+ }
+}
diff --git a/rust/noosphere-core/src/context/sync/mod.rs b/rust/noosphere-core/src/context/sync/mod.rs
new file mode 100644
index 000000000..1ce2773e9
--- /dev/null
+++ b/rust/noosphere-core/src/context/sync/mod.rs
@@ -0,0 +1,9 @@
+mod error;
+mod recovery;
+mod strategy;
+mod write;
+
+pub use error::*;
+pub use recovery::*;
+pub use strategy::*;
+pub use write::*;
diff --git a/rust/noosphere-core/src/context/sync/recovery.rs b/rust/noosphere-core/src/context/sync/recovery.rs
new file mode 100644
index 000000000..28bcdfb0d
--- /dev/null
+++ b/rust/noosphere-core/src/context/sync/recovery.rs
@@ -0,0 +1,9 @@
+/// Recovery strategies for cases when gateway synchronization fails but may be
+/// able to recover gracefully (e.g., when the gateway reports a conflict).
+#[derive(Debug)]
+pub enum SyncRecovery {
+ /// Do not attempt to recover
+ None,
+ /// Automatically retry the synchronization up to a certain number of times
+ Retry(u32),
+}
diff --git a/rust/noosphere-core/src/context/sync/strategy.rs b/rust/noosphere-core/src/context/sync/strategy.rs
new file mode 100644
index 000000000..8587f5ac9
--- /dev/null
+++ b/rust/noosphere-core/src/context/sync/strategy.rs
@@ -0,0 +1,467 @@
+use std::{collections::BTreeMap, marker::PhantomData};
+
+use crate::{
+ api::{
+ v0alpha1::FetchParameters,
+ v0alpha2::{PushBody, PushResponse},
+ },
+ stream::put_block_stream,
+};
+use crate::{
+ authority::{generate_capability, SphereAbility},
+ data::{Did, IdentityIpld, Jwt, Link, MemoIpld, LINK_RECORD_FACT_NAME},
+ view::{Sphere, Timeline},
+};
+use anyhow::{anyhow, Result};
+use noosphere_storage::{KeyValueStore, SphereDb, Storage};
+use tokio_stream::StreamExt;
+use ucan::builder::UcanBuilder;
+
+use crate::context::{
+ metadata::COUNTERPART, HasMutableSphereContext, SpherePetnameRead, SpherePetnameWrite,
+ SyncError,
+};
+
+type HandshakeResults = (Option<Link<MemoIpld>>, Did, Option<Link<MemoIpld>>);
+type FetchResults = (
+ Link<MemoIpld>,
+ Link<MemoIpld>,
+ BTreeMap<String, IdentityIpld>,
+);
+type CounterpartHistory<S> = Vec<Result<(Link<MemoIpld>, Sphere<SphereDb<S>>)>>;
+
+/// The default synchronization strategy is a git-like fetch->rebase->push flow.
+/// It depends on the corresponding history of a "counterpart" sphere that is
+/// owned by a gateway server. As revisions are pushed to the gateway server, it
+/// updates its own sphere to point to the tip of the latest lineage of the
+/// user's. When a new change needs to be synchronized, the latest history of
+/// the counterpart sphere is first fetched, and the local changes are rebased
+/// on the counterpart sphere's reckoning of the authoritative lineage of the
+/// user's sphere. Finally, after the rebase, the reconciled local lineage is
+/// pushed to the gateway.
+pub struct GatewaySyncStrategy<C, S>
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ has_context_type: PhantomData<C>,
+ store_type: PhantomData<S>,
+}
+
+impl<C, S> Default for GatewaySyncStrategy<C, S>
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ fn default() -> Self {
+ Self {
+ has_context_type: Default::default(),
+ store_type: Default::default(),
+ }
+ }
+}
+
+impl<C, S> GatewaySyncStrategy<C, S>
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ /// Synchronize a local sphere's data with the data in a gateway, and rollback
+ /// if there is an error. The returned [Link] is the latest version of the local
+ /// sphere lineage after the sync has completed.
+ pub async fn sync(&self, context: &mut C) -> Result<Link<MemoIpld>, SyncError>
+ where
+ C: HasMutableSphereContext<S>,
+ {
+ let (local_sphere_version, counterpart_sphere_identity, counterpart_sphere_version) =
+ self.handshake(context).await?;
+
+ let result: Result<Link<MemoIpld>, anyhow::Error> = {
+ let (mut local_sphere_version, counterpart_sphere_version, updated_names) = self
+ .fetch_remote_changes(
+ context,
+ local_sphere_version.as_ref(),
+ &counterpart_sphere_identity,
+ counterpart_sphere_version.as_ref(),
+ )
+ .await?;
+
+ if let Some(version) = self.adopt_names(context, updated_names).await? {
+ local_sphere_version = version;
+ }
+
+ self.push_local_changes(
+ context,
+ &local_sphere_version,
+ &counterpart_sphere_identity,
+ &counterpart_sphere_version,
+ )
+ .await?;
+
+ Ok(local_sphere_version)
+ };
+
+ // Rollback if there is an error while syncing
+ if result.is_err() {
+ self.rollback(
+ context,
+ local_sphere_version.as_ref(),
+ &counterpart_sphere_identity,
+ counterpart_sphere_version.as_ref(),
+ )
+ .await?
+ }
+
+ Ok(result?)
+ }
+
+ #[instrument(level = "debug", skip(self, context))]
+ async fn handshake(&self, context: &mut C) -> Result<HandshakeResults> {
+ let mut context = context.sphere_context_mut().await?;
+ let client = context.client().await?;
+ let counterpart_sphere_identity = client.session.sphere_identity.clone();
+
+ // TODO(#561): Some kind of due diligence to notify the caller when this
+ // value changes
+ context
+ .db_mut()
+ .set_key(COUNTERPART, &counterpart_sphere_identity)
+ .await?;
+
+ let local_sphere_identity = context.identity().clone();
+
+ let local_sphere_version = context.db().get_version(&local_sphere_identity).await?;
+ let counterpart_sphere_version = context
+ .db()
+ .get_version(&counterpart_sphere_identity)
+ .await?;
+
+ Ok((
+ local_sphere_version.map(|cid| cid.into()),
+ counterpart_sphere_identity,
+ counterpart_sphere_version.map(|cid| cid.into()),
+ ))
+ }
+
+ /// Fetches the latest changes from a gateway and updates the local lineage
+ /// using a conflict-free rebase strategy
+ #[instrument(level = "debug", skip(self, context))]
+ async fn fetch_remote_changes(
+ &self,
+ context: &mut C,
+ local_sphere_tip: Option<&Link<MemoIpld>>,
+ counterpart_sphere_identity: &Did,
+ counterpart_sphere_base: Option<&Link<MemoIpld>>,
+ ) -> Result<FetchResults> {
+ let mut context = context.sphere_context_mut().await?;
+ let local_sphere_identity = context.identity().clone();
+ let client = context.client().await?;
+
+ let fetch_response = client
+ .fetch(&FetchParameters {
+ since: counterpart_sphere_base.cloned(),
+ })
+ .await?;
+
+ let mut updated_names = BTreeMap::new();
+
+ let (counterpart_sphere_tip, block_stream) = match fetch_response {
+ Some((tip, stream)) => (tip, stream),
+ None => {
+ info!("Local history is already up to date...");
+ let local_sphere_tip = context
+ .db()
+ .require_version(&local_sphere_identity)
+ .await?
+ .into();
+ return Ok((
+ local_sphere_tip,
+ counterpart_sphere_base
+ .ok_or_else(|| anyhow!("Counterpart sphere history is missing!"))?
+ .clone(),
+ updated_names,
+ ));
+ }
+ };
+
+ put_block_stream(context.db_mut().clone(), block_stream).await?;
+
+ trace!("Finished putting block stream");
+
+ let counterpart_history: CounterpartHistory<S> =
+ Sphere::at(&counterpart_sphere_tip, context.db_mut())
+ .into_history_stream(counterpart_sphere_base)
+ .collect()
+ .await;
+
+ trace!("Iterating over counterpart history");
+
+ for item in counterpart_history.into_iter().rev() {
+ let (_, sphere) = item?;
+ sphere.hydrate().await?;
+ updated_names.append(
+ &mut sphere
+ .get_address_book()
+ .await?
+ .get_identities()
+ .await?
+ .get_added()
+ .await?,
+ );
+ }
+
+ let local_sphere_old_base = match counterpart_sphere_base {
+ Some(counterpart_sphere_base) => Sphere::at(counterpart_sphere_base, context.db())
+ .get_content()
+ .await?
+ .get(&local_sphere_identity)
+ .await?
+ .cloned(),
+ None => None,
+ };
+ let local_sphere_new_base = Sphere::at(&counterpart_sphere_tip, context.db())
+ .get_content()
+ .await?
+ .get(&local_sphere_identity)
+ .await?
+ .cloned();
+
+ let local_sphere_tip = match (
+ local_sphere_tip,
+ local_sphere_old_base,
+ local_sphere_new_base,
+ ) {
+ // History diverged, so rebase our local changes on the newly received branch
+ (Some(current_tip), Some(old_base), Some(new_base)) if old_base != new_base => {
+ info!(
+ ?current_tip,
+ ?old_base,
+ ?new_base,
+ "Syncing received local sphere revisions..."
+ );
+ Sphere::at(current_tip, context.db())
+ .rebase(
+ &old_base,
+ &new_base,
+ &context.author().key,
+ context.author().authorization.as_ref(),
+ )
+ .await?
+ }
+ // No diverged history, just new linear history based on our local tip
+ (None, old_base, Some(new_base)) => {
+ info!("Hydrating received local sphere revisions...");
+ let timeline = Timeline::new(context.db_mut());
+ Sphere::hydrate_timeslice(
+ &timeline.slice(&new_base, old_base.as_ref()).exclude_past(),
+ )
+ .await?;
+
+ new_base.clone()
+ }
+ // No new history at all
+ (Some(current_tip), _, _) => {
+ info!("Nothing to sync!");
+ current_tip.clone()
+ }
+ // We should have local history but we don't!
+ _ => {
+ return Err(anyhow!("Missing local history for sphere after sync!"));
+ }
+ };
+
+ context
+ .db_mut()
+ .set_version(&local_sphere_identity, &local_sphere_tip)
+ .await?;
+
+ debug!("Setting counterpart sphere version to {counterpart_sphere_tip}");
+
+ context
+ .db_mut()
+ .set_version(counterpart_sphere_identity, &counterpart_sphere_tip)
+ .await?;
+
+ Ok((local_sphere_tip, counterpart_sphere_tip, updated_names))
+ }
+
+ #[instrument(level = "debug", skip(self, context))]
+ async fn adopt_names(
+ &self,
+ context: &mut C,
+ updated_names: BTreeMap<String, IdentityIpld>,
+ ) -> Result<Option<Link<MemoIpld>>> {
+ if updated_names.is_empty() {
+ return Ok(None);
+ }
+ info!(
+ "Considering {} updated link records for adoption...",
+ updated_names.len()
+ );
+
+ let db = context.sphere_context().await?.db().clone();
+
+ for (name, address) in updated_names.into_iter() {
+ if let Some(link_record) = address.link_record(&db).await {
+ if let Some(identity) = context.get_petname(&name).await? {
+ if identity != address.did {
+ warn!("Updated link record for {name} referred to unexpected sphere; expected {identity}, but record referred to {}; ignoring...", address.did);
+ continue;
+ }
+
+ if context.resolve_petname(&name).await? == link_record.get_link() {
+ // TODO(#562): Should probably also verify record expiry
+ // in case we are dealing with a renewed record to the
+ // same link
+ debug!("Resolved got new link record for {name} but the link has not changed; skipping...");
+ continue;
+ }
+
+ if let Err(e) = context.set_petname_record(&name, &link_record).await {
+ warn!("Could not set petname record: {}", e);
+ continue;
+ }
+ } else {
+ debug!("Not adopting link record for {name}, which is no longer present in the address book")
+ }
+ }
+ }
+
+ Ok(if context.has_unsaved_changes().await? {
+ Some(context.save(None).await?)
+ } else {
+ None
+ })
+ }
+
+ /// Attempts to push the latest local lineage to the gateway, causing the
+ /// gateway to update its own pointer to the tip of the local sphere's history
+ #[instrument(level = "debug", skip(self, context))]
+ async fn push_local_changes(
+ &self,
+ context: &mut C,
+ local_sphere_tip: &Link<MemoIpld>,
+ counterpart_sphere_identity: &Did,
+ counterpart_sphere_tip: &Link<MemoIpld>,
+ ) -> Result<(), SyncError> {
+ let mut context = context.sphere_context_mut().await?;
+
+ let local_sphere_base = Sphere::at(counterpart_sphere_tip, context.db())
+ .get_content()
+ .await?
+ .get(context.identity())
+ .await?
+ .cloned();
+
+ if local_sphere_base.as_ref() == Some(local_sphere_tip) {
+ info!("Gateway is already up to date!");
+ return Ok(());
+ }
+
+ info!("Collecting blocks from new local history...");
+ debug!("Bundling until {:?}", local_sphere_base);
+
+ let client = context.client().await?;
+
+ let local_sphere_identity = context.identity();
+ let authorization = context
+ .author()
+ .require_authorization()?
+ .as_ucan(context.db())
+ .await?;
+
+ let name_record = Jwt(UcanBuilder::default()
+ .issued_by(&context.author().key)
+ .for_audience(local_sphere_identity)
+ .witnessed_by(&authorization, None)
+ .claiming_capability(&generate_capability(
+ local_sphere_identity,
+ SphereAbility::Publish,
+ ))
+ .with_lifetime(120)
+ .with_fact(LINK_RECORD_FACT_NAME, local_sphere_tip.to_string())
+ .build()?
+ .sign()
+ .await?
+ .encode()?);
+
+ info!(
+ "Pushing new local history to gateway {}...",
+ client.session.gateway_identity
+ );
+
+ let result = client
+ .push(&PushBody {
+ sphere: local_sphere_identity.clone(),
+ local_base: local_sphere_base,
+ local_tip: local_sphere_tip.clone(),
+ counterpart_tip: Some(counterpart_sphere_tip.clone()),
+ name_record: Some(name_record),
+ })
+ .await?;
+
+ let counterpart_sphere_updated_tip = match result {
+ PushResponse::Accepted { new_tip } => new_tip,
+ PushResponse::NoChange => {
+ return Err(SyncError::Other(anyhow!("Gateway already up to date!")));
+ }
+ };
+
+ info!("Saving updated counterpart sphere history...");
+
+ // TODO: Do this inside the client `push` method
+ //new_blocks.load_into(context.db_mut()).await?;
+
+ debug!(
+ "Hydrating updated counterpart sphere history (from {} back to {})...",
+ counterpart_sphere_tip, counterpart_sphere_updated_tip
+ );
+
+ let timeline = Timeline::new(context.db_mut());
+ Sphere::hydrate_timeslice(
+ &timeline
+ .slice(
+ &counterpart_sphere_updated_tip,
+ Some(counterpart_sphere_tip),
+ )
+ .exclude_past(),
+ )
+ .await?;
+
+ context
+ .db_mut()
+ .set_version(counterpart_sphere_identity, &counterpart_sphere_updated_tip)
+ .await?;
+
+ Ok(())
+ }
+
+ #[instrument(level = "debug", skip(self, context))]
+ async fn rollback(
+ &self,
+ context: &mut C,
+ original_sphere_version: Option<&Link<MemoIpld>>,
+ counterpart_identity: &Did,
+ original_counterpart_version: Option<&Link<MemoIpld>>,
+ ) -> Result<()> {
+ debug!("Rolling back!");
+ let sphere_identity = context.identity().await?;
+ let mut context = context.sphere_context_mut().await?;
+
+ if let Some(version) = original_sphere_version {
+ context
+ .db_mut()
+ .set_version(&sphere_identity, version)
+ .await?;
+ }
+
+ if let Some(version) = original_counterpart_version {
+ context
+ .db_mut()
+ .set_version(counterpart_identity, version)
+ .await?;
+ }
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/context/sync/write.rs b/rust/noosphere-core/src/context/sync/write.rs
new file mode 100644
index 000000000..1d25829c8
--- /dev/null
+++ b/rust/noosphere-core/src/context/sync/write.rs
@@ -0,0 +1,80 @@
+use crate::data::{Link, MemoIpld};
+use anyhow::Result;
+use async_trait::async_trait;
+use noosphere_storage::Storage;
+
+use crate::context::{HasMutableSphereContext, SyncError, SyncRecovery};
+
+use crate::context::GatewaySyncStrategy;
+
+/// Implementors of [SphereSync] are able to sychronize with a Noosphere gateway
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait SphereSync<S>
+where
+ S: Storage + 'static,
+{
+ /// If a gateway URL has been configured, attempt to synchronize local
+ /// sphere data with the gateway. Changes on the gateway will first be
+ /// fetched to local storage. Then, the local changes will be replayed on
+ /// top of those changes. Finally, the synchronized local history will be
+ /// pushed up to the gateway.
+ ///
+ /// The returned [Link] is the latest version of the local
+ /// sphere lineage after the sync has completed.
+ async fn sync(&mut self, recovery: SyncRecovery) -> Result<Link<MemoIpld>, SyncError>;
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl<C, S> SphereSync<S> for C
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ #[instrument(level = "debug", skip(self))]
+ async fn sync(&mut self, recovery: SyncRecovery) -> Result<Link<MemoIpld>, SyncError> {
+ debug!("Attempting to sync...");
+
+ let sync_strategy = GatewaySyncStrategy::default();
+
+ let version = match recovery {
+ SyncRecovery::None => sync_strategy.sync(self).await?,
+ SyncRecovery::Retry(max_retries) => {
+ let mut retries = 0;
+ let version;
+
+ loop {
+ match sync_strategy.sync(self).await {
+ Ok(result) => {
+ debug!("Sync success with {retries} retries");
+ version = result;
+ break;
+ }
+ Err(SyncError::Conflict) => {
+ if retries < max_retries {
+ warn!(
+ "Sync conflict; {} retries remaining...",
+ max_retries - retries
+ );
+ retries += 1;
+ } else {
+ warn!("Sync conflict; no retries remaining!");
+ return Err(SyncError::Conflict);
+ }
+ }
+ Err(other) => {
+ return Err(other);
+ }
+ }
+ }
+
+ version
+ }
+ };
+
+ self.sphere_context_mut().await?.reset_access();
+
+ Ok(version)
+ }
+}
diff --git a/rust/noosphere-core/src/context/walker.rs b/rust/noosphere-core/src/context/walker.rs
new file mode 100644
index 000000000..ef5e1a898
--- /dev/null
+++ b/rust/noosphere-core/src/context/walker.rs
@@ -0,0 +1,603 @@
+use crate::data::{Did, IdentityIpld, Jwt, Link, MapOperation, MemoIpld};
+use anyhow::Result;
+use std::{collections::BTreeSet, marker::PhantomData};
+
+use async_stream::try_stream;
+use noosphere_storage::Storage;
+use tokio::io::AsyncRead;
+use tokio_stream::{Stream, StreamExt};
+
+use crate::context::{
+ content::{SphereContentRead, SphereFile},
+ internal::SphereContextInternal,
+ HasSphereContext, SphereAuthorityRead, SpherePetnameRead,
+};
+
+/// A [SphereWalker] makes it possible to convert anything that implements
+/// [HasSphereContext] into an async [Stream] over sphere content, allowing
+/// incremental iteration over both the breadth of content at any version, or
+/// the depth of changes over a range of history.
+pub struct SphereWalker<'a, C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ has_sphere_context: &'a C,
+ storage: PhantomData<S>,
+}
+
+impl<'a, C, S> From<&'a C> for SphereWalker<'a, C, S>
+where
+ C: HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ fn from(has_sphere_context: &'a C) -> Self {
+ SphereWalker {
+ has_sphere_context,
+ storage: Default::default(),
+ }
+ }
+}
+
+impl<'a, C, S> SphereWalker<'a, C, S>
+where
+ C: SphereAuthorityRead<S> + HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ /// Get a stream that yields a link to every authorization to access the
+ /// sphere along with its corresponding [DelegationIpld]. Note that since a
+ /// revocation may be issued without necessarily removing its revoked
+ /// delegation, this will yield all authorizations regardless of revocation
+ /// status.
+ pub fn authorization_stream(
+ &self,
+ ) -> impl Stream<Item = Result<(String, Did, Link<Jwt>)>> + '_ {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let delegations = sphere.get_authority().await?.get_delegations().await?;
+ let stream = delegations.into_stream().await?;
+
+ for await entry in stream {
+ let (link, delegation) = entry?;
+ let ucan = delegation.resolve_ucan(sphere.store()).await?;
+ yield (delegation.name, Did(ucan.audience().to_string()), link);
+ }
+ }
+ }
+
+ /// Get a [BTreeSet] whose members are all the [Link<Jwt>]s to
+ /// authorizations that enable sphere access as of this version of the
+ /// sphere. Note that the full space of authoriztions may be very large; for
+ /// a more space-efficient approach, use
+ /// [SphereWalker::authorization_stream] to incrementally access all
+ /// authorizations in the sphere.
+ ///
+ /// This method is forgiving of missing or corrupted data, and will yield an
+ /// incomplete set of authorizations in the case that some or all names are
+ /// not able to be accessed.
+ pub async fn list_authorizations(&self) -> Result<BTreeSet<Link<Jwt>>> {
+ let sphere_identity = self.has_sphere_context.identity().await?;
+ let authorization_stream = self.authorization_stream();
+
+ tokio::pin!(authorization_stream);
+
+ Ok(authorization_stream
+ .fold(BTreeSet::new(), |mut delegations, another_delegation| {
+ match another_delegation {
+ Ok((_, _, delegation)) => {
+ delegations.insert(delegation);
+ }
+ Err(error) => {
+ warn!(
+ "Could not read a petname from {}: {}",
+ sphere_identity, error
+ )
+ }
+ };
+ delegations
+ })
+ .await)
+ }
+}
+
+impl<'a, C, S> SphereWalker<'a, C, S>
+where
+ C: SpherePetnameRead<S> + HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ /// Same as [SphereWalker::petname_stream], but consumes the [SphereWalker].
+ /// This is useful in cases where it would otherwise be necessary to borrow
+ /// a reference to [SphereWalker] for a static lifetime.
+ pub fn into_petname_stream(self) -> impl Stream<Item = Result<(String, IdentityIpld)>> + 'a {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let petnames = sphere.get_address_book().await?.get_identities().await?;
+ let stream = petnames.into_stream().await?;
+
+ for await entry in stream {
+ let (petname, address) = entry?;
+ yield (petname, address);
+ }
+ }
+ }
+
+ /// Get a stream that yields every petname in the namespace along with its
+ /// corresponding [AddressIpld]. This is useful for iterating over sphere
+ /// petnames incrementally without having to load the entire index into
+ /// memory at once.
+ pub fn petname_stream(&self) -> impl Stream<Item = Result<(String, IdentityIpld)>> + '_ {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let petnames = sphere.get_address_book().await?.get_identities().await?;
+ let stream = petnames.into_stream().await?;
+
+ for await entry in stream {
+ let (petname, address) = entry?;
+ yield (petname, address);
+ }
+ }
+ }
+
+ /// Get a stream that yields the set of petnames that changed at each
+ /// revision of the backing sphere, up to but excluding an optional `since`
+ /// CID parameter. To stream the entire history, pass `None` as the
+ /// parameter.
+ pub fn petname_change_stream<'b>(
+ &'b self,
+ since: Option<&'a Link<MemoIpld>>,
+ ) -> impl Stream<Item = Result<(Link<MemoIpld>, BTreeSet<String>)>> + 'b {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let since = since.cloned();
+ let stream = sphere.into_identities_changelog_stream(since.as_ref());
+
+ for await change in stream {
+ let (cid, changelog) = change?;
+ let mut changed_petnames = BTreeSet::new();
+
+ for operation in changelog.changes {
+ let petname = match operation {
+ MapOperation::Add { key, .. } => key,
+ MapOperation::Remove { key } => key,
+ };
+ changed_petnames.insert(petname);
+ }
+
+ yield (cid, changed_petnames);
+ }
+ }
+ }
+
+ /// Get a stream that yields the set of petnames that changed at each
+ /// revision of the backing sphere, up to but excluding an optional `since`
+ /// CID parameter. To stream the entire history, pass `None` as the
+ /// parameter.
+ pub fn into_petname_change_stream(
+ self,
+ since: Option<&'a Link<MemoIpld>>,
+ ) -> impl Stream<Item = Result<(Link<MemoIpld>, BTreeSet<String>)>> + '_ {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let since = since.cloned();
+ let stream = sphere.into_identities_changelog_stream(since.as_ref());
+
+ for await change in stream {
+ let (cid, changelog) = change?;
+ let mut changed_petnames = BTreeSet::new();
+
+ for operation in changelog.changes {
+ let petname = match operation {
+ MapOperation::Add { key, .. } => key,
+ MapOperation::Remove { key } => key,
+ };
+ changed_petnames.insert(petname);
+ }
+
+ yield (cid, changed_petnames);
+ }
+ }
+ }
+
+ /// Get a [BTreeSet] whose members are all the petnames that have addresses
+ /// as of this version of the sphere. Note that the full space of names may
+ /// be very large; for a more space-efficient approach, use
+ /// [SphereWalker::petname_stream] to incrementally access all petnames in
+ /// the sphere.
+ ///
+ /// This method is forgiving of missing or corrupted data, and will yield an
+ /// incomplete set of names in the case that some or all names are not able
+ /// to be accessed.
+ pub async fn list_petnames(&self) -> Result<BTreeSet<String>> {
+ let sphere_identity = self.has_sphere_context.identity().await?;
+ let petname_stream = self.petname_stream();
+
+ tokio::pin!(petname_stream);
+
+ Ok(petname_stream
+ .fold(BTreeSet::new(), |mut petnames, another_petname| {
+ match another_petname {
+ Ok((petname, _)) => {
+ petnames.insert(petname);
+ }
+ Err(error) => {
+ warn!(
+ "Could not read a petname from {}: {}",
+ sphere_identity, error
+ )
+ }
+ };
+ petnames
+ })
+ .await)
+ }
+
+ /// Get a [BTreeSet] whose members are all the petnames whose values have
+ /// changed at least once since the provided version of the sphere
+ /// (exclusive of the provided version; use `None` to get all petnames
+ /// changed since the beginning of the sphere's history).
+ ///
+ /// This method is forgiving of missing or corrupted history, and will yield
+ /// an incomplete set of changes in the case that some or all changes are
+ /// not able to be accessed.
+ ///
+ /// Note that this operation will scale in memory consumption and duration
+ /// proportionally to the size of the sphere and the length of its history.
+ /// For a more efficient method of accessing changes, consider using
+ /// [SphereWalker::petname_change_stream] instead.
+ pub async fn petname_changes(
+ &self,
+ since: Option<&Link<MemoIpld>>,
+ ) -> Result<BTreeSet<String>> {
+ let sphere_identity = self.has_sphere_context.identity().await?;
+ let change_stream = self.petname_change_stream(since);
+
+ tokio::pin!(change_stream);
+
+ Ok(change_stream
+ .fold(BTreeSet::new(), |mut all, some| {
+ match some {
+ Ok((_, mut changes)) => all.append(&mut changes),
+ Err(error) => warn!(
+ "Could not read some changes from {}: {}",
+ sphere_identity, error
+ ),
+ };
+ all
+ })
+ .await)
+ }
+}
+
+impl<'a, C, S> SphereWalker<'a, C, S>
+where
+ C: SphereContentRead<S> + HasSphereContext<S>,
+ S: Storage + 'static,
+{
+ /// Same as [SphereWalker::content_stream], but consumes the [SphereWalker].
+ /// This is useful in cases where it would otherwise be necessary to borrow
+ /// a reference to [SphereWalker] for a static lifetime.
+ pub fn into_content_stream(
+ self,
+ ) -> impl Stream<Item = Result<(String, SphereFile<impl AsyncRead>)>> + 'a {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let content = sphere.get_content().await?;
+ let stream = content.into_stream().await?;
+
+ for await entry in stream {
+ let (key, memo_link) = entry?;
+ let file = self.has_sphere_context.get_file(sphere.cid(), memo_link).await?;
+
+ yield (key.clone(), file);
+ }
+ }
+ }
+
+ /// Get a stream that yields every slug in the namespace along with its
+ /// corresponding [SphereFile]. This is useful for iterating over sphere
+ /// content incrementally without having to load the entire index into
+ /// memory at once.
+ pub fn content_stream(
+ &self,
+ ) -> impl Stream<Item = Result<(String, SphereFile<impl AsyncRead>)>> + '_ {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let links = sphere.get_content().await?;
+ let stream = links.into_stream().await?;
+
+ for await entry in stream {
+ let (key, memo) = entry?;
+ let file = self.has_sphere_context.get_file(sphere.cid(), memo).await?;
+
+ yield (key.clone(), file);
+ }
+ }
+ }
+
+ /// Get a stream that yields the set of slugs that changed at each revision
+ /// of the backing sphere, up to but excluding an optional CID. To stream
+ /// the entire history, pass `None` as the parameter.
+ pub fn into_content_change_stream(
+ self,
+ since: Option<&'a Link<MemoIpld>>,
+ ) -> impl Stream<Item = Result<(Link<MemoIpld>, BTreeSet<String>)>> + '_ {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let since = since.cloned();
+ let stream = sphere.into_content_changelog_stream(since.as_ref());
+
+ for await change in stream {
+ let (cid, changelog) = change?;
+ let mut changed_slugs = BTreeSet::new();
+
+ for operation in changelog.changes {
+ let slug = match operation {
+ MapOperation::Add { key, .. } => key,
+ MapOperation::Remove { key } => key,
+ };
+ changed_slugs.insert(slug);
+ }
+
+ yield (cid, changed_slugs);
+ }
+ }
+ }
+
+ /// Get a stream that yields the set of slugs that changed at each revision
+ /// of the backing sphere, up to but excluding an optional CID. To stream
+ /// the entire history, pass `None` as the parameter.
+ pub fn content_change_stream<'b>(
+ &'b self,
+ since: Option<&'b Link<MemoIpld>>,
+ ) -> impl Stream<Item = Result<(Link<MemoIpld>, BTreeSet<String>)>> + 'b {
+ try_stream! {
+ let sphere = self.has_sphere_context.to_sphere().await?;
+ let since = since.cloned();
+ let stream = sphere.into_content_changelog_stream(since.as_ref());
+
+ for await change in stream {
+ let (cid, changelog) = change?;
+ let mut changed_slugs = BTreeSet::new();
+
+ for operation in changelog.changes {
+ let slug = match operation {
+ MapOperation::Add { key, .. } => key,
+ MapOperation::Remove { key } => key,
+ };
+ changed_slugs.insert(slug);
+ }
+
+ yield (cid, changed_slugs);
+ }
+ }
+ }
+
+ /// Get a [BTreeSet] whose members are all the slugs that have values as of
+ /// this version of the sphere. Note that the full space of slugs may be
+ /// very large; for a more space-efficient approach, use
+ /// [SphereWalker::content_stream] or [SphereWalker::into_content_stream] to
+ /// incrementally access all slugs in the sphere.
+ ///
+ /// This method is forgiving of missing or corrupted data, and will yield an
+ /// incomplete set of links in the case that some or all links are not able
+ /// to be accessed.
+ pub async fn list_slugs(&self) -> Result<BTreeSet<String>> {
+ let sphere_identity = self.has_sphere_context.identity().await?;
+ let link_stream = self.content_stream();
+
+ tokio::pin!(link_stream);
+
+ Ok(link_stream
+ .fold(BTreeSet::new(), |mut links, another_link| {
+ match another_link {
+ Ok((slug, _)) => {
+ links.insert(slug);
+ }
+ Err(error) => {
+ warn!("Could not read a link from {}: {}", sphere_identity, error)
+ }
+ };
+ links
+ })
+ .await)
+ }
+
+ /// Get a [BTreeSet] whose members are all the slugs whose values have
+ /// changed at least once since the provided version of the sphere
+ /// (exclusive of the provided version; use `None` to get all slugs changed
+ /// since the beginning of the sphere's history).
+ ///
+ /// This method is forgiving of missing or corrupted history, and will yield
+ /// an incomplete set of changes in the case that some or all changes are
+ /// not able to be accessed.
+ ///
+ /// Note that this operation will scale in memory consumption and duration
+ /// proportionally to the size of the sphere and the length of its history.
+ /// For a more efficient method of accessing changes, consider using
+ /// [SphereWalker::content_change_stream] instead.
+ pub async fn content_changes(
+ &self,
+ since: Option<&Link<MemoIpld>>,
+ ) -> Result<BTreeSet<String>> {
+ let sphere_identity = self.has_sphere_context.identity().await?;
+ let change_stream = self.content_change_stream(since);
+
+ tokio::pin!(change_stream);
+
+ Ok(change_stream
+ .fold(BTreeSet::new(), |mut all, some| {
+ match some {
+ Ok((_, mut changes)) => all.append(&mut changes),
+ Err(error) => warn!(
+ "Could not read some changes from {}: {}",
+ sphere_identity, error
+ ),
+ };
+ all
+ })
+ .await)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use anyhow::Result;
+ use std::collections::BTreeSet;
+
+ use tokio::io::AsyncReadExt;
+ use tokio_stream::StreamExt;
+
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ use crate::{
+ context::{
+ HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SphereCursor,
+ SphereWalker,
+ },
+ data::{ContentType, Did},
+ helpers::{simulated_sphere_context, SimulationAccess},
+ };
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_be_initialized_with_a_context_or_a_cursor() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context.clone());
+
+ let changes = vec![
+ vec!["dogs", "birds"],
+ vec!["cats", "dogs"],
+ vec!["birds"],
+ vec!["cows", "beetles"],
+ ];
+
+ for change in changes {
+ for slug in change {
+ cursor
+ .write(slug, &ContentType::Subtext, b"are cool".as_ref(), None)
+ .await
+ .unwrap();
+ }
+
+ cursor.save(None).await.unwrap();
+ }
+
+ let walker_cursor = SphereWalker::from(&cursor);
+ let walker_context = SphereWalker::from(&sphere_context);
+
+ let slugs_cursor = walker_cursor.list_slugs().await.unwrap();
+ let slugs_context = walker_context.list_slugs().await.unwrap();
+
+ assert_eq!(slugs_cursor.len(), 5);
+ assert_eq!(slugs_cursor, slugs_context);
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_list_all_slugs_currently_in_a_sphere() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ let changes = vec![
+ vec!["dogs", "birds"],
+ vec!["cats", "dogs"],
+ vec!["birds"],
+ vec!["cows", "beetles"],
+ ];
+
+ for change in changes {
+ for slug in change {
+ cursor
+ .write(slug, &ContentType::Subtext, b"are cool".as_ref(), None)
+ .await
+ .unwrap();
+ }
+
+ cursor.save(None).await.unwrap();
+ }
+
+ let walker_cursor = cursor.clone();
+ let walker = SphereWalker::from(&walker_cursor);
+ let slugs = walker.list_slugs().await.unwrap();
+
+ assert_eq!(slugs.len(), 5);
+
+ cursor.remove("dogs").await.unwrap();
+ cursor.save(None).await.unwrap();
+
+ let slugs = walker.list_slugs().await.unwrap();
+
+ assert_eq!(slugs.len(), 4);
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_list_all_authorizations_currently_in_a_sphere() -> Result<()> {
+ let (sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let mut cursor = SphereCursor::latest(sphere_context);
+ let authorizations_to_add = 10;
+
+ for i in 0..authorizations_to_add {
+ cursor
+ .authorize(&format!("foo{}", i), &Did(format!("did:key:foo{}", i)))
+ .await?;
+ }
+
+ cursor.save(None).await?;
+
+ let authorizations = SphereWalker::from(&cursor).list_authorizations().await?;
+
+ assert_eq!(authorizations.len(), authorizations_to_add + 1);
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_stream_the_whole_index() {
+ let (sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+ let mut cursor = SphereCursor::latest(sphere_context);
+
+ let expected = BTreeSet::<(String, String)>::from([
+ ("cats".into(), "Cats are awesome".into()),
+ ("dogs".into(), "Dogs are pretty cool".into()),
+ ("birds".into(), "Birds rights".into()),
+ ("mice".into(), "Mice like cookies".into()),
+ ]);
+
+ for (slug, content) in &expected {
+ cursor
+ .write(slug.as_str(), &ContentType::Subtext, content.as_ref(), None)
+ .await
+ .unwrap();
+
+ cursor.save(None).await.unwrap();
+ }
+
+ let mut actual = BTreeSet::new();
+ let walker = SphereWalker::from(&cursor);
+ let stream = walker.content_stream();
+
+ tokio::pin!(stream);
+
+ while let Some(Ok((slug, mut file))) = stream.next().await {
+ let mut contents = String::new();
+ file.contents.read_to_string(&mut contents).await.unwrap();
+ actual.insert((slug, contents));
+ }
+
+ assert_eq!(expected, actual);
+ }
+}
diff --git a/rust/noosphere-core/src/data/address.rs b/rust/noosphere-core/src/data/address.rs
index ab1c7fadd..87edf8374 100644
--- a/rust/noosphere-core/src/data/address.rs
+++ b/rust/noosphere-core/src/data/address.rs
@@ -15,12 +15,15 @@ use super::{Did, IdentitiesIpld, Jwt, Link, MemoIpld};
#[cfg(docs)]
use crate::data::SphereIpld;
+/// The name of the fact (as defined for a [Ucan]) that contains the link for a
+/// [LinkRecord]
pub const LINK_RECORD_FACT_NAME: &str = "link";
/// A subdomain of a [SphereIpld] that pertains to the management and recording of
/// the petnames associated with the sphere.
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Hash)]
pub struct AddressBookIpld {
+ /// A pointer to the [IdentitiesIpld] associated with this address book
pub identities: Link<IdentitiesIpld>,
}
@@ -42,7 +45,9 @@ impl AddressBookIpld {
/// value if one has ever been resolved.
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Hash)]
pub struct IdentityIpld {
+ /// The [Did] of a peer
pub did: Did,
+ /// An optional pointer to a known [LinkRecord] for the peer
pub link_record: Option<Link<LinkRecord>>,
}
diff --git a/rust/noosphere-core/src/data/authority.rs b/rust/noosphere-core/src/data/authority.rs
index 78274acb4..1a911f292 100644
--- a/rust/noosphere-core/src/data/authority.rs
+++ b/rust/noosphere-core/src/data/authority.rs
@@ -16,7 +16,12 @@ use crate::data::SphereIpld;
/// access a sphere, as well as the revocations of that authority.
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
pub struct AuthorityIpld {
+ /// A pointer to the [DelegationsIpld] for this [AuthorityIpld], embodying
+ /// all authorizations for keys that operate on this sphere
pub delegations: Link<DelegationsIpld>,
+ /// A pointer to the [RevocationsIpld] for this [AuthorityIpld], embodying
+ /// revocations for any otherwise valid authorizations for keys issued by this
+ /// sphere in the past.
pub revocations: Link<RevocationsIpld>,
}
@@ -43,17 +48,24 @@ impl AuthorityIpld {
}
}
+#[cfg(doc)]
+use crate::data::Jwt;
+
/// This delegation represents the sharing of access to resources within a
/// sphere. The name of the delegation is for display purposes only, and helps
/// the user identify the client device or application that the delegation is
/// intended for.
#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)]
pub struct DelegationIpld {
+ /// The human-readable name of the delegation
pub name: String,
+ /// A pointer to the [Jwt] created for this [DelegationIpld]
pub jwt: Cid,
}
impl DelegationIpld {
+ /// Stores a [Ucan] that delegates authority to a key, and initializes a
+ /// [DelegationIpld] for it that is appropriate for storing in a sphere
pub async fn register<S: BlockStore>(name: &str, jwt: &str, store: &S) -> Result<Self> {
let mut store = UcanStore(store.clone());
let cid = store.write_token(jwt).await?;
@@ -64,6 +76,8 @@ impl DelegationIpld {
})
}
+ /// Resolve a [Ucan] from storage via the pointer to a [Jwt] in this
+ /// [DelegationIpld]
pub async fn resolve_ucan<S: BlockStore>(&self, store: &S) -> Result<Ucan> {
let store = UcanStore(store.clone());
let jwt = store.require_token(&self.jwt).await?;
@@ -87,6 +101,8 @@ pub struct RevocationIpld {
}
impl RevocationIpld {
+ /// Revoke a delegation by the [Cid] of its associated [Jwt], using a key credential
+ /// of an authorizing ancestor of the original delegation
pub async fn revoke<K: KeyMaterial>(cid: &Cid, issuer: &K) -> Result<Self> {
Ok(RevocationIpld {
iss: issuer.get_did().await?,
@@ -95,6 +111,8 @@ impl RevocationIpld {
})
}
+ /// Verify that the [RevocationIpld] is valid compared to the public key of
+ /// the issuer
pub async fn verify<K: KeyMaterial + ?Sized>(&self, claimed_issuer: &K) -> Result<()> {
let cid = Cid::try_from(self.revoke.as_str())?;
let challenge_payload = Self::make_challenge_payload(&cid);
diff --git a/rust/noosphere-core/src/data/body_chunk.rs b/rust/noosphere-core/src/data/body_chunk.rs
index 66d935164..74eb98352 100644
--- a/rust/noosphere-core/src/data/body_chunk.rs
+++ b/rust/noosphere-core/src/data/body_chunk.rs
@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
use noosphere_storage::BlockStore;
+/// The maximum size of a body chunk as produced by [BodyChunkIpld]
pub const BODY_CHUNK_MAX_SIZE: u32 = 1024 * 1024; // ~1mb/chunk worst case, ~.5mb/chunk average case
/// A body chunk is a simplified flexible byte layout used for linking
@@ -21,6 +22,8 @@ pub struct BodyChunkIpld {
}
impl BodyChunkIpld {
+ /// Chunk and encode a slice of bytes as linked [BodyChunkIpld], storing the chunks in storage
+ /// and returning the [Cid] of the head of the list.
// TODO(#498): Re-write to address potentially unbounded memory overhead
pub async fn store_bytes<S: BlockStore>(bytes: &[u8], store: &mut S) -> Result<Cid> {
let chunks = FastCDC::new(
@@ -56,6 +59,7 @@ impl BodyChunkIpld {
next_chunk_cid.ok_or_else(|| anyhow!("No CID; did you try to store zero bytes?"))
}
+ /// Fold all bytes in the [BodyChunkIpld] chain into a single buffer and return it
// TODO(#498): Re-write to address potentially unbounded memory overhead
pub async fn load_all_bytes<S: BlockStore>(&self, store: &S) -> Result<Vec<u8>> {
let mut all_bytes = self.bytes.clone();
diff --git a/rust/noosphere-core/src/data/bundle.rs b/rust/noosphere-core/src/data/bundle.rs
index fc36fe272..7d35963d4 100644
--- a/rust/noosphere-core/src/data/bundle.rs
+++ b/rust/noosphere-core/src/data/bundle.rs
@@ -1,3 +1,6 @@
+// We are removing this module, so not gonna bother documenting...
+#![allow(missing_docs)]
+
use std::{collections::BTreeMap, str::FromStr};
use anyhow::{anyhow, Result};
diff --git a/rust/noosphere-core/src/data/changelog.rs b/rust/noosphere-core/src/data/changelog.rs
index ed8d279ca..491e5f4e3 100644
--- a/rust/noosphere-core/src/data/changelog.rs
+++ b/rust/noosphere-core/src/data/changelog.rs
@@ -2,22 +2,32 @@ use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::default::Default;
+#[cfg(doc)]
+use crate::data::{Did, VersionedMapIpld};
+
+/// A [ChangelogIpld] records a series of changes that represent the delta of a
+/// given [VersionedMapIpld] from its immediate ancestor
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
pub struct ChangelogIpld<Op> {
+ /// The [Did] of the author of the change
pub did: Option<String>,
+ /// The changes that were made to the associated [VersionedMapIpld]
pub changes: Vec<Op>,
}
impl<Op> ChangelogIpld<Op> {
+ /// Returns true if the [ChangelogIpld] represents zero changes
pub fn is_empty(&self) -> bool {
self.changes.len() == 0
}
+ /// Adds a single change to the [ChangelogIpld]
pub fn push(&mut self, op: Op) -> Result<()> {
self.changes.push(op);
Ok(())
}
+ /// Initializes a [ChangelogIpld] for the author with the given [Did]
pub fn mark(&self, did: &str) -> Self {
ChangelogIpld {
did: Some(did.to_string()),
diff --git a/rust/noosphere-core/src/data/headers/content_type.rs b/rust/noosphere-core/src/data/headers/content_type.rs
index 8f98bef27..d57c97acc 100644
--- a/rust/noosphere-core/src/data/headers/content_type.rs
+++ b/rust/noosphere-core/src/data/headers/content_type.rs
@@ -1,13 +1,21 @@
use std::{convert::Infallible, fmt::Display, ops::Deref, str::FromStr};
+/// Various well-known mimes in Noosphere
#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Debug)]
pub enum ContentType {
+ /// Plain text
Text,
+ /// Subtext
Subtext,
+ /// A sphere
Sphere,
+ /// Raw bytes
Bytes,
+ /// CBOR bytes
Cbor,
+ /// JSON
Json,
+ /// All others
Unknown(String),
}
diff --git a/rust/noosphere-core/src/data/headers/header.rs b/rust/noosphere-core/src/data/headers/header.rs
index 9a9ce3547..aa00f1121 100644
--- a/rust/noosphere-core/src/data/headers/header.rs
+++ b/rust/noosphere-core/src/data/headers/header.rs
@@ -1,14 +1,25 @@
use std::{convert::Infallible, fmt::Display, ops::Deref, str::FromStr};
+/// Well-known headers in the Noosphere
pub enum Header {
+ /// Content-type, for mimes
ContentType,
+ /// A proof, typically a UCAN JWT
Proof,
+ /// The author's DID
Author,
+ /// A title for the associated content body
Title,
+ /// A signature by the author's key
Signature,
+ /// The Noosphere protocol version
Version,
+ /// A file extension to use when rendering the content to
+ /// the file system
FileExtension,
+ /// The logical order relative to any ancestors
LamportOrder,
+ /// All others
Unknown(String),
}
diff --git a/rust/noosphere-core/src/data/headers/version.rs b/rust/noosphere-core/src/data/headers/version.rs
index 2113b09b2..2c22af954 100644
--- a/rust/noosphere-core/src/data/headers/version.rs
+++ b/rust/noosphere-core/src/data/headers/version.rs
@@ -1,8 +1,11 @@
use anyhow::anyhow;
use std::{convert::Infallible, fmt::Display, str::FromStr};
+/// The Noosphere protocol version
pub enum Version {
+ #[allow(missing_docs)]
V0,
+ /// All others
Unknown(String),
}
diff --git a/rust/noosphere-core/src/data/link.rs b/rust/noosphere-core/src/data/link.rs
index bb3f38345..94c2c13c3 100644
--- a/rust/noosphere-core/src/data/link.rs
+++ b/rust/noosphere-core/src/data/link.rs
@@ -4,6 +4,7 @@ use libipld_core::{
codec::{Codec, Decode, Encode},
raw::RawCodec,
};
+use noosphere_common::ConditionalSend;
use std::fmt::Debug;
use std::{
fmt::{Display, Formatter},
@@ -18,18 +19,6 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
use noosphere_collections::hamt::Hash as HamtHash;
-#[cfg(not(target_arch = "wasm32"))]
-pub trait LinkSend: Send {}
-
-#[cfg(not(target_arch = "wasm32"))]
-impl<T> LinkSend for T where T: Send {}
-
-#[cfg(target_arch = "wasm32")]
-pub trait LinkSend {}
-
-#[cfg(target_arch = "wasm32")]
-impl<T> LinkSend for T {}
-
/// A [Link] is a [Cid] with a type attached. The type represents the data that
/// the [Cid] refers to. This is a helpful construct to use to ensure that data
/// structures whose fields or elements may be [Cid]s can still retain strong
@@ -45,6 +34,7 @@ pub struct Link<T>
where
T: Clone,
{
+ /// The wrapped [Cid] of this [Link]
pub cid: Cid,
linked_type: PhantomData<T>,
}
@@ -105,6 +95,7 @@ impl<T> Link<T>
where
T: Clone,
{
+ /// Wrap a given [Cid] in a typed [Link]
pub fn new(cid: Cid) -> Self {
Link {
cid,
@@ -200,7 +191,7 @@ where
impl<T> Link<T>
where
- T: Serialize + DeserializeOwned + Clone + LinkSend,
+ T: Serialize + DeserializeOwned + Clone + ConditionalSend,
{
/// Given a [BlockStore], attempt to load a value for the [Cid] of this
/// [Link]. The loaded block will be interpretted as the type that is
diff --git a/rust/noosphere-core/src/data/memo.rs b/rust/noosphere-core/src/data/memo.rs
index b1cf82e86..975021799 100644
--- a/rust/noosphere-core/src/data/memo.rs
+++ b/rust/noosphere-core/src/data/memo.rs
@@ -6,15 +6,17 @@ use std::{
use anyhow::{anyhow, Result};
use cid::Cid;
use libipld_cbor::DagCborCodec;
+use noosphere_common::ConditionalSend;
use serde::{Deserialize, Serialize};
use ucan::{crypto::KeyMaterial, Ucan};
use crate::data::Header;
-use noosphere_storage::{base64_encode, BlockStore, BlockStoreSend};
+use noosphere_storage::{base64_encode, BlockStore};
use super::{ContentType, Link};
+/// A designated int for lamport order values in [MemoIpld] headers
pub type LamportOrder = u32;
/// A basic Memo. A Memo is a history-retaining structure that pairs
@@ -99,7 +101,7 @@ impl MemoIpld {
/// Initializes a memo for the provided body, persisting the body to storage
/// and returning the memo. Note that only the body is persisted, not the
/// memo that wraps it.
- pub async fn for_body<S: BlockStore, Body: Serialize + BlockStoreSend>(
+ pub async fn for_body<S: BlockStore, Body: Serialize + ConditionalSend>(
store: &mut S,
body: Body,
) -> Result<MemoIpld> {
diff --git a/rust/noosphere-core/src/data/mod.rs b/rust/noosphere-core/src/data/mod.rs
index 552de5bd6..3631f7f4b 100644
--- a/rust/noosphere-core/src/data/mod.rs
+++ b/rust/noosphere-core/src/data/mod.rs
@@ -1,3 +1,7 @@
+//! Core data types in use by the Noosphere protocol. Data types in here
+//! represent the canonical structure of Noosphere data as expressed by
+//! block-encoded IPLD.
+
mod address;
mod authority;
mod body_chunk;
diff --git a/rust/noosphere-core/src/data/sphere.rs b/rust/noosphere-core/src/data/sphere.rs
index 25ec203c9..df89c0f6d 100644
--- a/rust/noosphere-core/src/data/sphere.rs
+++ b/rust/noosphere-core/src/data/sphere.rs
@@ -29,6 +29,7 @@ pub struct SphereIpld {
}
impl SphereIpld {
+ /// Initialize a new, empty [SphereIpld] with a given [Did] sphere identity.
pub async fn new<S>(identity: &Did, store: &mut S) -> Result<SphereIpld>
where
S: BlockStore,
@@ -58,31 +59,23 @@ impl SphereIpld {
#[cfg(test)]
mod tests {
use anyhow::Result;
- use ed25519_zebra::{SigningKey as Ed25519PrivateKey, VerificationKey as Ed25519PublicKey};
use libipld_cbor::DagCborCodec;
use ucan::{builder::UcanBuilder, crypto::KeyMaterial, store::UcanJwtStore};
- use ucan_key_support::ed25519::Ed25519KeyMaterial;
#[cfg(target_arch = "wasm32")]
use wasm_bindgen_test::wasm_bindgen_test;
use crate::{
- authority::{generate_capability, SphereAbility},
+ authority::{generate_capability, generate_ed25519_key, SphereAbility},
data::{ContentType, Did, Header, MemoIpld, SphereIpld},
view::Sphere,
};
use noosphere_storage::{BlockStore, MemoryStorage, SphereDb};
- fn generate_credential() -> Ed25519KeyMaterial {
- let private_key = Ed25519PrivateKey::new(rand::thread_rng());
- let public_key = Ed25519PublicKey::from(&private_key);
- Ed25519KeyMaterial(public_key, Some(private_key))
- }
-
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
#[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
async fn it_can_be_signed_by_identity_key_and_verified() -> Result<()> {
- let identity_credential = generate_credential();
+ let identity_credential = generate_ed25519_key();
let identity = Did(identity_credential.get_did().await?);
let mut store = SphereDb::new(&MemoryStorage::default()).await?;
@@ -128,8 +121,8 @@ mod tests {
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
#[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
async fn it_can_be_signed_by_an_authorized_key_and_verified() -> Result<()> {
- let identity_credential = generate_credential();
- let authorized_credential = generate_credential();
+ let identity_credential = generate_ed25519_key();
+ let authorized_credential = generate_ed25519_key();
let identity = Did(identity_credential.get_did().await?);
let authorized = Did(authorized_credential.get_did().await?);
diff --git a/rust/noosphere-core/src/data/versioned_map.rs b/rust/noosphere-core/src/data/versioned_map.rs
index 9e1a5ced5..fccf5b873 100644
--- a/rust/noosphere-core/src/data/versioned_map.rs
+++ b/rust/noosphere-core/src/data/versioned_map.rs
@@ -7,66 +7,79 @@ use std::{fmt::Display, hash::Hash, marker::PhantomData};
use noosphere_collections::hamt::{Hamt, Hash as HamtHash, Sha256};
use noosphere_storage::BlockStore;
+use noosphere_common::ConditionalSync;
+
use super::{ChangelogIpld, DelegationIpld, IdentityIpld, Jwt, Link, MemoIpld, RevocationIpld};
-pub type IdentitiesIpld = VersionedMapIpld<String, IdentityIpld>;
+/// A [VersionedMapIpld] that represents the content space of a sphere
pub type ContentIpld = VersionedMapIpld<String, Link<MemoIpld>>;
+/// A [VersionedMapIpld] that represents the petname space of a sphere
+pub type IdentitiesIpld = VersionedMapIpld<String, IdentityIpld>;
+/// A [VersionedMapIpld] that represents the key authorizations in a sphere
pub type DelegationsIpld = VersionedMapIpld<Link<Jwt>, DelegationIpld>;
+/// A [VersionedMapIpld] that represents the authority revocations in a sphere
pub type RevocationsIpld = VersionedMapIpld<Link<Jwt>, RevocationIpld>;
-#[cfg(not(target_arch = "wasm32"))]
-pub trait VersionedMapSendSync: Send + Sync {}
-
-#[cfg(not(target_arch = "wasm32"))]
-impl<T> VersionedMapSendSync for T where T: Send + Sync {}
-
-#[cfg(target_arch = "wasm32")]
-pub trait VersionedMapSendSync {}
-
-#[cfg(target_arch = "wasm32")]
-impl<T> VersionedMapSendSync for T {}
-
+/// A helper trait to simplify expressing the bounds of a valid [VersionedMapIpld] key
pub trait VersionedMapKey:
- Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + VersionedMapSendSync + Display
+ Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + ConditionalSync + Display
{
}
impl<T> VersionedMapKey for T where
- T: Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + VersionedMapSendSync + Display
+ T: Serialize + DeserializeOwned + HamtHash + Clone + Eq + Ord + ConditionalSync + Display
{
}
+/// A helper trait to simplify expressing the bounds of a valid [VersionedMapIpld] value
pub trait VersionedMapValue:
- Serialize + DeserializeOwned + Clone + Eq + Hash + VersionedMapSendSync
+ Serialize + DeserializeOwned + Clone + Eq + Hash + ConditionalSync
{
}
impl<T> VersionedMapValue for T where
- T: Serialize + DeserializeOwned + Clone + Eq + Hash + VersionedMapSendSync
+ T: Serialize + DeserializeOwned + Clone + Eq + Hash + ConditionalSync
{
}
+/// A [MapOperation] represents a single change to a [VersionedMapIpld] as it may
+/// be recorded in a [ChangelogIpld].
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
pub enum MapOperation<Key, Value> {
- Add { key: Key, value: Value },
- Remove { key: Key },
+ /// A [MapOperation] that represents an update or insert to a [VersionedMapIpld]
+ Add {
+ /// The key that was updated or inserted
+ key: Key,
+ /// The new value associated with the key
+ value: Value,
+ },
+ /// A [MapOperation] that represents a removal of a key from a [VersionedMapIpld]
+ Remove {
+ /// The key that was removed
+ key: Key,
+ },
}
+/// A [VersionedMapIpld] pairs a [Hamt] and a [ChangelogIpld] to enable a data structure
+/// that contains its difference from a historical ancestor without requiring that a diff
+/// be performed.
#[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize)]
pub struct VersionedMapIpld<Key, Value>
where
Key: VersionedMapKey,
Value: VersionedMapValue,
{
- /// A pointer to a HAMT
+ /// A pointer to a [Hamt] root
pub hamt: Cid,
// TODO(#262): The size of this vec is implicitly limited by the IPLD block
// size limit. This is probably fine most of the time; the vec only holds
// the delta changes, and N<10 probably holds in the majority of cases. But,
// it will be necessary to gracefully survive the outlier cases where
// N>~1000.
+ /// A pointer to a [ChangelogIpld]
pub changelog: Cid,
+ #[allow(missing_docs)]
#[serde(skip)]
pub signature: PhantomData<(Key, Value)>,
}
@@ -76,10 +89,12 @@ where
Key: VersionedMapKey,
Value: VersionedMapValue,
{
+ /// Load the [Hamt] root associated with this [VersionedMapIpld]
pub async fn load_hamt<S: BlockStore>(&self, store: &S) -> Result<Hamt<S, Value, Key, Sha256>> {
Hamt::load(&self.hamt, store.clone()).await
}
+ /// Load the [ChangelogIpld] associated with this [VersionedMapIpld]
pub async fn load_changelog<S: BlockStore>(
&self,
store: &S,
@@ -87,6 +102,8 @@ where
store.load::<DagCborCodec, _>(&self.changelog).await
}
+ /// Initialize an empty [VersionedMapIpld], creating an empty [Hamt] root
+ /// and [ChangelogIpld] as well
// NOTE: We currently don't have a mechanism to prepuplate the store with
// "empty" DAGs like a HAMT. So, we do it lazily by requiring async
// initialization of this struct even when it is empty.
diff --git a/rust/noosphere-core/src/error.rs b/rust/noosphere-core/src/error.rs
index efc0f86ab..e75799bd1 100644
--- a/rust/noosphere-core/src/error.rs
+++ b/rust/noosphere-core/src/error.rs
@@ -1,20 +1,28 @@
+//! Noosphere errors
+
use crate::authority::Authorization;
use thiserror::Error;
+/// High-level error types relevant to the Noosphere protocol
#[derive(Error, Debug)]
pub enum NoosphereError {
+ /// Any error not covered by the other errors
#[error("{0}")]
Other(anyhow::Error),
+ #[allow(missing_docs)]
#[error("Network access required but network is currently offline")]
NetworkOffline,
+ #[allow(missing_docs)]
#[error("No credentials configured")]
NoCredentials,
+ #[allow(missing_docs)]
#[error("Missing configuration: {0}")]
MissingConfiguration(&'static str),
+ #[allow(missing_docs)]
#[error("The provided authorization {0} is invalid: {1}")]
InvalidAuthorization(Authorization, String),
}
diff --git a/rust/noosphere-core/src/helpers/context.rs b/rust/noosphere-core/src/helpers/context.rs
new file mode 100644
index 000000000..05e06dbf2
--- /dev/null
+++ b/rust/noosphere-core/src/helpers/context.rs
@@ -0,0 +1,229 @@
+//! These helpers are intended for use in documentation examples and tests only.
+//! They are useful for quickly scaffolding common scenarios that would
+//! otherwise be verbosely rubber-stamped in a bunch of places.
+use std::sync::Arc;
+
+use crate::{
+ authority::{generate_capability, generate_ed25519_key, Author, SphereAbility},
+ data::{ContentType, Did, LinkRecord, Mnemonic, LINK_RECORD_FACT_NAME},
+ view::Sphere,
+};
+use anyhow::Result;
+use noosphere_storage::{BlockStore, MemoryStorage, SphereDb, TrackingStorage, UcanStore};
+use tokio::{io::AsyncReadExt, sync::Mutex};
+use ucan::{builder::UcanBuilder, crypto::KeyMaterial};
+
+use crate::{
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite,
+ SphereContext, SphereContextKey, SpherePetnameWrite,
+ },
+ stream::{walk_versioned_map_elements, walk_versioned_map_elements_and},
+};
+
+/// Access levels available when simulating a [SphereContext]
+pub enum SimulationAccess {
+ /// Access to the related [SphereContext] is read-only
+ Readonly,
+ /// Access to the related [SphereContext] is read+write
+ ReadWrite,
+}
+
+/// Create a temporary, non-persisted [SphereContext] that tracks usage
+/// internally. This is intended for use in docs and tests, and should otherwise
+/// be ignored. When creating the simulated [SphereContext], you can pass a
+/// [SimulationAccess] to control the kind of access the emphemeral credentials
+/// have to the [SphereContext].
+pub async fn simulated_sphere_context(
+ profile: SimulationAccess,
+ db: Option<SphereDb<TrackingStorage<MemoryStorage>>>,
+) -> Result<(
+ Arc<Mutex<SphereContext<TrackingStorage<MemoryStorage>>>>,
+ Mnemonic,
+)> {
+ let mut db = match db {
+ Some(db) => db,
+ None => {
+ let storage_provider = TrackingStorage::wrap(MemoryStorage::default());
+ SphereDb::new(&storage_provider).await?
+ }
+ };
+
+ let owner_key: SphereContextKey = Arc::new(Box::new(generate_ed25519_key()));
+ let owner_did = owner_key.get_did().await?;
+
+ let (sphere, proof, mnemonic) = Sphere::generate(&owner_did, &mut db).await?;
+
+ let sphere_identity = sphere.get_identity().await?;
+ let author = Author {
+ key: owner_key,
+ authorization: match profile {
+ SimulationAccess::Readonly => None,
+ SimulationAccess::ReadWrite => Some(proof),
+ },
+ };
+
+ db.set_version(&sphere_identity, sphere.cid()).await?;
+
+ Ok((
+ Arc::new(Mutex::new(
+ SphereContext::new(sphere_identity, author, db, None).await?,
+ )),
+ mnemonic,
+ ))
+}
+
+#[cfg(docs)]
+use crate::data::MemoIpld;
+
+/// Attempt to walk an entire sphere, touching every block up to and including
+/// any [MemoIpld] nodes, but excluding those memo's body content. This helper
+/// is useful for asserting that the blocks expected to be sent during
+/// replication have in fact been sent.
+pub async fn touch_all_sphere_blocks<S>(sphere: &Sphere<S>) -> Result<()>
+where
+ S: BlockStore + 'static,
+{
+ trace!("Touching content blocks...");
+ let content = sphere.get_content().await?;
+ let _ = content.load_changelog().await?;
+
+ walk_versioned_map_elements(content).await?;
+
+ trace!("Touching identity blocks...");
+ let identities = sphere.get_address_book().await?.get_identities().await?;
+ let _ = identities.load_changelog().await?;
+
+ walk_versioned_map_elements_and(
+ identities,
+ sphere.store().clone(),
+ |_, identity, store| async move {
+ let ucan_store = UcanStore(store);
+ if let Some(record) = identity.link_record(&ucan_store).await {
+ record.collect_proofs(&ucan_store).await?;
+ }
+ Ok(())
+ },
+ )
+ .await?;
+
+ trace!("Touching authority blocks...");
+ let authority = sphere.get_authority().await?;
+
+ trace!("Touching delegation blocks...");
+ let delegations = authority.get_delegations().await?;
+ walk_versioned_map_elements(delegations).await?;
+
+ trace!("Touching revocation blocks...");
+ let revocations = authority.get_revocations().await?;
+ walk_versioned_map_elements(revocations).await?;
+
+ Ok(())
+}
+
+/// A type of [HasMutableSphereContext] that uses [TrackingStorage] internally
+pub type TrackedHasMutableSphereContext = Arc<Mutex<SphereContext<TrackingStorage<MemoryStorage>>>>;
+
+/// Create a series of spheres where each sphere has the next as resolved
+/// entry in its address book; return a [HasMutableSphereContext] for the
+/// first sphere in the sequence.
+pub async fn make_sphere_context_with_peer_chain(
+ peer_chain: &[String],
+) -> Result<(TrackedHasMutableSphereContext, Vec<Did>)> {
+ let (origin_sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None)
+ .await
+ .unwrap();
+
+ let mut db = origin_sphere_context
+ .sphere_context()
+ .await
+ .unwrap()
+ .db()
+ .clone();
+
+ let mut contexts = vec![origin_sphere_context.clone()];
+
+ for name in peer_chain.iter() {
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, Some(db.clone()))
+ .await
+ .unwrap();
+
+ sphere_context
+ .write("my-name", &ContentType::Subtext, name.as_bytes(), None)
+ .await
+ .unwrap();
+ sphere_context.save(None).await.unwrap();
+
+ contexts.push(sphere_context);
+ }
+
+ let mut next_sphere_context: Option<TrackedHasMutableSphereContext> = None;
+ let mut dids = Vec::new();
+
+ for mut sphere_context in contexts.into_iter().rev() {
+ dids.push(sphere_context.identity().await?);
+ if let Some(next_sphere_context) = next_sphere_context {
+ let version = next_sphere_context.version().await.unwrap();
+
+ let next_author = next_sphere_context
+ .sphere_context()
+ .await
+ .unwrap()
+ .author()
+ .clone();
+ let next_identity = next_sphere_context.identity().await.unwrap();
+
+ let link_record = LinkRecord::from(
+ UcanBuilder::default()
+ .issued_by(&next_author.key)
+ .for_audience(&next_identity)
+ .witnessed_by(
+ &next_author
+ .authorization
+ .as_ref()
+ .unwrap()
+ .as_ucan(&db)
+ .await
+ .unwrap(),
+ None,
+ )
+ .claiming_capability(&generate_capability(
+ &next_identity,
+ SphereAbility::Publish,
+ ))
+ .with_lifetime(120)
+ .with_fact(LINK_RECORD_FACT_NAME, version.to_string())
+ .build()
+ .unwrap()
+ .sign()
+ .await
+ .unwrap(),
+ );
+
+ let mut name = String::new();
+ let mut file = next_sphere_context.read("my-name").await.unwrap().unwrap();
+ file.contents.read_to_string(&mut name).await.unwrap();
+
+ debug!("Adopting {name}");
+ sphere_context
+ .set_petname(&name, Some(next_identity))
+ .await?;
+ sphere_context.save(None).await?;
+
+ sphere_context
+ .set_petname_record(&name, &link_record)
+ .await
+ .unwrap();
+ let identity = sphere_context.identity().await?;
+
+ db.set_version(&identity, &sphere_context.save(None).await.unwrap())
+ .await
+ .unwrap();
+ }
+
+ next_sphere_context = Some(sphere_context);
+ }
+
+ Ok((origin_sphere_context, dids))
+}
diff --git a/rust/noosphere-core/src/helpers.rs b/rust/noosphere-core/src/helpers/link.rs
similarity index 100%
rename from rust/noosphere-core/src/helpers.rs
rename to rust/noosphere-core/src/helpers/link.rs
diff --git a/rust/noosphere-core/src/helpers/mod.rs b/rust/noosphere-core/src/helpers/mod.rs
new file mode 100644
index 000000000..e6b0cbc2c
--- /dev/null
+++ b/rust/noosphere-core/src/helpers/mod.rs
@@ -0,0 +1,8 @@
+//! Generic helper utilities intended to be used exclusively in tests and hidden
+//! setup for examples
+
+mod context;
+mod link;
+
+pub use context::*;
+pub use link::*;
diff --git a/rust/noosphere-core/src/lib.rs b/rust/noosphere-core/src/lib.rs
index 414745718..42a04f932 100644
--- a/rust/noosphere-core/src/lib.rs
+++ b/rust/noosphere-core/src/lib.rs
@@ -1,8 +1,21 @@
+#![warn(missing_docs)]
+
+//! This crate embodies the core implementation of the Noosphere protocol.
+//!
+//! It includes facilities to:
+//! - Get low-level access to Noosphere data structures ([view] and [data])
+//! - Interact with a Noosphere Gateway ([api::Client])
+//! - Read, update, and sync spheres via a high-level API ([context])
+//! - And more!
+
#[macro_use]
extern crate tracing as extern_tracing;
+pub mod api;
pub mod authority;
+pub mod context;
pub mod data;
+pub mod stream;
pub mod view;
pub mod error;
diff --git a/rust/noosphere-core/src/stream/block.rs b/rust/noosphere-core/src/stream/block.rs
new file mode 100644
index 000000000..2654af752
--- /dev/null
+++ b/rust/noosphere-core/src/stream/block.rs
@@ -0,0 +1,42 @@
+use anyhow::Result;
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use libipld_core::raw::RawCodec;
+use noosphere_storage::BlockStore;
+use tokio_stream::{Stream, StreamExt};
+
+/// Helper to put blocks from a [Stream] into any implementor of [BlockStore]
+///
+/// Implementation note: this is a stand-alone helper because defining this
+/// async function on a trait (necessitating use of `#[async_trait]`) creates an
+/// unergonomic `Send` bound on returned [Future].
+pub async fn put_block_stream<S, Str>(mut store: S, stream: Str) -> Result<()>
+where
+ S: BlockStore,
+ Str: Stream<Item = Result<(Cid, Vec<u8>)>>,
+{
+ tokio::pin!(stream);
+
+ let mut stream_count = 0usize;
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ stream_count += 1;
+ trace!(?cid, "Putting streamed block {stream_count}...");
+
+ store.put_block(&cid, &block).await?;
+
+ match cid.codec() {
+ codec_id if codec_id == u64::from(DagCborCodec) => {
+ store.put_links::<DagCborCodec>(&cid, &block).await?;
+ }
+ codec_id if codec_id == u64::from(RawCodec) => {
+ store.put_links::<RawCodec>(&cid, &block).await?;
+ }
+ codec_id => warn!("Unrecognized codec {}; skipping...", codec_id),
+ }
+ }
+
+ trace!("Successfully put {stream_count} blocks from stream...");
+
+ Ok(())
+}
diff --git a/rust/noosphere-core/src/stream/car.rs b/rust/noosphere-core/src/stream/car.rs
new file mode 100644
index 000000000..32926ae3f
--- /dev/null
+++ b/rust/noosphere-core/src/stream/car.rs
@@ -0,0 +1,160 @@
+use anyhow::Result;
+use async_stream::try_stream;
+use bytes::Bytes;
+use cid::Cid;
+use futures_util::{sink::SinkExt, TryStreamExt};
+use iroh_car::{CarHeader, CarReader, CarWriter};
+use noosphere_common::ConditionalSend;
+use std::io::{Error as IoError, ErrorKind as IoErrorKind};
+use tokio::sync::mpsc::channel;
+use tokio_stream::Stream;
+use tokio_util::{
+ io::{CopyToBytes, SinkWriter, StreamReader},
+ sync::PollSender,
+};
+
+/// Takes a [Bytes] stream and interprets it as a CARv1, returning a stream of
+/// `(Cid, Vec<u8>)` blocks.
+pub fn from_car_stream<S, E>(
+ stream: S,
+) -> impl Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend + 'static
+where
+ E: std::error::Error + Send + Sync + 'static,
+ S: Stream<Item = Result<Bytes, E>> + ConditionalSend + 'static,
+{
+ let stream = stream.map_err(|error| std::io::Error::new(std::io::ErrorKind::Other, error));
+
+ try_stream! {
+ tokio::pin!(stream);
+
+ let reader = CarReader::new(StreamReader::new(stream)).await?;
+ let stream = reader.stream();
+
+ tokio::pin!(stream);
+
+ while let Some(entry) = tokio_stream::StreamExt::try_next(&mut stream).await? {
+ yield entry;
+ }
+ }
+}
+
+/// Takes a list of roots and a stream of blocks (pairs of [Cid] and
+/// corresponding [Vec<u8>]), and produces an async byte stream that yields a
+/// valid [CARv1](https://ipld.io/specs/transport/car/carv1/)
+pub fn to_car_stream<S>(
+ mut roots: Vec<Cid>,
+ block_stream: S,
+) -> impl Stream<Item = Result<Bytes, IoError>> + ConditionalSend
+where
+ S: Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend,
+{
+ if roots.is_empty() {
+ roots = vec![Cid::default()]
+ }
+
+ try_stream! {
+ let (tx, mut rx) = channel::<Bytes>(16);
+ let sink =
+ PollSender::new(tx).sink_map_err(|error| {
+ error!("Failed to send CAR frame: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ });
+
+ let mut car_buffer = SinkWriter::new(CopyToBytes::new(sink));
+ let car_header = CarHeader::new_v1(roots);
+ let mut car_writer = CarWriter::new(car_header, &mut car_buffer);
+ let mut sent_blocks = false;
+
+ for await item in block_stream {
+ sent_blocks = true;
+ let (cid, block) = item.map_err(|error| {
+ error!("Failed to stream blocks: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ })?;
+
+ car_writer.write(cid, block).await.map_err(|error| {
+ error!("Failed to write CAR frame: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ })?;
+
+ car_writer.flush().await.map_err(|error| {
+ error!("Failed to flush CAR frames: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ })?;
+
+ while let Ok(block) = rx.try_recv() {
+ yield block;
+ }
+ }
+
+ if !sent_blocks {
+ car_writer.write_header().await.map_err(|error| {
+ error!("Failed to write CAR frame: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ })?;
+ car_writer.flush().await.map_err(|error| {
+ error!("Failed to flush CAR frames: {}", error);
+ IoError::from(IoErrorKind::BrokenPipe)
+ })?;
+
+ while let Ok(block) = rx.try_recv() {
+ yield block;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use anyhow::Result;
+ use async_stream::try_stream;
+ use cid::Cid;
+ use futures_util::Stream;
+ use libipld_cbor::DagCborCodec;
+ use noosphere_common::helpers::TestEntropy;
+ use noosphere_storage::{BlockStore, MemoryStorage, Storage};
+
+ use rand::Rng;
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ use crate::stream::{from_car_stream, put_block_stream, to_car_stream};
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_converts_block_streams_to_car_streams_and_back() -> Result<()> {
+ let test_entropy = TestEntropy::default();
+ let rng_one = test_entropy.to_rng();
+ let storage = MemoryStorage::default();
+ let store_one = storage.get_block_store("one").await?;
+ let store_two = storage.get_block_store("two").await?;
+
+ let block_stream = {
+ let mut store_one = store_one.clone();
+ try_stream! {
+ for _ in 0..10 {
+ let block_bytes = rng_one.lock().await.gen::<[u8; 32]>();
+ let block_cid = store_one.save::<DagCborCodec, _>(block_bytes.as_ref()).await?;
+
+ yield (block_cid, Vec::from(block_bytes));
+ }
+ }
+ };
+ // See: https://github.com/tokio-rs/async-stream/issues/33#issuecomment-1261435381
+ let _: &dyn Stream<Item = Result<(Cid, Vec<u8>)>> = &block_stream;
+
+ let car_stream = to_car_stream(vec![], block_stream);
+
+ let output_block_stream = from_car_stream(car_stream);
+
+ put_block_stream(store_two.clone(), output_block_stream).await?;
+
+ assert_eq!(store_one.entries.lock().await.len(), 10);
+ store_one.expect_replica_in(&store_two).await?;
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/stream/memo.rs b/rust/noosphere-core/src/stream/memo.rs
new file mode 100644
index 000000000..3da8482f5
--- /dev/null
+++ b/rust/noosphere-core/src/stream/memo.rs
@@ -0,0 +1,283 @@
+use std::str::FromStr;
+
+use crate::{
+ authority::collect_ucan_proofs,
+ data::{ContentType, Link, MemoIpld, SphereIpld},
+ view::Sphere,
+};
+use anyhow::{anyhow, Result};
+use async_stream::try_stream;
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use noosphere_common::{spawn, ConditionalSend, TaskQueue};
+use noosphere_storage::{BlockStore, BlockStoreTap, UcanStore};
+use tokio::select;
+use tokio_stream::{Stream, StreamExt};
+use ucan::{store::UcanJwtStore, Ucan};
+
+use crate::stream::walk::{
+ walk_versioned_map_changes_and, walk_versioned_map_elements, walk_versioned_map_elements_and,
+};
+use crate::view::BodyChunkDecoder;
+
+/// Stream all the blocks required to reconstruct the history of a sphere since a
+/// given point in time (or else the beginning of the history).
+// TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy warning
+#[allow(clippy::let_with_type_underscore)]
+#[instrument(level = "trace", skip(store))]
+pub fn memo_history_stream<S>(
+ store: S,
+ latest: &Link<MemoIpld>,
+ since: Option<&Link<MemoIpld>>,
+ include_content: bool,
+) -> impl Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend
+where
+ S: BlockStore + 'static,
+{
+ debug!("Streaming history via memo...");
+
+ let latest = latest.clone();
+ let since = since.cloned();
+
+ try_stream! {
+
+ let (store, mut rx) = BlockStoreTap::new(store.clone(), 64);
+ let memo = store.load::<DagCborCodec, MemoIpld>(&latest).await?;
+
+ match memo.content_type() {
+ Some(ContentType::Sphere) => {
+ let mut history_task = Box::pin(spawn(async move {
+ let sphere = Sphere::from_memo(&memo, &store)?;
+ let identity = sphere.get_identity().await?;
+ let mut tasks = TaskQueue::default();
+
+ let mut previous_sphere_body_version = None;
+ let mut previous_sphere_body: Option<SphereIpld> = None;
+
+ let history_stream = sphere.into_history_stream(since.as_ref());
+
+ tokio::pin!(history_stream);
+
+ while let Some((version, sphere)) = history_stream.try_next().await? {
+ if let Some(previous_sphere_body_version) = previous_sphere_body_version {
+ let memo = sphere.to_memo().await?;
+ if memo.body == previous_sphere_body_version {
+ warn!("Skipping {version} delta for {identity}, no sphere changes detected...");
+ continue;
+ }
+ }
+
+ debug!("Replicating {version} delta for {identity}");
+
+ let sphere_body = sphere.to_body().await?;
+ let (replicate_authority, replicate_address_book, replicate_content) = {
+ if let Some(previous_sphere_body) = previous_sphere_body {
+ (previous_sphere_body.authority != sphere_body.authority,
+ previous_sphere_body.address_book != sphere_body.address_book,
+ previous_sphere_body.content != sphere_body.content)
+ } else {
+ (true, true, true)
+ }
+ };
+
+ if replicate_authority {
+ debug!("Replicating authority...");
+ let authority = sphere.get_authority().await?;
+ let store = store.clone();
+
+ tasks.spawn(async move {
+ let delegations = authority.get_delegations().await?;
+
+ walk_versioned_map_changes_and(delegations, store, |_, delegation, store| async move {
+ let ucan_store = UcanStore(store);
+
+ collect_ucan_proofs(&Ucan::from_str(&ucan_store.require_token(&delegation.jwt).await?)?, &ucan_store).await?;
+
+ Ok(())
+ }).await?;
+
+ let revocations = authority.get_revocations().await?;
+ revocations.load_changelog().await?;
+
+ Ok(()) as Result<_, anyhow::Error>
+ });
+ }
+
+ if replicate_address_book {
+ debug!("Replicating address book...");
+ let address_book = sphere.get_address_book().await?;
+ let identities = address_book.get_identities().await?;
+
+ tasks.spawn(walk_versioned_map_changes_and(identities, store.clone(), |name, identity, store| async move {
+ let ucan_store = UcanStore(store);
+ trace!("Replicating proofs for {}", name);
+ if let Some(link_record) = identity.link_record(&ucan_store).await {
+ link_record.collect_proofs(&ucan_store).await?;
+ };
+
+ Ok(())
+ }));
+ }
+
+ if replicate_content {
+ debug!("Replicating content...");
+ let content = sphere.get_content().await?;
+ let include_content = include_content;
+
+ tasks.spawn(walk_versioned_map_changes_and(content, store.clone(), move |_, link, store| async move {
+ if include_content {
+ walk_memo_body(store, &link).await?;
+ } else {
+ link.load_from(&store).await?;
+ };
+ Ok(())
+ }));
+ }
+
+ previous_sphere_body = Some(sphere_body);
+ previous_sphere_body_version = Some(sphere.to_memo().await?.body);
+
+ drop(sphere);
+ }
+
+ drop(store);
+
+ tasks.join().await?;
+
+ trace!("Done replicating!");
+
+ Ok(()) as Result<(), anyhow::Error>
+ }));
+
+ let mut receiver_is_open = true;
+ let mut history_task_finished = false;
+ let mut yield_count = 0usize;
+
+ while receiver_is_open {
+ select! {
+ next = rx.recv() => {
+ if let Some(block) = next {
+ trace!(cid = ?block.0, "Yielding block {yield_count}...");
+ yield_count += 1;
+ yield block;
+ } else {
+ trace!("Receiver closed!");
+ receiver_is_open = false;
+ }
+ Ok(Ok::<_, anyhow::Error>(()))
+ },
+ history_result = &mut history_task, if !history_task_finished => {
+ trace!("History task completed!");
+ history_task_finished = true;
+ history_result
+ }
+ }??;
+ }
+
+ trace!("Done yielding {yield_count} blocks!");
+ }
+ _ => {
+ Err(anyhow!("History streaming is only supported for spheres, but {latest} has content type {:?})", memo.content_type()))?;
+ }
+ }
+ }
+}
+
+/// Stream all the blocks required to read the sphere at a given version (making no
+/// assumptions of what historical data may already be available to the reader).
+// TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy warning
+#[allow(clippy::let_with_type_underscore)]
+#[instrument(level = "trace", skip(store))]
+pub fn memo_body_stream<S>(
+ store: S,
+ memo_version: &Cid,
+) -> impl Stream<Item = Result<(Cid, Vec<u8>)>> + ConditionalSend
+where
+ S: BlockStore + 'static,
+{
+ debug!("Streaming body via memo...");
+
+ let memo_version = *memo_version;
+
+ try_stream! {
+ let (store, mut rx) = BlockStoreTap::new(store.clone(), 1024);
+
+ let mut receiver_is_open = true;
+ let mut walk_memo_finished = false;
+ let mut walk_memo_finishes = Box::pin(walk_memo_body(store, &memo_version));
+
+ while receiver_is_open {
+ select! {
+ next = rx.recv() => {
+ if let Some(block) = next {
+ trace!("Yielding {}", block.0);
+ yield block;
+ } else {
+ receiver_is_open = false;
+ }
+ Ok::<_, anyhow::Error>(())
+ },
+ walk_memo_results = &mut walk_memo_finishes, if !walk_memo_finished => {
+ walk_memo_finished = true;
+ walk_memo_results
+ }
+ }?;
+ }
+ }
+}
+
+#[allow(clippy::let_with_type_underscore)]
+#[instrument(level = "trace", skip(store))]
+async fn walk_memo_body<S>(store: S, memo_version: &Cid) -> Result<()>
+where
+ S: BlockStore + 'static,
+{
+ let memo = store.load::<DagCborCodec, MemoIpld>(memo_version).await?;
+ match memo.content_type() {
+ Some(ContentType::Sphere) => {
+ let sphere = Sphere::from_memo(&memo, &store)?;
+ let authority = sphere.get_authority().await?;
+ let address_book = sphere.get_address_book().await?;
+ let content = sphere.get_content().await?;
+ let identities = address_book.get_identities().await?;
+ let delegations = authority.get_delegations().await?;
+ let revocations = authority.get_revocations().await?;
+
+ let mut tasks = TaskQueue::default();
+
+ tasks.spawn(walk_versioned_map_elements_and(
+ identities,
+ store.clone(),
+ |_, identity, store| async move {
+ let ucan_store = UcanStore(store);
+ if let Some(link_record) = identity.link_record(&ucan_store).await {
+ link_record.collect_proofs(&ucan_store).await?;
+ };
+ Ok(())
+ },
+ ));
+ tasks.spawn(walk_versioned_map_elements_and(
+ content,
+ store.clone(),
+ move |_, link, store| async move {
+ link.load_from(&store).await?;
+ Ok(())
+ },
+ ));
+ tasks.spawn(walk_versioned_map_elements(delegations));
+ tasks.spawn(walk_versioned_map_elements(revocations));
+
+ tasks.join().await?;
+ }
+ Some(_) => {
+ let stream = BodyChunkDecoder(&memo.body, &store).stream();
+
+ tokio::pin!(stream);
+
+ while (stream.try_next().await?).is_some() {}
+ }
+ None => (),
+ };
+
+ Ok(())
+}
diff --git a/rust/noosphere-core/src/stream/mod.rs b/rust/noosphere-core/src/stream/mod.rs
new file mode 100644
index 000000000..36496667a
--- /dev/null
+++ b/rust/noosphere-core/src/stream/mod.rs
@@ -0,0 +1,406 @@
+//! Utilities to support producing streams of blocks, as well as converting
+//! streams of blocks to and from CARv1-encoded byte streams
+
+mod block;
+mod car;
+mod memo;
+mod walk;
+
+pub use block::*;
+pub use car::*;
+pub use memo::*;
+pub use walk::*;
+
+#[cfg(test)]
+mod tests {
+ use anyhow::Result;
+ use std::collections::BTreeSet;
+ use ucan::store::UcanJwtStore;
+
+ use crate::{
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereContentWrite, SpherePetnameWrite,
+ },
+ data::{BodyChunkIpld, ContentType, LinkRecord, MemoIpld},
+ helpers::{
+ make_valid_link_record, simulated_sphere_context, touch_all_sphere_blocks,
+ SimulationAccess,
+ },
+ stream::{from_car_stream, memo_body_stream, memo_history_stream, to_car_stream},
+ tracing::initialize_tracing,
+ view::{BodyChunkDecoder, Sphere},
+ };
+ use libipld_cbor::DagCborCodec;
+ use noosphere_storage::{BlockStore, MemoryStore, UcanStore};
+ use tokio_stream::StreamExt;
+
+ #[cfg(target_arch = "wasm32")]
+ use wasm_bindgen_test::wasm_bindgen_test;
+
+ #[cfg(target_arch = "wasm32")]
+ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_includes_all_link_records_and_proofs_from_the_address_book() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+ let mut db = sphere_context.sphere_context().await?.db().clone();
+
+ let (foo_did, foo_link_record, foo_link_record_link) =
+ make_valid_link_record(&mut db).await?;
+
+ sphere_context.set_petname("foo", Some(foo_did)).await?;
+ sphere_context.save(None).await?;
+ sphere_context
+ .set_petname_record("foo", &foo_link_record)
+ .await?;
+ let final_version = sphere_context.save(None).await?;
+
+ let mut other_store = MemoryStore::default();
+
+ let stream = memo_body_stream(
+ sphere_context.sphere_context().await?.db().clone(),
+ &final_version,
+ );
+
+ tokio::pin!(stream);
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ debug!("Received {cid}");
+ other_store.put_block(&cid, &block).await?;
+ }
+
+ let ucan_store = UcanStore(other_store);
+
+ let link_record =
+ LinkRecord::try_from(ucan_store.require_token(&foo_link_record_link).await?)?;
+
+ assert_eq!(link_record, foo_link_record);
+
+ link_record.collect_proofs(&ucan_store).await?;
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_stream_all_blocks_in_a_sphere_version() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let changes = vec![
+ (vec!["dogs", "birds"], vec!["alice", "bob"]),
+ (vec!["cats", "dogs"], vec!["gordon"]),
+ (vec!["birds"], vec!["cdata"]),
+ (vec!["cows", "beetles"], vec!["jordan", "ben"]),
+ ];
+
+ for (content_change, petname_change) in changes.iter() {
+ for slug in content_change {
+ sphere_context
+ .write(
+ slug,
+ &ContentType::Subtext,
+ format!("{} are cool", slug).as_bytes(),
+ None,
+ )
+ .await?;
+ }
+
+ for petname in petname_change {
+ sphere_context
+ .set_petname(petname, Some(format!("did:key:{}", petname).into()))
+ .await?;
+ }
+
+ sphere_context.save(None).await?;
+ }
+
+ let final_version = sphere_context.version().await?;
+
+ let mut other_store = MemoryStore::default();
+
+ let mut received = BTreeSet::new();
+
+ let stream = memo_body_stream(
+ sphere_context.sphere_context().await?.db().clone(),
+ &final_version,
+ );
+
+ tokio::pin!(stream);
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ debug!("Received {cid}");
+ assert!(
+ !received.contains(&cid),
+ "Got {cid} but we already received it",
+ );
+ received.insert(cid);
+ other_store.put_block(&cid, &block).await?;
+ }
+
+ let sphere = Sphere::at(&final_version, &other_store);
+
+ let content = sphere.get_content().await?;
+ let identities = sphere.get_address_book().await?.get_identities().await?;
+
+ for (content_change, petname_change) in changes.iter() {
+ for slug in content_change {
+ let _ = content.get(&slug.to_string()).await?.cloned().unwrap();
+ }
+
+ for petname in petname_change {
+ let _ = identities.get(&petname.to_string()).await?;
+ }
+ }
+
+ touch_all_sphere_blocks(&sphere).await?;
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_stream_all_delta_blocks_for_a_range_of_history() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let changes = vec![
+ (vec!["dogs", "birds"], vec!["alice", "bob"]),
+ (vec!["cats", "dogs"], vec!["gordon"]),
+ (vec!["birds"], vec!["cdata"]),
+ (vec!["cows", "beetles"], vec!["jordan", "ben"]),
+ ];
+
+ let original_store = sphere_context.sphere_context().await?.db().clone();
+ let mut versions = Vec::new();
+
+ for (content_change, petname_change) in changes.iter() {
+ for slug in content_change {
+ sphere_context
+ .write(
+ slug,
+ &ContentType::Subtext,
+ format!("{} are cool", slug).as_bytes(),
+ None,
+ )
+ .await?;
+ }
+
+ for petname in petname_change {
+ let (id, record, _) =
+ make_valid_link_record(&mut UcanStore(original_store.clone())).await?;
+ sphere_context.set_petname(petname, Some(id)).await?;
+ versions.push(sphere_context.save(None).await?);
+ sphere_context.set_petname_record(petname, &record).await?;
+ }
+
+ versions.push(sphere_context.save(None).await?);
+ }
+
+ let mut other_store = MemoryStore::default();
+
+ let first_version = versions.first().unwrap();
+ let stream = memo_body_stream(original_store.clone(), first_version);
+
+ tokio::pin!(stream);
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ other_store.put_block(&cid, &block).await?;
+ }
+
+ let sphere = Sphere::at(first_version, &other_store);
+
+ touch_all_sphere_blocks(&sphere).await?;
+
+ for i in 1..=3 {
+ let version = versions.get(i).unwrap();
+ let sphere = Sphere::at(version, &other_store);
+
+ assert!(touch_all_sphere_blocks(&sphere).await.is_err());
+ }
+
+ let stream = memo_history_stream(
+ original_store,
+ versions.last().unwrap(),
+ Some(first_version),
+ false,
+ );
+
+ tokio::pin!(stream);
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ other_store.put_block(&cid, &block).await?;
+ }
+
+ for i in 1..=3 {
+ let version = versions.get(i).unwrap();
+ let sphere = Sphere::at(version, &other_store);
+ sphere.hydrate().await?;
+
+ touch_all_sphere_blocks(&sphere).await?;
+ }
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_stream_all_blocks_in_some_sphere_content() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+ let mut db = sphere_context.sphere_context().await?.db_mut().clone();
+
+ let chunks = [b"foo", b"bar", b"baz"];
+
+ let mut next_chunk_cid = None;
+
+ for bytes in chunks.iter().rev() {
+ next_chunk_cid = Some(
+ db.save::<DagCborCodec, _>(&BodyChunkIpld {
+ bytes: bytes.to_vec(),
+ next: next_chunk_cid,
+ })
+ .await?,
+ );
+ }
+
+ let content_cid = sphere_context
+ .link("foo", &ContentType::Bytes, &next_chunk_cid.unwrap(), None)
+ .await?;
+
+ let stream = memo_body_stream(
+ sphere_context.sphere_context().await?.db().clone(),
+ &content_cid,
+ );
+
+ let mut store = MemoryStore::default();
+
+ tokio::pin!(stream);
+
+ while let Some((cid, block)) = stream.try_next().await? {
+ store.put_block(&cid, &block).await?;
+ }
+
+ let memo = store.load::<DagCborCodec, MemoIpld>(&content_cid).await?;
+
+ let mut buffer = Vec::new();
+ let body_stream = BodyChunkDecoder(&memo.body, &store).stream();
+
+ tokio::pin!(body_stream);
+
+ while let Some(bytes) = body_stream.try_next().await? {
+ buffer.append(&mut Vec::from(bytes));
+ }
+
+ assert_eq!(buffer.as_slice(), b"foobarbaz");
+
+ Ok(())
+ }
+
+ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+ #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+ async fn it_can_stream_all_blocks_in_a_sphere_version_as_a_car() -> Result<()> {
+ initialize_tracing(None);
+
+ let (mut sphere_context, _) =
+ simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+
+ let changes = vec![
+ (vec!["dogs", "birds"], vec!["alice", "bob"]),
+ (vec!["cats", "dogs"], vec!["gordon"]),
+ (vec!["birds"], vec!["cdata"]),
+ (vec!["cows", "beetles"], vec!["jordan", "ben"]),
+ ];
+
+ for (content_change, petname_change) in changes.iter() {
+ for slug in content_change {
+ sphere_context
+ .write(
+ slug,
+ &ContentType::Subtext,
+ format!("{} are cool", slug).as_bytes(),
+ None,
+ )
+ .await?;
+ }
+
+ for petname in petname_change {
+ sphere_context
+ .set_petname(petname, Some(format!("did:key:{}", petname).into()))
+ .await?;
+ }
+
+ sphere_context.save(None).await?;
+ }
+
+ let mut db = sphere_context.sphere_context().await?.db().clone();
+ let (id, link_record, _) = make_valid_link_record(&mut db).await?;
+ sphere_context.set_petname("hasrecord", Some(id)).await?;
+ sphere_context.save(None).await?;
+ sphere_context
+ .set_petname_record("hasrecord", &link_record)
+ .await?;
+ sphere_context.save(None).await?;
+
+ let final_version = sphere_context.version().await?;
+
+ let mut other_store = MemoryStore::default();
+
+ let stream = to_car_stream(
+ vec![final_version.clone().into()],
+ memo_body_stream(db.clone(), &final_version),
+ );
+
+ let block_stream = from_car_stream(stream);
+
+ let mut received = BTreeSet::new();
+ tokio::pin!(block_stream);
+
+ while let Some((cid, block)) = block_stream.try_next().await? {
+ debug!("Received {cid}");
+ assert!(
+ !received.contains(&cid),
+ "Got {cid} but we already received it",
+ );
+ received.insert(cid);
+ other_store.put_block(&cid, &block).await?;
+ }
+
+ let sphere = Sphere::at(&final_version, &other_store);
+
+ let content = sphere.get_content().await?;
+ let identities = sphere.get_address_book().await?.get_identities().await?;
+
+ for (content_change, petname_change) in changes.iter() {
+ for slug in content_change {
+ let _ = content.get(&slug.to_string()).await?.cloned().unwrap();
+ }
+
+ for petname in petname_change {
+ let _ = identities.get(&petname.to_string()).await?;
+ }
+ }
+
+ let has_record = identities.get(&"hasrecord".into()).await?.unwrap();
+ let has_record_version = has_record.link_record(&UcanStore(other_store)).await;
+
+ assert!(
+ has_record_version.is_some(),
+ "We got a resolved link record from the stream"
+ );
+
+ touch_all_sphere_blocks(&sphere).await?;
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-core/src/stream/walk.rs b/rust/noosphere-core/src/stream/walk.rs
new file mode 100644
index 000000000..b205b7717
--- /dev/null
+++ b/rust/noosphere-core/src/stream/walk.rs
@@ -0,0 +1,72 @@
+use crate::{
+ data::{MapOperation, VersionedMapKey, VersionedMapValue},
+ view::VersionedMap,
+};
+use anyhow::Result;
+use noosphere_storage::BlockStore;
+use std::ops::Fn;
+use tokio_stream::StreamExt;
+
+/// Given a [VersionedMap], visit its changelog and all of its underlying entries
+pub async fn walk_versioned_map_elements<K, V, S>(
+ versioned_map: VersionedMap<K, V, S>,
+) -> Result<()>
+where
+ K: VersionedMapKey + 'static,
+ V: VersionedMapValue + 'static,
+ S: BlockStore + 'static,
+{
+ versioned_map.get_changelog().await?;
+ let stream = versioned_map.into_stream().await?;
+ tokio::pin!(stream);
+ while (stream.try_next().await?).is_some() {}
+ Ok(())
+}
+
+/// Given a [VersionedMap] and [BlockStore], visit the [VersionedMap]'s
+/// changelog and all of its underlying entries, invoking a callback for each
+/// entry
+pub async fn walk_versioned_map_elements_and<K, V, S, F, Fut>(
+ versioned_map: VersionedMap<K, V, S>,
+ store: S,
+ callback: F,
+) -> Result<()>
+where
+ K: VersionedMapKey + 'static,
+ V: VersionedMapValue + 'static,
+ S: BlockStore + 'static,
+ Fut: std::future::Future<Output = Result<()>>,
+ F: 'static + Fn(K, V, S) -> Fut,
+{
+ versioned_map.get_changelog().await?;
+ let stream = versioned_map.into_stream().await?;
+ tokio::pin!(stream);
+ while let Some((key, value)) = stream.try_next().await? {
+ callback(key, value, store.clone()).await?;
+ }
+ Ok(())
+}
+
+/// Given a [VersionedMap] and [BlockStore], visit the [VersionedMap]'s
+/// changelog; then, invoke the provided callback with each entry associated
+/// with an 'add' operation in the changelog
+pub async fn walk_versioned_map_changes_and<K, V, S, F, Fut>(
+ versioned_map: VersionedMap<K, V, S>,
+ store: S,
+ callback: F,
+) -> Result<()>
+where
+ K: VersionedMapKey + 'static,
+ V: VersionedMapValue + 'static,
+ S: BlockStore + 'static,
+ Fut: std::future::Future<Output = Result<()>>,
+ F: 'static + Fn(K, V, S) -> Fut,
+{
+ let changelog = versioned_map.load_changelog().await?;
+ for op in changelog.changes {
+ if let MapOperation::Add { key, value } = op {
+ callback(key, value, store.clone()).await?;
+ }
+ }
+ Ok(())
+}
diff --git a/rust/noosphere-core/src/tracing.rs b/rust/noosphere-core/src/tracing.rs
index 027e9b0e4..181b4c1e9 100644
--- a/rust/noosphere-core/src/tracing.rs
+++ b/rust/noosphere-core/src/tracing.rs
@@ -9,6 +9,7 @@ pub static NOOSPHERE_LOG_LEVEL_CRATES: &[&str] = &[
"noosphere",
"noosphere_core",
"noosphere_storage",
+ "noosphere_common",
"noosphere_sphere",
"noosphere_into",
"noosphere_gateway",
@@ -116,16 +117,22 @@ impl Default for NoosphereLogFormat {
/// [`env-filter`](https://docs.rs/env_logger/0.10.0/env_logger/#enabling-logging)
#[derive(Clone, Display, EnumString)]
pub enum NoosphereLogLevel {
+ /// Equivalent to [tracing::Level::TRACE]
#[strum(serialize = "trace")]
Trace,
+ /// Equivalent to [tracing::Level::DEBUG]
#[strum(serialize = "debug")]
Debug,
+ /// Equivalent to [tracing::Level::INFO]
#[strum(serialize = "info")]
Info,
+ /// Equivalent to [tracing::Level::WARN]
#[strum(serialize = "warn")]
Warn,
+ /// Equivalent to [tracing::Level::ERROR]
#[strum(serialize = "error")]
Error,
+ /// Disables logging entirely
#[strum(serialize = "off")]
Off,
}
@@ -159,6 +166,8 @@ mod inner {
use std::sync::Once;
static INITIALIZE_TRACING: Once = Once::new();
+ /// Initialize tracing-based logging throughout the Noosphere body of code,
+ /// as well as dependencies that implement tracing-based logging.
pub fn initialize_tracing(_noosphere_log: Option<NoosphereLog>) {
INITIALIZE_TRACING.call_once(|| {
console_error_panic_hook::set_once();
diff --git a/rust/noosphere-core/src/view/authority.rs b/rust/noosphere-core/src/view/authority.rs
index f1ed4ac6c..e67c2d378 100644
--- a/rust/noosphere-core/src/view/authority.rs
+++ b/rust/noosphere-core/src/view/authority.rs
@@ -23,10 +23,13 @@ impl<S> Authority<S>
where
S: BlockStore,
{
+ /// Get the [Cid] that refers to the underlying [AuthorityIpld]
pub fn cid(&self) -> &Cid {
&self.cid
}
+ /// Initialize an [Authority] view over the [AuthorityIpld] referred to by
+ /// the specified [Cid]
pub fn at(cid: &Cid, store: &S) -> Self {
Authority {
cid: *cid,
@@ -45,6 +48,8 @@ where
.clone())
}
+ /// Helper to initialize an empty [AuthorityIpld] in the case that no [Cid]
+ /// is specified
pub async fn at_or_empty<C>(cid: Option<C>, store: &mut S) -> Result<Authority<S>>
where
C: Deref<Target = Cid>,
@@ -55,6 +60,8 @@ where
})
}
+ /// Initialize an empty [AuthorityIpld] and return an [Authority] view over
+ /// it
pub async fn empty(store: &mut S) -> Result<Self> {
let ipld = AuthorityIpld::empty(store).await?;
let cid = store.save::<DagCborCodec, _>(ipld).await?;
@@ -66,12 +73,14 @@ where
})
}
+ /// Initialize the [Delegations] associated with this [Authority]
pub async fn get_delegations(&self) -> Result<Delegations<S>> {
let ipld = self.to_body().await?;
Delegations::at_or_empty(Some(ipld.delegations), &mut self.store.clone()).await
}
+ /// Initialize the [Revocations] associated with this [Authority]
pub async fn get_revocations(&self) -> Result<Revocations<S>> {
let ipld = self.to_body().await?;
diff --git a/rust/noosphere-core/src/view/content.rs b/rust/noosphere-core/src/view/content.rs
new file mode 100644
index 000000000..cfc13029c
--- /dev/null
+++ b/rust/noosphere-core/src/view/content.rs
@@ -0,0 +1,29 @@
+use crate::data::BodyChunkIpld;
+use async_stream::try_stream;
+use bytes::Bytes;
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use noosphere_storage::BlockStore;
+use tokio_stream::Stream;
+
+/// Helper to easily decode a linked list of `BodyChunkIpld` as a byte stream
+pub struct BodyChunkDecoder<'a, 'b, S: BlockStore>(pub &'a Cid, pub &'b S);
+
+impl<'a, 'b, S: BlockStore> BodyChunkDecoder<'a, 'b, S> {
+ /// Consume the [BodyChunkDecoder] and return an async [Stream] of bytes
+ /// representing the raw body contents
+ pub fn stream(self) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Unpin {
+ let mut next = Some(*self.0);
+ let store = self.1.clone();
+ Box::pin(try_stream! {
+ while let Some(cid) = next {
+ debug!("Unpacking block {}...", cid);
+ let chunk = store.load::<DagCborCodec, BodyChunkIpld>(&cid).await.map_err(|error| {
+ std::io::Error::new(std::io::ErrorKind::UnexpectedEof, error.to_string())
+ })?;
+ yield Bytes::from(chunk.bytes);
+ next = chunk.next;
+ }
+ })
+ }
+}
diff --git a/rust/noosphere-core/src/view/mod.rs b/rust/noosphere-core/src/view/mod.rs
index 940b1e5b1..457817e6b 100644
--- a/rust/noosphere-core/src/view/mod.rs
+++ b/rust/noosphere-core/src/view/mod.rs
@@ -1,11 +1,16 @@
+//! Views over raw Noosphere data to support efficient reading and traversal, as
+//! well as provide higher-level operations related to same.
+
mod address;
mod authority;
+mod content;
mod mutation;
mod sphere;
mod timeline;
mod versioned_map;
pub use authority::*;
+pub use content::*;
pub use mutation::*;
pub use sphere::*;
pub use timeline::*;
diff --git a/rust/noosphere-core/src/view/mutation.rs b/rust/noosphere-core/src/view/mutation.rs
index 03069311c..882d51504 100644
--- a/rust/noosphere-core/src/view/mutation.rs
+++ b/rust/noosphere-core/src/view/mutation.rs
@@ -12,9 +12,22 @@ use crate::{
use noosphere_storage::{BlockStore, UcanStore};
+#[cfg(doc)]
+use crate::{
+ data::VersionedMapIpld,
+ view::versioned_map::{Content, Delegations, Identities, Revocations},
+};
+
+/// A [VersionedMapMutation] that corresponds to [Content]
pub type ContentMutation = VersionedMapMutation<String, Link<MemoIpld>>;
+
+/// A [VersionedMapMutation] that corresponds to [Identities]
pub type IdentitiesMutation = VersionedMapMutation<String, IdentityIpld>;
+
+/// A [VersionedMapMutation] that corresponds to [Delegations]
pub type DelegationsMutation = VersionedMapMutation<Link<Jwt>, DelegationIpld>;
+
+/// A [VersionedMapMutation] that corresponds to [Revocations]
pub type RevocationsMutation = VersionedMapMutation<Link<Jwt>, RevocationIpld>;
#[cfg(doc)]
@@ -27,12 +40,19 @@ use crate::view::Sphere;
/// history by the sphere's key.
#[derive(Debug)]
pub struct SphereRevision<S: BlockStore> {
+ /// The [Did] of the sphere that this revision corresponds to
pub sphere_identity: Did,
+ /// A [BlockStore] that contains blocks assocaited with the sphere that this
+ /// revision corresponds to
pub store: S,
+ /// The unsigned memo that wraps the root of the [SphereIpld] for this
+ /// sphere revision
pub memo: MemoIpld,
}
impl<S: BlockStore> SphereRevision<S> {
+ /// Sign the [SphereRevision] with the provided credential and return the
+ /// [Link<MemoIpld>] pointing to the root of the new, signed revision
pub async fn sign<Credential: KeyMaterial>(
&mut self,
credential: &Credential,
@@ -82,6 +102,8 @@ pub struct SphereMutation {
}
impl SphereMutation {
+ /// Initialize a new [SphereMutation] with the [Did] of the author of the
+ /// mutation
pub fn new(did: &str) -> Self {
SphereMutation {
did: did.into(),
@@ -108,38 +130,55 @@ impl SphereMutation {
self.revocations = RevocationsMutation::new(&self.did);
}
+ /// The [Did] of the author of the mutation
pub fn did(&self) -> &str {
&self.did
}
+ /// A mutable reference to the changes to sphere content, given as a
+ /// [ContentMutation]
pub fn content_mut(&mut self) -> &mut ContentMutation {
&mut self.content
}
+ /// An immutable reference to the changes to sphere content, given as a
+ /// [ContentMutation]
pub fn content(&self) -> &ContentMutation {
&self.content
}
+ /// A mutable reference to the changes to sphere identities (petnames),
+ /// given as a [IdentitiesMutation]
pub fn identities_mut(&mut self) -> &mut IdentitiesMutation {
&mut self.identities
}
+ /// An immutable reference to the changes to sphere identities (petnames),
+ /// given as a [IdentitiesMutation]
pub fn identities(&self) -> &IdentitiesMutation {
&self.identities
}
+ /// A mutable reference to the changes to sphere delegations (of authority),
+ /// given as a [DelegationsMutation]
pub fn delegations_mut(&mut self) -> &mut DelegationsMutation {
&mut self.delegations
}
+ /// An immutable reference to the changes to sphere delegations (of
+ /// authority), given as a [DelegationsMutation]
pub fn delegations(&self) -> &DelegationsMutation {
&self.delegations
}
+ /// A mutable reference to the changes to sphere revocations (of authority),
+ /// given as a [RevocationsMutation]
pub fn revocations_mut(&mut self) -> &mut RevocationsMutation {
&mut self.revocations
}
+ /// An immutable reference to the changes to sphere revocations (of
+ /// authority), given as a [RevocationsMutation]
pub fn revocations(&self) -> &RevocationsMutation {
&self.revocations
}
@@ -154,6 +193,7 @@ impl SphereMutation {
}
}
+/// A generalized expression of a mutation to a [VersionedMapIpld]
#[derive(Default, Debug)]
pub struct VersionedMapMutation<K, V>
where
@@ -169,6 +209,9 @@ where
K: VersionedMapKey,
V: VersionedMapValue,
{
+ /// Set the changes as expressed by a [ChangelogIpld] to this
+ /// [VersionedMapMutation]; the mutation will adopt the author of the
+ /// changelog as its own author
pub fn apply_changelog(&mut self, changelog: &ChangelogIpld<MapOperation<K, V>>) -> Result<()> {
let did = changelog
.did
@@ -188,20 +231,26 @@ where
Ok(())
}
+ /// Initialize a new [VersionedMapMutation] whose author has the given [Did]
pub fn new(did: &str) -> Self {
VersionedMapMutation {
did: did.into(),
changes: Default::default(),
}
}
+
+ /// Get the [Did] of the author of the [VersionedMapMutation]
pub fn did(&self) -> &str {
&self.did
}
+ /// Get the changes (as [MapOperation]s) represented by this
+ /// [VersionedMapMutation]
pub fn changes(&self) -> &[MapOperation<K, V>] {
&self.changes
}
+ /// Record a change to the related [VersionedMapIpld] by key and value
pub fn set(&mut self, key: &K, value: &V) {
self.changes.push(MapOperation::Add {
key: key.clone(),
@@ -209,6 +258,7 @@ where
});
}
+ /// Remove a change from the [VersionedMapMutation]
pub fn remove(&mut self, key: &K) {
self.changes.push(MapOperation::Remove { key: key.clone() });
}
diff --git a/rust/noosphere-core/src/view/sphere.rs b/rust/noosphere-core/src/view/sphere.rs
index 914d8cc2a..a4ad89fab 100644
--- a/rust/noosphere-core/src/view/sphere.rs
+++ b/rust/noosphere-core/src/view/sphere.rs
@@ -31,6 +31,10 @@ use noosphere_storage::{base64_decode, block_serialize, BlockStore, SphereDb, St
use super::{address::AddressBook, Authority, Delegations, Identities, Revocations, Timeslice};
+/// An arbitrarily long value for sphere authorizations that should outlive the
+/// keys they authorize
+// TODO: Recent UCAN versions allow setting a null expiry; we should use that
+// instead
pub const SPHERE_LIFETIME: u64 = 315360000000; // 10,000 years (arbitrarily high)
/// High-level Sphere I/O
@@ -224,10 +228,14 @@ impl<S: BlockStore> Sphere<S> {
})
}
+ /// Get an immutable reference to the [BlockStore] that is in use by this
+ /// [Sphere] view
pub fn store(&self) -> &S {
&self.store
}
+ /// Get a mutable reference to the [BlockStore] that is in use by this
+ /// [Sphere] view
pub fn store_mut(&mut self) -> &mut S {
&mut self.store
}
@@ -307,6 +315,8 @@ impl<S: BlockStore> Sphere<S> {
Ok(Authority::at(&sphere.authority, &self.store.clone()))
}
+ /// Attempt to load the [AddressBook] of this sphere. If no address book is
+ /// found, an empty one is initialized.
pub async fn get_address_book(&self) -> Result<AddressBook<S>> {
let sphere = self.to_body().await?;
diff --git a/rust/noosphere-core/src/view/timeline.rs b/rust/noosphere-core/src/view/timeline.rs
index afef5b29b..d9985ece9 100644
--- a/rust/noosphere-core/src/view/timeline.rs
+++ b/rust/noosphere-core/src/view/timeline.rs
@@ -11,16 +11,26 @@ use noosphere_storage::BlockStore;
// Assumptions:
// - network operations are _always_ mediated by a "remote" agent (no client-to-client syncing)
// - the "remote" always has the authoritative state (we always rebase merge onto remote's tip)
+
+#[cfg(doc)]
+use tokio_stream::Stream;
+
+/// A helper for turning contiguous ranges of [Link<MemoIpld>]s into [Timeslice]s.
#[derive(Debug)]
pub struct Timeline<'a, S: BlockStore> {
+ /// The [BlockStore] that will be shared with any [Timeslice]s produced by
+ /// this [Timeline]
pub store: &'a S,
}
impl<'a, S: BlockStore> Timeline<'a, S> {
+ /// Initialize a new [Timeline] with a backing [BlockStore]
pub fn new(store: &'a S) -> Self {
Timeline { store }
}
+ /// Produce a [Timeslice], which represents a reverse-chronological series
+ /// of [Link<MemoIpld>] that occur between a specified bounds.
pub fn slice(
&'a self,
future: &'a Link<MemoIpld>,
@@ -37,6 +47,9 @@ impl<'a, S: BlockStore> Timeline<'a, S> {
// TODO(#263): Consider using async-stream crate for this
// TODO(tokio-rs/tracing#2503): instrument + impl trait causes clippy
// warning
+ /// Produce a [TryStream] whose items are a series of [(Link<MemoIpld>,
+ /// MemoIpld)], each one the ancestor of the last, yielded in
+ /// reverse-chronological order.
#[allow(clippy::let_with_type_underscore)]
#[instrument(level = "trace", skip(self))]
pub fn stream(
@@ -74,30 +87,46 @@ impl<'a, S: BlockStore> Timeline<'a, S> {
}
}
+/// A [Timeslice] represents a bounded chronological range of [Link<MemoIpld>]
+/// within a [Timeline].
#[derive(Debug)]
pub struct Timeslice<'a, S: BlockStore> {
+ /// The associated [Timeline] of this [Timeslice]
pub timeline: &'a Timeline<'a, S>,
+ /// The bound in the chronological "past," e.g., the earliest version;
+ /// `None` means "the (inclusive) beginning"
pub past: Option<&'a Link<MemoIpld>>,
+ /// The bound in the chronological "future" e.g., the most recent version
pub future: &'a Link<MemoIpld>,
+ /// Whether or not to exclude the configured `past` from any iteration over
+ /// the series of versions
pub exclude_past: bool,
}
impl<'a, S: BlockStore> Timeslice<'a, S> {
+ /// Produce a [TryStream] from this [Timeslice] that yields sphere versions
+ /// and their memos in reverse-chronological order
pub fn stream(&self) -> impl TryStream<Item = Result<(Link<MemoIpld>, MemoIpld)>> {
self.timeline
.stream(self.future, self.past, self.exclude_past)
}
+ /// Configure the [Timeslice] to be inclusive of the `past` bound
pub fn include_past(mut self) -> Self {
self.exclude_past = false;
self
}
+ /// Configure the [Timeslice] to be exclusive of the `past` bound
pub fn exclude_past(mut self) -> Self {
self.exclude_past = true;
self
}
+ /// Aggregate an array of versions in chronological order and return it;
+ /// note that this can be quite memory costly (depending on how much history
+ /// is being aggregated), so it is better to stream in reverse-chronological
+ /// order if possible.
pub async fn to_chronological(&self) -> Result<Vec<(Link<MemoIpld>, MemoIpld)>> {
let mut chronological = VecDeque::new();
let mut stream = Box::pin(self.stream());
diff --git a/rust/noosphere-core/src/view/versioned_map.rs b/rust/noosphere-core/src/view/versioned_map.rs
index e3aa56b66..ef4525826 100644
--- a/rust/noosphere-core/src/view/versioned_map.rs
+++ b/rust/noosphere-core/src/view/versioned_map.rs
@@ -20,9 +20,13 @@ use noosphere_storage::{block_serialize, BlockStore};
use super::VersionedMapMutation;
+/// A [VersionedMap] that represents the content space of a sphere
pub type Content<S> = VersionedMap<String, Link<MemoIpld>, S>;
+/// A [VersionedMap] that represents the petname space of a sphere
pub type Identities<S> = VersionedMap<String, IdentityIpld, S>;
+/// A [VersionedMap] that represents the key authorizations in a sphere
pub type Delegations<S> = VersionedMap<Link<Jwt>, DelegationIpld, S>;
+/// A [VersionedMap] that represents the authority revocations in a sphere
pub type Revocations<S> = VersionedMap<Link<Jwt>, RevocationIpld, S>;
/// A view over a [VersionedMapIpld] which provides high-level traversal of the
@@ -61,12 +65,16 @@ where
.clone())
}
+ /// Get the [ChangelogIpld] for the underlying [VersionedMapIpld], loading
+ /// it from storage if it is not available
pub async fn get_changelog(&self) -> Result<&ChangelogIpld<MapOperation<K, V>>> {
self.changelog
.get_or_try_init(|| async { self.load_changelog().await })
.await
}
+ /// Load the [ChangelogIpld] for the underlying [VersionedMapIpld] from
+ /// storage
pub async fn load_changelog(&self) -> Result<ChangelogIpld<MapOperation<K, V>>> {
let ipld = self.to_body().await?;
self.store
@@ -74,17 +82,22 @@ where
.await
}
+ /// Get the [Hamt] for the underlying [VersionedMapIpld], loading it from
+ /// storage if it is not available
pub async fn get_hamt(&self) -> Result<&Hamt<S, V, K>> {
self.hamt
.get_or_try_init(|| async { self.load_hamt().await })
.await
}
+ /// Load the [Hamt] for the underlying [VersionedMapIpld] from storage
async fn load_hamt(&self) -> Result<Hamt<S, V, K>> {
let ipld = self.to_body().await?;
ipld.load_hamt(&self.store).await
}
+ /// Initialize the [VersionedMap] over a [VersionedMapIpld] referred to by its [Cid] if known, or else
+ /// a newly-initialized, empty [VersionedMapIpld].
pub async fn at_or_empty<C>(cid: Option<C>, store: &mut S) -> Result<VersionedMap<K, V, S>>
where
C: Deref<Target = Cid>,
@@ -95,10 +108,12 @@ where
})
}
+ /// Get the [Cid] of the underlying [VersionedMapIpld]
pub fn cid(&self) -> &Cid {
&self.cid
}
+ /// Initialize the [VersionedMap] over a [VersionedMapIpld] referred to by its [Cid]
pub fn at(cid: &Cid, store: &S) -> VersionedMap<K, V, S> {
VersionedMap {
cid: *cid,
@@ -109,6 +124,8 @@ where
}
}
+ /// Initialize and store an empty [VersionedMapIpld], configuring a [VersionedMap] to
+ /// point to it by its [Cid]
pub async fn empty(store: &mut S) -> Result<VersionedMap<K, V, S>> {
let ipld = VersionedMapIpld::<K, V>::empty(store).await?;
let cid = store.save::<DagCborCodec, _>(ipld).await?;
@@ -183,6 +200,8 @@ where
.ok_or_else(|| anyhow!("Key {} not found!", key))
}
+ /// Apply a [VersionedMapMutation] to the underlying [VersionedMapIpld] by iterating
+ /// over the changes in the mutation and performing them one at a time.
pub async fn apply_with_cid<C>(
cid: Option<C>,
mutation: &VersionedMapMutation<K, V>,
@@ -219,6 +238,9 @@ where
store.save::<DagCborCodec, _>(&links_ipld).await
}
+ /// Iterate over the keys and values of the underlying [VersionedMapIpld]
+ /// sequentially. Note: consider using [VersionedMap::stream] instead for
+ /// better ergonomics.
pub async fn for_each<ForEach>(&self, for_each: ForEach) -> Result<()>
where
ForEach: FnMut(&K, &V) -> Result<()>,
@@ -226,6 +248,8 @@ where
self.get_hamt().await?.for_each(for_each).await
}
+ /// Produce a [Stream] that yields `(key, value)` tuples for all entries
+ /// in the underlying [VersionedMapIpld].
pub async fn stream<'a>(
&'a self,
) -> Result<Pin<Box<dyn Stream<Item = Result<(&'a K, &'a V)>> + 'a>>> {
@@ -239,6 +263,8 @@ where
V: VersionedMapValue + 'static,
S: BlockStore + 'static,
{
+ /// Consume the [VersionedMap] and produce a [Stream] that yields `(key,
+ /// value)` tuples for all entries in the underlying [VersionedMapIpld].
pub async fn into_stream(self) -> Result<impl Stream<Item = Result<(K, V)>>> {
Ok(self.load_hamt().await?.into_stream())
}
diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml
index 359aeeb6a..5fa9aa6fd 100644
--- a/rust/noosphere-gateway/Cargo.toml
+++ b/rust/noosphere-gateway/Cargo.toml
@@ -15,6 +15,9 @@ repository = "https://github.com/subconsciousnetwork/noosphere"
homepage = "https://github.com/subconsciousnetwork/noosphere"
readme = "README.md"
+[dependencies]
+tracing = { workspace = true }
+
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls", "stream"] }
@@ -33,8 +36,8 @@ tower = { workspace = true }
tower-http = { workspace = true, features = ["cors", "trace"] }
async-trait = "~0.1"
async-stream = { workspace = true }
-tracing = { workspace = true }
wnfs-namefilter = { version = "0.1.21" }
+wnfs-common = "=0.1.23"
url = { workspace = true, features = ["serde"] }
mime_guess = "^2"
@@ -43,8 +46,6 @@ noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs" }
noosphere-core = { version = "0.15.2", path = "../noosphere-core" }
noosphere-ns = { version = "0.10.2", path = "../noosphere-ns" }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
-noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" }
-noosphere-api = { version = "0.12.2", path = "../noosphere-api" }
ucan = { workspace = true }
ucan-key-support = { workspace = true }
cid = { workspace = true }
diff --git a/rust/noosphere-gateway/src/authority.rs b/rust/noosphere-gateway/src/authority.rs
index 4be66c298..7b81988df 100644
--- a/rust/noosphere-gateway/src/authority.rs
+++ b/rust/noosphere-gateway/src/authority.rs
@@ -10,7 +10,7 @@ use axum::{
};
use libipld_core::cid::Cid;
use noosphere_core::authority::{SphereAbility, SphereReference, SPHERE_SEMANTICS};
-use noosphere_sphere::SphereContext;
+use noosphere_core::context::SphereContext;
use noosphere_storage::NativeStorage;
use tokio::sync::Mutex;
diff --git a/rust/noosphere-gateway/src/error.rs b/rust/noosphere-gateway/src/error.rs
new file mode 100644
index 000000000..bd529cc1e
--- /dev/null
+++ b/rust/noosphere-gateway/src/error.rs
@@ -0,0 +1,62 @@
+use axum::http::StatusCode;
+use axum::response::IntoResponse;
+use axum::Json;
+use noosphere_core::api::v0alpha2::PushError;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct GatewayError {
+ pub error: String,
+}
+
+pub struct GatewayErrorResponse(StatusCode, GatewayError);
+
+impl IntoResponse for GatewayErrorResponse {
+ fn into_response(self) -> axum::response::Response {
+ (self.0, Json(self.1)).into_response()
+ }
+}
+
+impl From<anyhow::Error> for GatewayErrorResponse {
+ fn from(value: anyhow::Error) -> Self {
+ GatewayErrorResponse(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ GatewayError {
+ error: value.to_string(),
+ },
+ )
+ }
+}
+
+impl From<PushError> for GatewayErrorResponse {
+ fn from(value: PushError) -> Self {
+ GatewayErrorResponse(
+ StatusCode::from(&value),
+ GatewayError {
+ error: value.to_string(),
+ },
+ )
+ }
+}
+
+impl From<StatusCode> for GatewayErrorResponse {
+ fn from(value: StatusCode) -> Self {
+ GatewayErrorResponse(
+ value,
+ GatewayError {
+ error: value.to_string(),
+ },
+ )
+ }
+}
+
+impl From<axum::Error> for GatewayErrorResponse {
+ fn from(value: axum::Error) -> Self {
+ GatewayErrorResponse(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ GatewayError {
+ error: value.to_string(),
+ },
+ )
+ }
+}
diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs
index f72914e77..9a0fe28f7 100644
--- a/rust/noosphere-gateway/src/gateway.rs
+++ b/rust/noosphere-gateway/src/gateway.rs
@@ -3,19 +3,20 @@ use axum::extract::DefaultBodyLimit;
use axum::http::{HeaderValue, Method};
use axum::routing::{get, put};
use axum::{Extension, Router, Server};
+use noosphere_core::context::HasMutableSphereContext;
use noosphere_core::data::Did;
use noosphere_ipfs::KuboClient;
-use noosphere_sphere::HasMutableSphereContext;
use noosphere_storage::Storage;
use std::net::TcpListener;
use tower_http::cors::{Any, CorsLayer};
use tower_http::trace::TraceLayer;
use url::Url;
-use noosphere_api::route::Route as GatewayRoute;
+use noosphere_core::api::{v0alpha1, v0alpha2};
+// use noosphere_core::api::route::Route as GatewayRoute;
use crate::{
- route::{did_route, fetch_route, identify_route, push_route, replicate_route},
+ handlers,
worker::{
start_ipfs_syndication, start_name_system, NameSystemConfiguration,
NameSystemConnectionType,
@@ -85,17 +86,31 @@ where
);
let app = Router::new()
- .route(&GatewayRoute::Did.to_string(), get(did_route))
.route(
- &GatewayRoute::Replicate(None).to_string(),
- get(replicate_route::<C, S>),
+ &v0alpha1::Route::Did.to_string(),
+ get(handlers::v0alpha1::did_route),
)
.route(
- &GatewayRoute::Identify.to_string(),
- get(identify_route::<C, S>),
+ &v0alpha1::Route::Replicate(None).to_string(),
+ get(handlers::v0alpha1::replicate_route::<C, S>),
+ )
+ .route(
+ &v0alpha1::Route::Identify.to_string(),
+ get(handlers::v0alpha1::identify_route::<C, S>),
+ )
+ .route(
+ &v0alpha1::Route::Push.to_string(),
+ #[allow(deprecated)]
+ put(handlers::v0alpha1::push_route::<C, S>),
+ )
+ .route(
+ &v0alpha2::Route::Push.to_string(),
+ put(handlers::v0alpha2::push_route::<C, S>),
+ )
+ .route(
+ &v0alpha1::Route::Fetch.to_string(),
+ get(handlers::v0alpha1::fetch_route::<C, S>),
)
- .route(&GatewayRoute::Push.to_string(), put(push_route::<C, S>))
- .route(&GatewayRoute::Fetch.to_string(), get(fetch_route::<C, S>))
.layer(Extension(sphere_context.clone()))
.layer(Extension(gateway_scope.clone()))
.layer(Extension(ipfs_client))
diff --git a/rust/noosphere-gateway/src/handlers/mod.rs b/rust/noosphere-gateway/src/handlers/mod.rs
new file mode 100644
index 000000000..9f424705d
--- /dev/null
+++ b/rust/noosphere-gateway/src/handlers/mod.rs
@@ -0,0 +1,2 @@
+pub mod v0alpha1;
+pub mod v0alpha2;
diff --git a/rust/noosphere-gateway/src/route/did.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/did.rs
similarity index 100%
rename from rust/noosphere-gateway/src/route/did.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/did.rs
diff --git a/rust/noosphere-gateway/src/route/fetch.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs
similarity index 92%
rename from rust/noosphere-gateway/src/route/fetch.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs
index 37231affb..8067394da 100644
--- a/rust/noosphere-gateway/src/route/fetch.rs
+++ b/rust/noosphere-gateway/src/handlers/v0alpha1/fetch.rs
@@ -4,14 +4,15 @@ use anyhow::Result;
use axum::{body::StreamBody, extract::Query, http::StatusCode, Extension};
use bytes::Bytes;
-use noosphere_api::data::FetchParameters;
use noosphere_core::{
+ api::v0alpha1::FetchParameters,
authority::{generate_capability, SphereAbility},
+ context::HasSphereContext,
data::{Link, MemoIpld},
+ stream::{memo_history_stream, to_car_stream},
view::Sphere,
};
use noosphere_ipfs::{IpfsStore, KuboClient};
-use noosphere_sphere::{car_stream, memo_history_stream, HasSphereContext};
use noosphere_storage::{BlockStoreRetry, SphereDb, Storage};
use tokio_stream::{Stream, StreamExt};
@@ -71,7 +72,7 @@ where
.map(|cid| cid.to_string())
.unwrap_or_else(|| "the beginning...".into())
);
- return Ok(Box::pin(car_stream(vec![], tokio_stream::empty())));
+ return Ok(Box::pin(to_car_stream(vec![], tokio_stream::empty())));
}
debug!(
@@ -84,7 +85,7 @@ where
let store = BlockStoreRetry::from(IpfsStore::new(db.clone(), Some(ipfs_client)));
- let stream = memo_history_stream(store.clone(), &latest_local_sphere_cid, since);
+ let stream = memo_history_stream(store.clone(), &latest_local_sphere_cid, since, false);
debug!("Resolving latest counterpart sphere version...");
@@ -116,18 +117,19 @@ where
.unwrap_or_else(|| "the beginning".into())
);
- return Ok(Box::pin(car_stream(
+ return Ok(Box::pin(to_car_stream(
vec![latest_local_sphere_cid.into()],
stream.merge(memo_history_stream(
store,
latest_counterpart_sphere_cid,
since.as_ref(),
+ false,
)),
)));
}
None => {
warn!("No revisions found for counterpart {}!", scope.counterpart);
- Ok(Box::pin(car_stream(
+ Ok(Box::pin(to_car_stream(
vec![latest_local_sphere_cid.into()],
stream,
)))
diff --git a/rust/noosphere-gateway/src/route/identify.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs
similarity index 92%
rename from rust/noosphere-gateway/src/route/identify.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs
index df42026cd..7d522d21f 100644
--- a/rust/noosphere-gateway/src/route/identify.rs
+++ b/rust/noosphere-gateway/src/handlers/v0alpha1/identify.rs
@@ -1,8 +1,8 @@
use crate::{authority::GatewayAuthority, GatewayScope};
use axum::{http::StatusCode, response::IntoResponse, Extension, Json};
-use noosphere_api::data::IdentifyResponse;
+use noosphere_core::api::v0alpha1::IdentifyResponse;
use noosphere_core::authority::{generate_capability, SphereAbility};
-use noosphere_sphere::HasSphereContext;
+use noosphere_core::context::HasSphereContext;
use noosphere_storage::Storage;
pub async fn identify_route<C, S>(
@@ -12,7 +12,7 @@ pub async fn identify_route<C, S>(
) -> Result<impl IntoResponse, StatusCode>
where
C: HasSphereContext<S>,
- S: Storage,
+ S: Storage + 'static,
{
debug!("Invoking identify route...");
diff --git a/rust/noosphere-gateway/src/route/mod.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/mod.rs
similarity index 100%
rename from rust/noosphere-gateway/src/route/mod.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/mod.rs
diff --git a/rust/noosphere-gateway/src/route/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs
similarity index 98%
rename from rust/noosphere-gateway/src/route/push.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/push.rs
index 7888314b1..858f5a5d6 100644
--- a/rust/noosphere-gateway/src/route/push.rs
+++ b/rust/noosphere-gateway/src/handlers/v0alpha1/push.rs
@@ -4,13 +4,13 @@ use anyhow::Result;
use axum::{http::StatusCode, Extension};
-use noosphere_api::data::{PushBody, PushError, PushResponse};
+use noosphere_core::api::v0alpha1::{PushBody, PushError, PushResponse};
+use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor};
use noosphere_core::{
authority::{generate_capability, SphereAbility},
data::{Bundle, Link, LinkRecord, MapOperation, MemoIpld},
view::Sphere,
};
-use noosphere_sphere::{HasMutableSphereContext, SphereContentWrite, SphereCursor};
use noosphere_storage::Storage;
use tokio::sync::mpsc::UnboundedSender;
use tokio_stream::StreamExt;
@@ -23,6 +23,7 @@ use crate::{
};
// #[debug_handler]
+#[deprecated(since = "0.8.1", note = "Please migrate to v0alpha2")]
#[instrument(
level = "debug",
skip(
diff --git a/rust/noosphere-gateway/src/route/replicate.rs b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs
similarity index 96%
rename from rust/noosphere-gateway/src/route/replicate.rs
rename to rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs
index 860612b77..0d213fc11 100644
--- a/rust/noosphere-gateway/src/route/replicate.rs
+++ b/rust/noosphere-gateway/src/handlers/v0alpha1/replicate.rs
@@ -10,15 +10,14 @@ use axum::{
use bytes::Bytes;
use cid::Cid;
use libipld_cbor::DagCborCodec;
-use noosphere_api::data::ReplicateParameters;
+use noosphere_core::api::v0alpha1::ReplicateParameters;
+use noosphere_core::context::HasMutableSphereContext;
+use noosphere_core::stream::{memo_body_stream, memo_history_stream, to_car_stream};
use noosphere_core::{
authority::{generate_capability, SphereAbility},
data::{ContentType, MemoIpld},
};
use noosphere_ipfs::{IpfsStore, KuboClient};
-use noosphere_sphere::{
- car_stream, memo_body_stream, memo_history_stream, HasMutableSphereContext,
-};
use noosphere_storage::{BlockStore, BlockStoreRetry, Storage};
use tokio_stream::Stream;
@@ -85,9 +84,9 @@ where
// of usage. Maybe somewhere in the ballpark of 1~10k revisions. It
// should be a large-but-finite number.
debug!("Streaming revisions from {} to {}", since, memo_version);
- return Ok(StreamBody::new(Box::pin(car_stream(
+ return Ok(StreamBody::new(Box::pin(to_car_stream(
vec![memo_version],
- memo_history_stream(store, &memo_version.into(), Some(&since)),
+ memo_history_stream(store, &memo_version.into(), Some(&since), false),
))));
} else {
error!("Suggested version {since} is not a valid ancestor of {memo_version}");
@@ -104,7 +103,7 @@ where
debug!("Streaming entire version for {}", memo_version);
// Always fall back to a full replication
- Ok(StreamBody::new(Box::pin(car_stream(
+ Ok(StreamBody::new(Box::pin(to_car_stream(
vec![memo_version],
memo_body_stream(store, &memo_version),
))))
@@ -159,16 +158,16 @@ mod tests {
use super::is_allowed_to_replicate_incrementally;
use anyhow::Result;
+ use noosphere_core::context::{
+ HasMutableSphereContext, HasSphereContext, SphereContext, SphereContextKey, SphereCursor,
+ };
+ use noosphere_core::helpers::{simulated_sphere_context, SimulationAccess};
use noosphere_core::{
authority::{
generate_capability, generate_ed25519_key, Author, Authorization, SphereAbility,
},
data::{DelegationIpld, RevocationIpld},
};
- use noosphere_sphere::{
- helpers::{simulated_sphere_context, SimulationAccess},
- HasMutableSphereContext, HasSphereContext, SphereContext, SphereContextKey, SphereCursor,
- };
use tokio::sync::Mutex;
use ucan::{builder::UcanBuilder, crypto::KeyMaterial};
diff --git a/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs b/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs
new file mode 100644
index 000000000..621b20604
--- /dev/null
+++ b/rust/noosphere-gateway/src/handlers/v0alpha2/mod.rs
@@ -0,0 +1,3 @@
+mod push;
+
+pub use push::*;
diff --git a/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs
new file mode 100644
index 000000000..09b5f110c
--- /dev/null
+++ b/rust/noosphere-gateway/src/handlers/v0alpha2/push.rs
@@ -0,0 +1,376 @@
+use std::{collections::BTreeSet, marker::PhantomData};
+
+use anyhow::Result;
+
+use async_stream::try_stream;
+use axum::{body::StreamBody, extract::BodyStream, Extension};
+
+use bytes::Bytes;
+use cid::Cid;
+use libipld_cbor::DagCborCodec;
+use noosphere_core::api::v0alpha2::{PushBody, PushError, PushResponse};
+use noosphere_core::context::{HasMutableSphereContext, SphereContentWrite, SphereCursor};
+use noosphere_core::stream::{
+ from_car_stream, memo_history_stream, put_block_stream, to_car_stream,
+};
+use noosphere_core::{
+ authority::{generate_capability, SphereAbility},
+ data::{Link, LinkRecord, MapOperation, MemoIpld},
+ view::Sphere,
+};
+use noosphere_storage::{block_deserialize, block_serialize, Storage};
+use tokio::sync::mpsc::UnboundedSender;
+use tokio_stream::{Stream, StreamExt};
+
+use crate::{
+ authority::GatewayAuthority,
+ error::GatewayErrorResponse,
+ worker::{NameSystemJob, SyndicationJob},
+ GatewayScope,
+};
+
+// #[debug_handler]
+#[instrument(
+ level = "debug",
+ skip(
+ authority,
+ gateway_scope,
+ sphere_context,
+ syndication_tx,
+ name_system_tx,
+ stream
+ )
+)]
+pub async fn push_route<C, S>(
+ authority: GatewayAuthority,
+ Extension(sphere_context): Extension<C>,
+ Extension(gateway_scope): Extension<GatewayScope>,
+ Extension(syndication_tx): Extension<UnboundedSender<SyndicationJob<C>>>,
+ Extension(name_system_tx): Extension<UnboundedSender<NameSystemJob<C>>>,
+ stream: BodyStream,
+) -> Result<StreamBody<impl Stream<Item = Result<Bytes, std::io::Error>>>, GatewayErrorResponse>
+where
+ C: HasMutableSphereContext<S>,
+ S: Storage + 'static,
+{
+ debug!("Invoking push route...");
+
+ authority.try_authorize(&generate_capability(
+ &gateway_scope.counterpart,
+ SphereAbility::Push,
+ ))?;
+
+ let gateway_push_routine = GatewayPushRoutine {
+ sphere_context,
+ gateway_scope,
+ syndication_tx,
+ name_system_tx,
+ block_stream: Box::pin(from_car_stream(stream)),
+ storage_type: PhantomData,
+ };
+
+ Ok(StreamBody::new(gateway_push_routine.invoke().await?))
+}
+
+pub struct GatewayPushRoutine<C, S, St>
+where
+ C: HasMutableSphereContext<S> + 'static,
+ S: Storage + 'static,
+ St: Stream<Item = Result<(Cid, Vec<u8>)>> + Unpin + 'static,
+{
+ sphere_context: C,
+ gateway_scope: GatewayScope,
+ syndication_tx: UnboundedSender<SyndicationJob<C>>,
+ name_system_tx: UnboundedSender<NameSystemJob<C>>,
+ block_stream: St,
+ storage_type: PhantomData<S>,
+}
+
+impl<C, S, St> GatewayPushRoutine<C, S, St>
+where
+ C: HasMutableSphereContext<S> + 'static,
+ S: Storage + 'static,
+ St: Stream<Item = Result<(Cid, Vec<u8>)>> + Unpin + 'static,
+{
+ #[instrument(level = "debug", skip(self))]
+ pub async fn invoke(
+ mut self,
+ ) -> Result<impl Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static, PushError> {
+ debug!("Invoking gateway push...");
+
+ let push_body = self.verify_history().await?;
+
+ debug!(?push_body, "Received valid push body...");
+
+ self.incorporate_history(&push_body).await?;
+ self.synchronize_names(&push_body).await?;
+
+ let (next_version, new_blocks) = self.update_gateway_sphere().await?;
+
+ // These steps are order-independent
+ let _ = tokio::join!(
+ self.notify_name_resolver(&push_body),
+ self.notify_ipfs_syndicator(next_version.clone())
+ );
+
+ let roots = vec![next_version.clone().into()];
+
+ let block_stream = try_stream! {
+ yield block_serialize::<DagCborCodec, _>(PushResponse::Accepted {
+ new_tip: next_version
+ })?;
+
+ for await block in new_blocks {
+ yield block?;
+ }
+ };
+
+ Ok(to_car_stream(roots, block_stream))
+ }
+
+ /// Ensure that the pushed history is not in direct conflict with our
+ /// history (in which case the pusher typically must sync first), and that
+ /// there is no missing history implied by the pushed history (in which
+ /// case, there is probably a sync bug in the client implementation).
+ async fn verify_history(&mut self) -> Result<PushBody, PushError> {
+ debug!("Verifying pushed sphere history...");
+
+ let push_body = if let Some((_, first_block)) = self.block_stream.try_next().await? {
+ block_deserialize::<DagCborCodec, PushBody>(&first_block)?
+ } else {
+ return Err(PushError::UnexpectedBody);
+ };
+
+ let gateway_sphere_tip = self.sphere_context.version().await?;
+ if Some(&gateway_sphere_tip) != push_body.counterpart_tip.as_ref() {
+ warn!(
+ "Gateway sphere conflict; we have {gateway_sphere_tip}, they have {:?}",
+ push_body.counterpart_tip
+ );
+ return Err(PushError::Conflict);
+ }
+
+ let sphere_identity = &push_body.sphere;
+ let gateway_sphere_context = self.sphere_context.sphere_context().await?;
+ let db = gateway_sphere_context.db();
+
+ let local_sphere_base_cid = db.get_version(sphere_identity).await?.map(|cid| cid.into());
+ let request_sphere_base_cid = push_body.local_base.clone();
+
+ match (local_sphere_base_cid, request_sphere_base_cid) {
+ (Some(mine), theirs) => {
+ // TODO(#26): Probably should do some diligence here to check if
+ // their base is even in our lineage. Note that this condition
+ // will be hit if theirs is ahead of mine, which actually
+ // should be a "missing revisions" condition.
+ let conflict = match &theirs {
+ Some(cid) if cid != &mine => true,
+ None => true,
+ _ => false,
+ };
+
+ if conflict {
+ warn!(
+ "Counterpart sphere conflict; we have {mine}, they have {:?}",
+ theirs
+ );
+ return Err(PushError::Conflict);
+ }
+
+ if push_body.local_tip == mine {
+ warn!("No new changes in push body!");
+ return Err(PushError::UpToDate);
+ }
+ }
+ (None, Some(_)) => {
+ error!("Missing local lineage!");
+ return Err(PushError::MissingHistory);
+ }
+ _ => (),
+ };
+
+ Ok(push_body)
+ }
+
+ /// Incorporate the pushed history into our storage, hydrating each new
+ /// revision in the history as we go. Then, update our local pointer to the
+ /// tip of the pushed history.
+ async fn incorporate_history(&mut self, push_body: &PushBody) -> Result<(), PushError> {
+ {
+ debug!("Merging pushed sphere history...");
+ let mut sphere_context = self.sphere_context.sphere_context_mut().await?;
+
+ put_block_stream(sphere_context.db_mut().clone(), &mut self.block_stream).await?;
+
+ let PushBody {
+ local_base: base,
+ local_tip: tip,
+ ..
+ } = &push_body;
+
+ let history: Vec<Result<(Link<MemoIpld>, Sphere<_>)>> =
+ Sphere::at(tip, sphere_context.db())
+ .into_history_stream(base.as_ref())
+ .collect()
+ .await;
+
+ for step in history.into_iter().rev() {
+ let (cid, sphere) = step?;
+ debug!("Hydrating {}", cid);
+ sphere.hydrate().await?;
+ }
+
+ debug!(
+ "Setting {} tip to {}...",
+ self.gateway_scope.counterpart, tip
+ );
+
+ sphere_context
+ .db_mut()
+ .set_version(&self.gateway_scope.counterpart, tip)
+ .await?;
+ }
+
+ self.sphere_context
+ .link_raw(&self.gateway_scope.counterpart, &push_body.local_tip)
+ .await?;
+
+ Ok(())
+ }
+
+ async fn synchronize_names(&mut self, push_body: &PushBody) -> Result<(), PushError> {
+ debug!("Synchronizing name changes to local sphere...");
+
+ let my_sphere = self.sphere_context.to_sphere().await?;
+ let my_names = my_sphere.get_address_book().await?.get_identities().await?;
+
+ let sphere = Sphere::at(&push_body.local_tip, my_sphere.store());
+ let stream = sphere.into_history_stream(push_body.local_base.as_ref());
+
+ tokio::pin!(stream);
+
+ let mut updated_names = BTreeSet::<String>::new();
+ let mut removed_names = BTreeSet::<String>::new();
+
+ // Walk backwards through the history of the pushed sphere and aggregate
+ // name changes into a single mutation
+ while let Ok(Some((_, sphere))) = stream.try_next().await {
+ let changed_names = sphere
+ .get_address_book()
+ .await?
+ .get_identities()
+ .await?
+ .load_changelog()
+ .await?;
+ for operation in changed_names.changes {
+ match operation {
+ MapOperation::Add { key, value } => {
+ // Since we are walking backwards through history, we
+ // can ignore changes to names in the past that we have
+ // already encountered in the future
+ if updated_names.contains(&key) || removed_names.contains(&key) {
+ trace!("Skipping name add for '{}' (already seen)...", key);
+ continue;
+ }
+
+ let my_value = my_names.get(&key).await?;
+
+ // Only add to the mutation if the value has actually
+ // changed to avoid redundantly recording updates made
+ // on the client due to a previous sync
+ if my_value != Some(&value) {
+ debug!("Adding name '{}' ({})...", key, value.did);
+ self.sphere_context
+ .sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .identities_mut()
+ .set(&key, &value);
+ }
+
+ updated_names.insert(key);
+ }
+ MapOperation::Remove { key } => {
+ removed_names.insert(key.clone());
+
+ if updated_names.contains(&key) {
+ trace!("Skipping name removal for '{}' (already seen)...", key);
+ continue;
+ }
+
+ debug!("Removing name '{}'...", key);
+ self.sphere_context
+ .sphere_context_mut()
+ .await?
+ .mutation_mut()
+ .identities_mut()
+ .remove(&key);
+
+ updated_names.insert(key);
+ }
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Apply any mutations accrued during the push operation to the local
+ /// sphere and return the new version, along with the blocks needed to
+ /// synchronize the pusher with the latest local history.
+ async fn update_gateway_sphere(
+ &mut self,
+ ) -> Result<(Link<MemoIpld>, impl Stream<Item = Result<(Cid, Vec<u8>)>>), PushError> {
+ debug!("Updating the gateway's sphere...");
+
+ // NOTE CDATA: "Previous version" doesn't cover all cases; this needs to be a version given
+ // in the push body, or else we don't know how far back we actually have to go (e.g., the name
+ // system may have created a new version in the mean time.
+ let previous_version = self.sphere_context.version().await?;
+ let next_version = SphereCursor::latest(self.sphere_context.clone())
+ .save(None)
+ .await?;
+
+ let db = self.sphere_context.sphere_context().await?.db().clone();
+ let block_stream = memo_history_stream(db, &next_version, Some(&previous_version), false);
+
+ Ok((next_version, block_stream))
+ }
+
+ /// Notify the name system that new names may need to be resolved
+ async fn notify_name_resolver(&self, push_body: &PushBody) -> Result<()> {
+ if let Some(name_record) = &push_body.name_record {
+ if let Err(error) = self.name_system_tx.send(NameSystemJob::Publish {
+ context: self.sphere_context.clone(),
+ record: LinkRecord::try_from(name_record)?,
+ republish: false,
+ }) {
+ warn!("Failed to request name record publish: {}", error);
+ }
+ }
+
+ if let Err(error) = self.name_system_tx.send(NameSystemJob::ResolveSince {
+ context: self.sphere_context.clone(),
+ since: push_body.local_base.clone(),
+ }) {
+ warn!("Failed to request name system resolutions: {}", error);
+ };
+
+ Ok(())
+ }
+
+ /// Request that new history be syndicated to IPFS
+ async fn notify_ipfs_syndicator(&self, next_version: Link<MemoIpld>) -> Result<()> {
+ // TODO(#156): This should not be happening on every push, but rather on
+ // an explicit publish action. Move this to the publish handler when we
+ // have added it to the gateway.
+ if let Err(error) = self.syndication_tx.send(SyndicationJob {
+ revision: next_version,
+ context: self.sphere_context.clone(),
+ }) {
+ warn!("Failed to queue IPFS syndication job: {}", error);
+ };
+
+ Ok(())
+ }
+}
diff --git a/rust/noosphere-gateway/src/lib.rs b/rust/noosphere-gateway/src/lib.rs
index 88eca547e..f988c0250 100644
--- a/rust/noosphere-gateway/src/lib.rs
+++ b/rust/noosphere-gateway/src/lib.rs
@@ -1,4 +1,3 @@
-#[cfg(not(target_arch = "wasm32"))]
#[macro_use]
extern crate tracing;
@@ -15,7 +14,10 @@ mod extractor;
mod worker;
#[cfg(not(target_arch = "wasm32"))]
-mod route;
+mod handlers;
+
+#[cfg(not(target_arch = "wasm32"))]
+mod error;
#[cfg(not(target_arch = "wasm32"))]
mod gateway;
diff --git a/rust/noosphere-gateway/src/worker/name_system.rs b/rust/noosphere-gateway/src/worker/name_system.rs
index 059a0f5be..ea255cce7 100644
--- a/rust/noosphere-gateway/src/worker/name_system.rs
+++ b/rust/noosphere-gateway/src/worker/name_system.rs
@@ -1,28 +1,27 @@
use crate::try_or_reset::TryOrReset;
use anyhow::anyhow;
use anyhow::Result;
-use noosphere_core::data::Link;
-use noosphere_core::data::MemoIpld;
-use noosphere_core::data::{ContentType, Did, IdentityIpld, LinkRecord, MapOperation};
+use noosphere_core::{
+ context::{
+ HasMutableSphereContext, SphereContentRead, SphereContentWrite, SphereCursor,
+ SpherePetnameRead, SpherePetnameWrite, COUNTERPART,
+ },
+ data::{ContentType, Did, IdentityIpld, Link, LinkRecord, MapOperation, MemoIpld},
+};
use noosphere_ipfs::{IpfsStore, KuboClient};
use noosphere_ns::{server::HttpClient as NameSystemHttpClient, NameResolver};
-use noosphere_sphere::{
- HasMutableSphereContext, SphereCursor, SpherePetnameRead, SpherePetnameWrite,
-};
-use noosphere_sphere::{SphereContentRead, SphereContentWrite, COUNTERPART};
-use noosphere_storage::KeyValueStore;
-use noosphere_storage::{BlockStoreRetry, Storage, UcanStore};
-use std::fmt::Display;
-use std::future::Future;
+use noosphere_storage::{BlockStoreRetry, KeyValueStore, Storage, UcanStore};
use std::{
collections::{BTreeMap, BTreeSet},
+ fmt::Display,
+ future::Future,
string::ToString,
sync::Arc,
time::Duration,
};
use strum_macros::Display;
-use tokio::io::AsyncReadExt;
use tokio::{
+ io::AsyncReadExt,
sync::{
mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
oneshot::Sender,
@@ -514,9 +513,9 @@ mod tests {
use noosphere_core::{
authority::{generate_capability, SphereAbility},
data::LINK_RECORD_FACT_NAME,
+ helpers::{simulated_sphere_context, SimulationAccess},
};
use noosphere_ns::helpers::KeyValueNameResolver;
- use noosphere_sphere::helpers::{simulated_sphere_context, SimulationAccess};
use ucan::builder::UcanBuilder;
use super::*;
diff --git a/rust/noosphere-gateway/src/worker/syndication.rs b/rust/noosphere-gateway/src/worker/syndication.rs
index 90c1154f2..d5810e356 100644
--- a/rust/noosphere-gateway/src/worker/syndication.rs
+++ b/rust/noosphere-gateway/src/worker/syndication.rs
@@ -2,15 +2,15 @@ use std::{io::Cursor, sync::Arc};
use anyhow::Result;
use libipld_cbor::DagCborCodec;
+use noosphere_core::context::{
+ metadata::COUNTERPART, HasMutableSphereContext, SphereContentRead, SphereContentWrite,
+ SphereCursor,
+};
use noosphere_core::{
data::{ContentType, Did, Link, MemoIpld},
view::Timeline,
};
use noosphere_ipfs::{IpfsClient, KuboClient};
-use noosphere_sphere::{
- metadata::COUNTERPART, HasMutableSphereContext, SphereContentRead, SphereContentWrite,
- SphereCursor,
-};
use noosphere_storage::{block_deserialize, block_serialize, BlockStore, KeyValueStore, Storage};
use serde::{Deserialize, Serialize};
use tokio::{
@@ -158,11 +158,10 @@ where
let stream = db.query_links(&cid, {
let filter = Arc::new(syndicated_blocks.clone());
- let kubo_client = kubo_client.clone();
move |cid| {
let filter = filter.clone();
- let kubo_client = kubo_client.clone();
+ // let kubo_client = kubo_client.clone();
let cid = *cid;
async move {
@@ -177,11 +176,7 @@ where
return Ok(true);
}
- // This will probably end up being rather noisy for the
- // IPFS node, but hopefully checking for a pin is not
- // overly costly. We may have to come up with a
- // different strategy if this turns out to be too noisy.
- Ok(!kubo_client.block_is_pinned(&cid).await?)
+ Ok(false)
}
}
});
diff --git a/rust/noosphere-into/Cargo.toml b/rust/noosphere-into/Cargo.toml
index 7b5ad78eb..07fd3cf57 100644
--- a/rust/noosphere-into/Cargo.toml
+++ b/rust/noosphere-into/Cargo.toml
@@ -19,7 +19,6 @@ readme = "README.md"
[dependencies]
noosphere-core = { version = "0.15.2", path = "../noosphere-core" }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
-noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" }
subtext = { version = "0.3.2", features = ["stream"] }
async-trait = "~0.1"
url = { workspace = true }
@@ -36,7 +35,7 @@ tokio-stream = { workspace = true }
tokio = { workspace = true, features = ["io-util", "macros", "test-util"] }
async-stream = { workspace = true }
-futures = { version = "~0.3" }
+futures = { workspace = true }
async-compat = { version = "~0.2" }
async-utf8-decoder = { version = "~0.3" }
@@ -44,7 +43,7 @@ ucan = { workspace = true }
ucan-key-support = { workspace = true }
[dev-dependencies]
-noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere", features = ["helpers"] }
+noosphere-core = { version = "0.15.1", path = "../noosphere-core", features = ["helpers"] }
wasm-bindgen-test = { workspace = true }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
diff --git a/rust/noosphere-into/examples/notes-to-html/implementation.rs b/rust/noosphere-into/examples/notes-to-html/implementation.rs
index b945378dd..0f323d073 100644
--- a/rust/noosphere-into/examples/notes-to-html/implementation.rs
+++ b/rust/noosphere-into/examples/notes-to-html/implementation.rs
@@ -1,9 +1,9 @@
use anyhow::{anyhow, Result};
use axum::{error_handling::HandleErrorLayer, http::StatusCode, routing::get_service};
-use noosphere_into::{sphere_into_html, NativeFs};
-use noosphere_sphere::{
+use noosphere_core::context::{
HasMutableSphereContext, SphereContentWrite, SphereContext, SphereContextKey, SphereCursor,
};
+use noosphere_into::{sphere_into_html, NativeFs};
use std::{ffi::OsStr, net::SocketAddr, path::Path, sync::Arc};
use tempfile::TempDir;
use tokio::{
diff --git a/rust/noosphere-into/src/into/html/sphere.rs b/rust/noosphere-into/src/into/html/sphere.rs
index c3b8752d2..86c6c7800 100644
--- a/rust/noosphere-into/src/into/html/sphere.rs
+++ b/rust/noosphere-into/src/into/html/sphere.rs
@@ -2,7 +2,7 @@ use std::{collections::BTreeSet, io::Cursor, path::PathBuf, sync::Arc};
use anyhow::{anyhow, Result};
use cid::Cid;
-use noosphere_sphere::{HasSphereContext, SphereContentRead, SphereCursor};
+use noosphere_core::context::{HasSphereContext, SphereContentRead, SphereCursor};
use noosphere_storage::Storage;
use tokio::sync::Mutex;
use tokio_stream::StreamExt;
@@ -160,17 +160,16 @@ where
pub mod tests {
use std::path::PathBuf;
- use noosphere_core::data::{ContentType, Header};
-
#[cfg(target_arch = "wasm32")]
use wasm_bindgen_test::wasm_bindgen_test;
wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
use crate::write::MemoryWriteTarget;
- use noosphere_sphere::{
+ use noosphere_core::{
+ context::{HasMutableSphereContext, SphereContentWrite, SphereCursor},
+ data::{ContentType, Header},
helpers::{simulated_sphere_context, SimulationAccess},
- HasMutableSphereContext, SphereContentWrite, SphereCursor,
};
use super::sphere_into_html;
diff --git a/rust/noosphere-into/src/transcluder/content.rs b/rust/noosphere-into/src/transcluder/content.rs
index 8249c7fbf..7ea9a75d9 100644
--- a/rust/noosphere-into/src/transcluder/content.rs
+++ b/rust/noosphere-into/src/transcluder/content.rs
@@ -3,8 +3,8 @@ use std::marker::PhantomData;
use crate::{ResolvedLink, TextTransclude, Transclude, Transcluder};
use anyhow::Result;
use async_trait::async_trait;
+use noosphere_core::context::{HasSphereContext, SphereContentRead};
use noosphere_core::data::Header;
-use noosphere_sphere::{HasSphereContext, SphereContentRead};
use noosphere_storage::Storage;
use subtext::{block::Block, primitive::Entity, Peer};
use tokio_stream::StreamExt;
diff --git a/rust/noosphere-into/src/transform/file.rs b/rust/noosphere-into/src/transform/file.rs
index 85b5068e5..26f1834ab 100644
--- a/rust/noosphere-into/src/transform/file.rs
+++ b/rust/noosphere-into/src/transform/file.rs
@@ -1,7 +1,7 @@
use async_stream::stream;
use futures::Stream;
+use noosphere_core::context::SphereFile;
use noosphere_core::data::ContentType;
-use noosphere_sphere::SphereFile;
use tokio::io::AsyncRead;
use crate::{subtext_to_html_document_stream, subtext_to_html_fragment_stream, Transform};
diff --git a/rust/noosphere-into/src/transform/sphere/html.rs b/rust/noosphere-into/src/transform/sphere/html.rs
index 581845b9d..dc35bd7fb 100644
--- a/rust/noosphere-into/src/transform/sphere/html.rs
+++ b/rust/noosphere-into/src/transform/sphere/html.rs
@@ -1,9 +1,9 @@
use crate::{html_document_envelope, subtext_to_html_fragment_stream, Transform, TransformStream};
use async_stream::stream;
use futures::Stream;
+use noosphere_core::context::{HasSphereContext, SphereFile};
use noosphere_core::data::{ContentType, Header};
use noosphere_core::view::Sphere;
-use noosphere_sphere::{HasSphereContext, SphereFile};
use noosphere_storage::{BlockStore, Storage};
/// Given a [Transform] and a [Sphere], produce a stream that yields the file
diff --git a/rust/noosphere-into/src/transform/subtext/html/document.rs b/rust/noosphere-into/src/transform/subtext/html/document.rs
index 3e2787858..c8895d964 100644
--- a/rust/noosphere-into/src/transform/subtext/html/document.rs
+++ b/rust/noosphere-into/src/transform/subtext/html/document.rs
@@ -1,7 +1,7 @@
use crate::{html_document_envelope, subtext_to_html_fragment_stream, Transform};
use async_stream::stream;
use futures::Stream;
-use noosphere_sphere::SphereFile;
+use noosphere_core::context::SphereFile;
use tokio::io::AsyncRead;
/// Given a [Transform] and a [SphereFile], produce a stream that yields the
diff --git a/rust/noosphere-into/src/transform/subtext/html/fragment.rs b/rust/noosphere-into/src/transform/subtext/html/fragment.rs
index c73651225..8fc8b9405 100644
--- a/rust/noosphere-into/src/transform/subtext/html/fragment.rs
+++ b/rust/noosphere-into/src/transform/subtext/html/fragment.rs
@@ -6,8 +6,8 @@ use async_stream::stream;
use futures::Stream;
use horrorshow::{html, Raw};
+use noosphere_core::context::SphereFile;
use noosphere_core::data::ContentType;
-use noosphere_sphere::SphereFile;
use subtext::{block::Block, primitive::Entity, Slashlink};
use tokio::io::AsyncRead;
diff --git a/rust/noosphere-into/src/transform/transform_implementation.rs b/rust/noosphere-into/src/transform/transform_implementation.rs
index 44fb81e87..e1b78227b 100644
--- a/rust/noosphere-into/src/transform/transform_implementation.rs
+++ b/rust/noosphere-into/src/transform/transform_implementation.rs
@@ -1,4 +1,4 @@
-use noosphere_sphere::HasSphereContext;
+use noosphere_core::context::HasSphereContext;
use noosphere_storage::Storage;
use crate::{Resolver, SphereContentTranscluder, StaticHtmlResolver, Transcluder};
diff --git a/rust/noosphere-ipfs/Cargo.toml b/rust/noosphere-ipfs/Cargo.toml
index 3c9f44b22..26ddda0da 100644
--- a/rust/noosphere-ipfs/Cargo.toml
+++ b/rust/noosphere-ipfs/Cargo.toml
@@ -22,7 +22,7 @@ readme = "README.md"
[features]
default = ["storage"]
storage = ["ucan"]
-test_kubo = []
+test-kubo = []
[dependencies]
anyhow = { workspace = true }
@@ -40,6 +40,7 @@ tokio-stream = { workspace = true }
tracing = { workspace = true }
url = { workspace = true, features = [ "serde" ] }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
+noosphere-common = { version = "0.1.0", path = "../noosphere-common" }
ucan = { workspace = true, optional = true }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -48,8 +49,6 @@ hyper-multipart-rfc7578 = "~0.8"
ipfs-api-prelude = "0.6"
[dev-dependencies]
-
-[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
rand = { workspace = true }
iroh-car = { workspace = true }
libipld-cbor = { workspace = true }
diff --git a/rust/noosphere-ipfs/src/client/kubo.rs b/rust/noosphere-ipfs/src/client/kubo.rs
index 41b52c584..0979cbbed 100644
--- a/rust/noosphere-ipfs/src/client/kubo.rs
+++ b/rust/noosphere-ipfs/src/client/kubo.rs
@@ -163,7 +163,7 @@ impl IpfsClient for KuboClient {
// Note that these tests require that there is a locally available IPFS Kubo
// node running with the RPC API enabled
-#[cfg(all(test, feature = "test_kubo"))]
+#[cfg(all(test, feature = "test-kubo"))]
mod tests {
use std::io::Cursor;
diff --git a/rust/noosphere-ipfs/src/storage.rs b/rust/noosphere-ipfs/src/storage.rs
index 67b785c76..f088eb845 100644
--- a/rust/noosphere-ipfs/src/storage.rs
+++ b/rust/noosphere-ipfs/src/storage.rs
@@ -2,6 +2,7 @@ use crate::IpfsClient;
use anyhow::Result;
use async_trait::async_trait;
use cid::Cid;
+use noosphere_common::ConditionalSync;
use noosphere_storage::{BlockStore, Storage};
use std::sync::Arc;
use tokio::sync::RwLock;
@@ -36,24 +37,24 @@ where
}
}
-#[cfg(not(target_arch = "wasm32"))]
-pub trait IpfsStorageConditionalSendSync: Send + Sync {}
+// #[cfg(not(target_arch = "wasm32"))]
+// pub trait IpfsStorageConditionalSendSync: Send + Sync {}
-#[cfg(not(target_arch = "wasm32"))]
-impl<S> IpfsStorageConditionalSendSync for S where S: Send + Sync {}
+// #[cfg(not(target_arch = "wasm32"))]
+// impl<S> IpfsStorageConditionalSendSync for S where S: Send + Sync {}
-#[cfg(target_arch = "wasm32")]
-pub trait IpfsStorageConditionalSendSync {}
+// #[cfg(target_arch = "wasm32")]
+// pub trait IpfsStorageConditionalSendSync {}
-#[cfg(target_arch = "wasm32")]
-impl<S> IpfsStorageConditionalSendSync for S {}
+// #[cfg(target_arch = "wasm32")]
+// impl<S> IpfsStorageConditionalSendSync for S {}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl<S, C> Storage for IpfsStorage<S, C>
where
- S: Storage + IpfsStorageConditionalSendSync,
- C: IpfsClient + IpfsStorageConditionalSendSync,
+ S: Storage + ConditionalSync,
+ C: IpfsClient + ConditionalSync,
{
type BlockStore = IpfsStore<S::BlockStore, C>;
@@ -79,7 +80,7 @@ where
pub struct IpfsStore<B, C>
where
B: BlockStore,
- C: IpfsClient + IpfsStorageConditionalSendSync,
+ C: IpfsClient + ConditionalSync,
{
local_store: Arc<RwLock<B>>,
ipfs_client: Option<C>,
@@ -88,7 +89,7 @@ where
impl<B, C> IpfsStore<B, C>
where
B: BlockStore,
- C: IpfsClient + IpfsStorageConditionalSendSync,
+ C: IpfsClient + ConditionalSync,
{
pub fn new(block_store: B, ipfs_client: Option<C>) -> Self {
IpfsStore {
@@ -103,7 +104,7 @@ where
impl<B, C> BlockStore for IpfsStore<B, C>
where
B: BlockStore,
- C: IpfsClient + IpfsStorageConditionalSendSync,
+ C: IpfsClient + ConditionalSync,
{
#[instrument(skip(self), level = "trace")]
async fn put_block(&mut self, cid: &Cid, block: &[u8]) -> Result<()> {
@@ -141,7 +142,7 @@ where
// Note that these tests require that there is a locally available IPFS Kubo
// node running with the RPC API enabled
-#[cfg(all(test, feature = "test_kubo"))]
+#[cfg(all(test, feature = "test-kubo", not(target_arch = "wasm32")))]
mod tests {
use std::time::Duration;
diff --git a/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs b/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs
index 7846f6906..b5a632fd7 100644
--- a/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs
+++ b/rust/noosphere-ns/src/bin/orb-ns/cli/address.rs
@@ -263,7 +263,7 @@ pub mod test {
assert_eq!(&socket.to_string(), expectation);
}
- for failure_addr in vec![
+ for failure_addr in [
CLIAddress::Url("http://127.0.0.1:6666".parse()?),
CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?),
] {
@@ -303,7 +303,8 @@ pub mod test {
);
}
- for failure_addr in vec![CLIAddress::Url("http://127.0.0.1:6666".parse()?)] {
+ {
+ let failure_addr = CLIAddress::Url("http://127.0.0.1:6666".parse()?);
let result: Result<Multiaddr> = failure_addr.try_into();
assert!(result.is_err());
}
@@ -329,7 +330,8 @@ pub mod test {
assert_eq!(&url.to_string(), expectation);
}
- for failure_addr in vec![CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?)] {
+ {
+ let failure_addr = CLIAddress::Multiaddr("/ip4/127.0.0.1/tcp/6666".parse()?);
let result: Result<Url> = failure_addr.try_into();
assert!(result.is_err());
}
diff --git a/rust/noosphere-sphere/src/cursor.rs b/rust/noosphere-sphere/src/cursor.rs
index 3cb6f61c0..eb2cbbf77 100644
--- a/rust/noosphere-sphere/src/cursor.rs
+++ b/rust/noosphere-sphere/src/cursor.rs
@@ -3,6 +3,7 @@ use async_trait::async_trait;
use noosphere_api::data::ReplicateParameters;
use noosphere_core::{
data::{Link, MemoIpld},
+ stream::put_block_stream,
view::{Sphere, Timeline},
};
use noosphere_storage::Storage;
@@ -192,14 +193,14 @@ where
let replicate_parameters = since.as_ref().map(|since| ReplicateParameters {
since: Some(since.clone()),
});
- let (mut db, client) = {
+ let (db, client) = {
let sphere_context = cursor.sphere_context().await?;
(sphere_context.db().clone(), sphere_context.client().await?)
};
let stream = client
.replicate(&version, replicate_parameters.as_ref())
.await?;
- db.put_block_stream(stream).await?;
+ put_block_stream(db.clone(), stream).await?;
// If this was incremental replication, we have to hydrate...
if let Some(since) = since {
diff --git a/rust/noosphere-sphere/src/internal.rs b/rust/noosphere-sphere/src/internal.rs
index d6913ed17..b736381f5 100644
--- a/rust/noosphere-sphere/src/internal.rs
+++ b/rust/noosphere-sphere/src/internal.rs
@@ -10,6 +10,7 @@ use cid::Cid;
use noosphere_core::{
authority::Access,
data::{ContentType, Header, Link, MemoIpld},
+ stream::put_block_stream,
};
/// A module-private trait for internal trait methods; this is a workaround for
@@ -54,7 +55,7 @@ where
sphere_revision: &Cid,
memo_link: Link<MemoIpld>,
) -> Result<SphereFile<Box<dyn AsyncFileBody>>> {
- let mut db = self.sphere_context().await?.db().clone();
+ let db = self.sphere_context().await?.db().clone();
let memo = memo_link.load_from(&db).await?;
// If we have a memo, but not the content it refers to, we should try to
@@ -76,7 +77,7 @@ where
// into the local DB
let stream = client.replicate(&memo_link, None).await?;
- db.put_block_stream(stream).await?;
+ put_block_stream(db.clone(), stream).await?;
}
let content_type = match memo.get_first_header(&Header::ContentType) {
diff --git a/rust/noosphere-sphere/src/lib.rs b/rust/noosphere-sphere/src/lib.rs
index 680dfa861..6a0bf6dd7 100644
--- a/rust/noosphere-sphere/src/lib.rs
+++ b/rust/noosphere-sphere/src/lib.rs
@@ -11,43 +11,52 @@
//! traversing the Noosphere content graph.
//!
//! ```rust
-//! use anyhow::Result;
-//! use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess};
-//!
-//! use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SphereContentWrite};
-//!
-//! #[tokio::main(flavor = "multi_thread")]
-//! async fn main() -> Result<()> {
-//! let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
-//!
-//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?;
-//! sphere_context.save(None).await?;
-//!
-//! Ok(())
-//! }
+//! # use anyhow::Result;
+//! # #[cfg(feature = "helpers")]
+//! # use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess};
+//! #
+//! # use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SphereContentWrite};
+//! #
+//! # #[cfg(feature = "helpers")]
+//! # #[tokio::main(flavor = "multi_thread")]
+//! # async fn main() -> Result<()> {
+//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+//! #
+//! sphere_context.write("foo", "text/plain", "bar".as_ref(), None).await?;
+//! sphere_context.save(None).await?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # #[cfg(not(feature = "helpers"))]
+//! # fn main() {}
//! ```
//!
//! You can also use a [SphereContext] to access petnames in the sphere:
//!
//! ```rust
-//! use anyhow::Result;
-//! use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess};
-//! use noosphere_core::data::Did;
-//!
-//! use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite};
-//!
-//! #[tokio::main(flavor = "multi_thread")]
-//! async fn main() -> Result<()> {
-//! let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
-//!
-//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?;
-//! sphere_context.save(None).await?;
-//!
-//! Ok(())
-//! }
+//! # use anyhow::Result;
+//! # #[cfg(feature = "helpers")]
+//! # use noosphere_sphere::helpers::{simulated_sphere_context,SimulationAccess};
+//! # use noosphere_core::data::Did;
+//! #
+//! # use noosphere_sphere::{SphereCursor, HasMutableSphereContext, SpherePetnameWrite};
+//! #
+//! # #[cfg(feature = "helpers")]
+//! # #[tokio::main(flavor = "multi_thread")]
+//! # async fn main() -> Result<()> {
+//! # let (mut sphere_context, _) = simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
+//! #
+//! sphere_context.set_petname("cdata", Some("did:key:example".into())).await?;
+//! sphere_context.save(None).await?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # #[cfg(not(feature = "helpers"))]
+//! # fn main() {}
//! ```
//!
-//!
#![warn(missing_docs)]
@@ -71,7 +80,7 @@ mod has;
mod replication;
mod walker;
-#[cfg(any(test, feature = "helpers"))]
+#[cfg(any(doctest, test, feature = "helpers"))]
pub mod helpers;
mod internal;
diff --git a/rust/noosphere-sphere/src/replication/stream.rs b/rust/noosphere-sphere/src/replication/stream.rs
index 6150afbb7..c3227e085 100644
--- a/rust/noosphere-sphere/src/replication/stream.rs
+++ b/rust/noosphere-sphere/src/replication/stream.rs
@@ -504,7 +504,7 @@ mod tests {
simulated_sphere_context(SimulationAccess::ReadWrite, None).await?;
let mut db = sphere_context.sphere_context().await?.db_mut().clone();
- let chunks = vec![b"foo", b"bar", b"baz"];
+ let chunks = [b"foo", b"bar", b"baz"];
let mut next_chunk_cid = None;
diff --git a/rust/noosphere-sphere/src/sync/strategy.rs b/rust/noosphere-sphere/src/sync/strategy.rs
index 115816b3a..b04ac3f9a 100644
--- a/rust/noosphere-sphere/src/sync/strategy.rs
+++ b/rust/noosphere-sphere/src/sync/strategy.rs
@@ -5,6 +5,7 @@ use noosphere_api::data::{FetchParameters, PushBody, PushResponse};
use noosphere_core::{
authority::{generate_capability, SphereAbility},
data::{Did, IdentityIpld, Jwt, Link, MemoIpld, LINK_RECORD_FACT_NAME},
+ stream::put_block_stream,
view::{Sphere, Timeline},
};
use noosphere_storage::{KeyValueStore, SphereDb, Storage};
@@ -178,7 +179,7 @@ where
}
};
- context.db_mut().put_block_stream(block_stream).await?;
+ put_block_stream(context.db_mut().clone(), block_stream).await?;
trace!("Finished putting block stream");
diff --git a/rust/noosphere-storage/Cargo.toml b/rust/noosphere-storage/Cargo.toml
index b3ad41872..1a6042bdf 100644
--- a/rust/noosphere-storage/Cargo.toml
+++ b/rust/noosphere-storage/Cargo.toml
@@ -17,14 +17,13 @@ repository = "https://github.com/subconsciousnetwork/noosphere"
homepage = "https://github.com/subconsciousnetwork/noosphere"
readme = "README.md"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
[dependencies]
anyhow = { workspace = true }
async-trait = "~0.1"
async-stream = { workspace = true }
tokio-stream = { workspace = true }
cid = { workspace = true }
+noosphere-common = { version = "0.1.0", path = "../noosphere-common" }
tracing = "~0.1"
ucan = { workspace = true }
libipld-core = { workspace = true }
diff --git a/rust/noosphere-storage/src/block.rs b/rust/noosphere-storage/src/block.rs
index 26aa0a2ac..22cc7180c 100644
--- a/rust/noosphere-storage/src/block.rs
+++ b/rust/noosphere-storage/src/block.rs
@@ -14,35 +14,12 @@ use libipld_core::{
codec::{Codec, Decode, Encode},
ipld::Ipld,
};
+use noosphere_common::{ConditionalSend, ConditionalSync};
use serde::{de::DeserializeOwned, Serialize};
#[cfg(doc)]
use serde::Deserialize;
-#[cfg(not(target_arch = "wasm32"))]
-pub trait BlockStoreSendSync: Send + Sync {}
-
-#[cfg(not(target_arch = "wasm32"))]
-impl<T> BlockStoreSendSync for T where T: Send + Sync {}
-
-#[cfg(target_arch = "wasm32")]
-pub trait BlockStoreSendSync {}
-
-#[cfg(target_arch = "wasm32")]
-impl<T> BlockStoreSendSync for T {}
-
-#[cfg(not(target_arch = "wasm32"))]
-pub trait BlockStoreSend: Send {}
-
-#[cfg(not(target_arch = "wasm32"))]
-impl<T> BlockStoreSend for T where T: Send {}
-
-#[cfg(target_arch = "wasm32")]
-pub trait BlockStoreSend {}
-
-#[cfg(target_arch = "wasm32")]
-impl<T> BlockStoreSend for T {}
-
/// An interface for storage backends that are suitable for storing blocks. A
/// block is a chunk of bytes that can be addressed by a
/// [CID](https://docs.ipfs.tech/concepts/content-addressing/#identifier-formats).
@@ -50,7 +27,7 @@ impl<T> BlockStoreSend for T {}
/// retrieve blocks given a [Cid].
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
-pub trait BlockStore: Clone + BlockStoreSendSync {
+pub trait BlockStore: Clone + ConditionalSync {
/// Given a CID and a block, store the links (any [Cid] that is part of the
/// encoded data) in a suitable location for later retrieval. This method is
/// optional, and its default implementation is a no-op. It should be
@@ -77,7 +54,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync {
async fn put<C, T>(&mut self, data: T) -> Result<Cid>
where
C: Codec + Default,
- T: Encode<C> + BlockStoreSend,
+ T: Encode<C> + ConditionalSend,
Ipld: References<C>,
{
let codec = C::default();
@@ -114,7 +91,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync {
async fn save<C, T>(&mut self, data: T) -> Result<Cid>
where
C: Codec + Default,
- T: Serialize + BlockStoreSend,
+ T: Serialize + ConditionalSend,
Ipld: Encode<C> + References<C>,
{
self.put::<C, Ipld>(to_ipld(data)?).await
@@ -127,7 +104,7 @@ pub trait BlockStore: Clone + BlockStoreSendSync {
async fn load<C, T>(&self, cid: &Cid) -> Result<T>
where
C: Codec + Default,
- T: DeserializeOwned + BlockStoreSend,
+ T: DeserializeOwned + ConditionalSend,
u64: From<C>,
Ipld: Decode<C>,
{
diff --git a/rust/noosphere-storage/src/db.rs b/rust/noosphere-storage/src/db.rs
index 09660d07d..e54501032 100644
--- a/rust/noosphere-storage/src/db.rs
+++ b/rust/noosphere-storage/src/db.rs
@@ -7,28 +7,17 @@ use libipld_core::{
ipld::Ipld,
raw::RawCodec,
};
+use noosphere_common::ConditionalSend;
use serde::{de::DeserializeOwned, Serialize};
use std::future::Future;
use std::{collections::BTreeSet, fmt::Debug};
-use tokio_stream::{Stream, StreamExt};
-use ucan::store::{UcanStore, UcanStoreConditionalSend};
+use tokio_stream::Stream;
+use ucan::store::UcanStore;
-use crate::{BlockStore, BlockStoreSend, KeyValueStore, MemoryStore, Storage};
+use crate::{BlockStore, KeyValueStore, MemoryStore, Storage};
use async_stream::try_stream;
-#[cfg(not(target_arch = "wasm32"))]
-pub trait SphereDbSendSync: Send + Sync {}
-
-#[cfg(not(target_arch = "wasm32"))]
-impl<T> SphereDbSendSync for T where T: Send + Sync {}
-
-#[cfg(target_arch = "wasm32")]
-pub trait SphereDbSendSync {}
-
-#[cfg(target_arch = "wasm32")]
-impl<T> SphereDbSendSync for T {}
-
pub const BLOCK_STORE: &str = "blocks";
pub const LINK_STORE: &str = "links";
pub const VERSION_STORE: &str = "versions";
@@ -203,36 +192,6 @@ where
}
}
- pub async fn put_block_stream<Str>(&mut self, stream: Str) -> Result<()>
- where
- Str: Stream<Item = Result<(Cid, Vec<u8>)>>,
- {
- tokio::pin!(stream);
-
- let mut stream_count = 0usize;
-
- while let Some((cid, block)) = stream.try_next().await? {
- stream_count += 1;
- trace!(?cid, "Putting streamed block {stream_count}...");
-
- self.put_block(&cid, &block).await?;
-
- match cid.codec() {
- codec_id if codec_id == u64::from(DagCborCodec) => {
- self.put_links::<DagCborCodec>(&cid, &block).await?;
- }
- codec_id if codec_id == u64::from(RawCodec) => {
- self.put_links::<RawCodec>(&cid, &block).await?;
- }
- codec_id => warn!("Unrecognized codec {}; skipping...", codec_id),
- }
- }
-
- trace!("Loaded {stream_count} blocks from stream...");
-
- Ok(())
- }
-
/// Get an owned copy of the underlying primitive [BlockStore] for this
/// [SphereDb]
pub fn to_block_store(&self) -> S::BlockStore {
@@ -278,23 +237,23 @@ where
{
async fn set_key<K, V>(&mut self, key: K, value: V) -> Result<()>
where
- K: AsRef<[u8]> + BlockStoreSend,
- V: Serialize + BlockStoreSend,
+ K: AsRef<[u8]> + ConditionalSend,
+ V: Serialize + ConditionalSend,
{
self.metadata_store.set_key(key, value).await
}
async fn unset_key<K>(&mut self, key: K) -> Result<()>
where
- K: AsRef<[u8]> + BlockStoreSend,
+ K: AsRef<[u8]> + ConditionalSend,
{
self.metadata_store.unset_key(key).await
}
async fn get_key<K, V>(&self, key: K) -> Result<Option<V>>
where
- K: AsRef<[u8]> + BlockStoreSend,
- V: DeserializeOwned + BlockStoreSend,
+ K: AsRef<[u8]> + ConditionalSend,
+ V: DeserializeOwned + ConditionalSend,
{
self.metadata_store.get_key(key).await
}
@@ -310,7 +269,7 @@ where
self.get::<RawCodec, T>(cid).await
}
- async fn write<T: Encode<RawCodec> + UcanStoreConditionalSend + Debug>(
+ async fn write<T: Encode<RawCodec> + ConditionalSend + Debug>(
&mut self,
token: T,
) -> Result<Cid> {
diff --git a/rust/noosphere/Cargo.toml b/rust/noosphere/Cargo.toml
index 5d0227fdc..d594a8420 100644
--- a/rust/noosphere/Cargo.toml
+++ b/rust/noosphere/Cargo.toml
@@ -19,6 +19,7 @@ crate-type = ["rlib", "staticlib", "cdylib"]
default = []
headers = ["safer-ffi/headers"]
ipfs-storage = ["noosphere-ipfs"]
+test-kubo = []
[dependencies]
anyhow = { workspace = true }
@@ -39,15 +40,20 @@ libipld-cbor = { workspace = true }
bytes = "^1"
noosphere-core = { version = "0.15.2", path = "../noosphere-core" }
-noosphere-sphere = { version = "0.10.2", path = "../noosphere-sphere" }
noosphere-storage = { version = "0.8.1", path = "../noosphere-storage" }
-noosphere-api = { version = "0.12.2", path = "../noosphere-api" }
noosphere-ipfs = { version = "0.7.4", path = "../noosphere-ipfs", optional = true }
ucan = { workspace = true }
ucan-key-support = { workspace = true }
[dev-dependencies]
libipld-core = { workspace = true }
+rand = { workspace = true }
+serde_json = { workspace = true }
+noosphere-cli = { version = "0.14.1", path = "../noosphere-cli", features = ["helpers"] }
+noosphere-core = { version = "0.15.1", path = "../noosphere-core", features = ["helpers"] }
+noosphere-common = { version = "0.1.0", path = "../noosphere-common", features = ["helpers"] }
+noosphere-gateway = { version = "0.8.1", path = "../noosphere-gateway" }
+noosphere-ns = { version = "0.10.1", path = "../noosphere-ns" }
[target.'cfg(target_arch = "wasm32")'.dependencies]
# TODO: We should eventually support gateway storage as a specialty target only,
@@ -60,7 +66,7 @@ js-sys = { workspace = true }
noosphere-into = { version = "0.10.6", path = "../noosphere-into" }
[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
-version = "~0.3"
+workspace = true
features = [
"CryptoKey",
]
@@ -71,6 +77,7 @@ tokio = { workspace = true, features = ["full"] }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
tempfile = { workspace = true }
+reqwest = { workspace = true }
[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
wasm-bindgen-test = { workspace = true }
diff --git a/rust/noosphere/src/ffi/authority.rs b/rust/noosphere/src/ffi/authority.rs
index dca3400ad..7c1d0a14c 100644
--- a/rust/noosphere/src/ffi/authority.rs
+++ b/rust/noosphere/src/ffi/authority.rs
@@ -1,11 +1,13 @@
use anyhow::anyhow;
use cid::Cid;
+use noosphere_core::context::{
+ HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker,
+};
use noosphere_core::{
authority::Authorization,
data::{Did, Jwt, Link, Mnemonic},
error::NoosphereError,
};
-use noosphere_sphere::{HasSphereContext, SphereAuthorityRead, SphereAuthorityWrite, SphereWalker};
use safer_ffi::{char_p::InvalidNulTerminator, prelude::*};
use std::ffi::c_void;
diff --git a/rust/noosphere/src/ffi/context.rs b/rust/noosphere/src/ffi/context.rs
index 524d51d29..dd756dad9 100644
--- a/rust/noosphere/src/ffi/context.rs
+++ b/rust/noosphere/src/ffi/context.rs
@@ -15,7 +15,7 @@ use crate::{
platform::{PlatformSphereChannel, PlatformStorage},
};
-use noosphere_sphere::{
+use noosphere_core::context::{
AsyncFileBody, HasMutableSphereContext, HasSphereContext, SphereContentRead,
SphereContentWrite, SphereContext, SphereCursor, SphereFile, SphereReplicaRead, SphereWalker,
};
diff --git a/rust/noosphere/src/ffi/petname.rs b/rust/noosphere/src/ffi/petname.rs
index 5cc73ead1..a13c6001e 100644
--- a/rust/noosphere/src/ffi/petname.rs
+++ b/rust/noosphere/src/ffi/petname.rs
@@ -5,8 +5,8 @@ use std::{ffi::c_void, str::FromStr};
use anyhow::anyhow;
use cid::Cid;
+use noosphere_core::context::{SpherePetnameRead, SpherePetnameWrite, SphereWalker};
use noosphere_core::data::Did;
-use noosphere_sphere::{SpherePetnameRead, SpherePetnameWrite, SphereWalker};
use safer_ffi::{char_p::InvalidNulTerminator, prelude::*};
use crate::ffi::{NsError, TryOrInitialize};
diff --git a/rust/noosphere/src/ffi/sphere.rs b/rust/noosphere/src/ffi/sphere.rs
index b8f6a5dfa..32a3224a1 100644
--- a/rust/noosphere/src/ffi/sphere.rs
+++ b/rust/noosphere/src/ffi/sphere.rs
@@ -2,8 +2,8 @@ use std::ffi::c_void;
use anyhow::anyhow;
use cid::Cid;
+use noosphere_core::context::{HasSphereContext, SphereSync, SyncRecovery};
use noosphere_core::{authority::Authorization, data::Did};
-use noosphere_sphere::{HasSphereContext, SphereSync, SyncRecovery};
use safer_ffi::char_p::InvalidNulTerminator;
use safer_ffi::prelude::*;
diff --git a/rust/noosphere/src/noosphere.rs b/rust/noosphere/src/noosphere.rs
index 6541330f5..7759a984b 100644
--- a/rust/noosphere/src/noosphere.rs
+++ b/rust/noosphere/src/noosphere.rs
@@ -4,7 +4,7 @@ use anyhow::{anyhow, Result};
use noosphere_core::{authority::Authorization, data::Did};
use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
-use noosphere_sphere::{SphereContext, SphereCursor};
+use noosphere_core::context::{SphereContext, SphereCursor};
use tokio::sync::Mutex;
use url::Url;
diff --git a/rust/noosphere/src/platform.rs b/rust/noosphere/src/platform.rs
index 3b1fa0839..71353adfa 100644
--- a/rust/noosphere/src/platform.rs
+++ b/rust/noosphere/src/platform.rs
@@ -146,7 +146,7 @@ mod inner {
use std::sync::Arc;
pub use inner::*;
-use noosphere_sphere::{SphereContext, SphereCursor};
+use noosphere_core::context::{SphereContext, SphereCursor};
use tokio::sync::Mutex;
use crate::sphere::SphereChannel;
diff --git a/rust/noosphere/src/sphere/builder.rs b/rust/noosphere/src/sphere/builder.rs
index b76dea9a4..ce401d0ed 100644
--- a/rust/noosphere/src/sphere/builder.rs
+++ b/rust/noosphere/src/sphere/builder.rs
@@ -19,7 +19,7 @@ use noosphere_storage::{KeyValueStore, MemoryStore, SphereDb};
use ucan::crypto::KeyMaterial;
use url::Url;
-use noosphere_sphere::{
+use noosphere_core::context::{
metadata::{AUTHORIZATION, IDENTITY, USER_KEY_NAME},
SphereContext, SphereContextKey,
};
@@ -365,7 +365,7 @@ mod tests {
use wasm_bindgen_test::wasm_bindgen_test;
use crate::{key::KeyStorage, platform::make_temporary_platform_primitives};
- use noosphere_sphere::SphereContext;
+ use noosphere_core::context::SphereContext;
#[cfg(target_arch = "wasm32")]
wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
@@ -373,7 +373,7 @@ mod tests {
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
#[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
async fn it_can_create_a_sphere_and_later_open_it() {
- let (storage_path, key_storage, temporary_directories) =
+ let (storage_path, key_storage, _temporary_directories) =
make_temporary_platform_primitives().await.unwrap();
key_storage.create_key("foo").await.unwrap();
@@ -404,14 +404,12 @@ mod tests {
.into();
assert_eq!(&sphere_identity, context.identity());
-
- drop(temporary_directories);
}
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
#[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
async fn it_can_create_a_scoped_sphere_and_later_open_it() {
- let (storage_path, key_storage, temporary_directories) =
+ let (storage_path, key_storage, _temporary_directories) =
make_temporary_platform_primitives().await.unwrap();
key_storage.create_key("foo").await.unwrap();
@@ -444,14 +442,12 @@ mod tests {
.into();
assert_eq!(&sphere_identity, context.identity());
-
- drop(temporary_directories);
}
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
#[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
async fn it_can_initialize_a_sphere_to_sync_from_elsewhere() {
- let (storage_path, key_storage, temporary_directories) =
+ let (storage_path, key_storage, _temporary_directories) =
make_temporary_platform_primitives().await.unwrap();
key_storage.create_key("foo").await.unwrap();
@@ -469,7 +465,5 @@ mod tests {
let context: SphereContext<_> = artifacts.into();
assert_eq!(context.identity().as_str(), "did:key:foo");
-
- drop(temporary_directories);
}
}
diff --git a/rust/noosphere/src/sphere/channel.rs b/rust/noosphere/src/sphere/channel.rs
index 894ae4b34..64a2a57cf 100644
--- a/rust/noosphere/src/sphere/channel.rs
+++ b/rust/noosphere/src/sphere/channel.rs
@@ -1,6 +1,8 @@
use std::{marker::PhantomData, sync::Arc};
-use noosphere_sphere::{HasMutableSphereContext, HasSphereContext, SphereContext, SphereCursor};
+use noosphere_core::context::{
+ HasMutableSphereContext, HasSphereContext, SphereContext, SphereCursor,
+};
use noosphere_storage::Storage;
use tokio::sync::Mutex;
@@ -14,7 +16,7 @@ use tokio::sync::Mutex;
#[derive(Clone)]
pub struct SphereChannel<S, Ci, Cm>
where
- S: Storage,
+ S: Storage + 'static,
Ci: HasSphereContext<S>,
Cm: HasMutableSphereContext<S>,
{
@@ -25,7 +27,7 @@ where
impl<S, Ci, Cm> SphereChannel<S, Ci, Cm>
where
- S: Storage,
+ S: Storage + 'static,
Ci: HasSphereContext<S>,
Cm: HasMutableSphereContext<S>,
{
diff --git a/rust/noosphere/src/wasm/file.rs b/rust/noosphere/src/wasm/file.rs
index 9ff4dfcd3..1d49459bb 100644
--- a/rust/noosphere/src/wasm/file.rs
+++ b/rust/noosphere/src/wasm/file.rs
@@ -9,7 +9,7 @@ use tokio_stream::StreamExt;
use anyhow::{anyhow, Result};
use js_sys::{Function, Promise, Uint8Array};
-use noosphere_sphere::{
+use noosphere_core::context::{
AsyncFileBody, HasSphereContext, SphereContext, SphereCursor, SphereFile as SphereFileImpl,
};
use tokio::{io::AsyncReadExt, sync::Mutex};
diff --git a/rust/noosphere/src/wasm/fs.rs b/rust/noosphere/src/wasm/fs.rs
index 7bb5c53bc..2f13585bb 100644
--- a/rust/noosphere/src/wasm/fs.rs
+++ b/rust/noosphere/src/wasm/fs.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
use crate::{platform::PlatformStorage, wasm::SphereFile};
use js_sys::{Array, Function};
-use noosphere_sphere::{
+use noosphere_core::context::{
HasMutableSphereContext, SphereContentRead, SphereContentWrite, SphereContext, SphereCursor,
SphereWalker,
};
diff --git a/rust/noosphere/src/wasm/sphere.rs b/rust/noosphere/src/wasm/sphere.rs
index 04b3185c5..b05daa6be 100644
--- a/rust/noosphere/src/wasm/sphere.rs
+++ b/rust/noosphere/src/wasm/sphere.rs
@@ -2,7 +2,7 @@ use anyhow::Result;
use cid::Cid;
use crate::{platform::PlatformSphereChannel, wasm::SphereFs};
-use noosphere_sphere::SphereCursor;
+use noosphere_core::context::SphereCursor;
use wasm_bindgen::prelude::*;
#[wasm_bindgen]
diff --git a/rust/noosphere/tests/cli.rs b/rust/noosphere/tests/cli.rs
new file mode 100644
index 000000000..55b912c94
--- /dev/null
+++ b/rust/noosphere/tests/cli.rs
@@ -0,0 +1,155 @@
+#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))]
+
+//! Integration tests to demonstrate that the Noosphere CLI, aka "orb", works
+//! end-to-end in concert with the name system and backing block syndication
+
+use anyhow::Result;
+use noosphere_cli::helpers::CliSimulator;
+use noosphere_cli::paths::SPHERE_DIRECTORY;
+use noosphere_common::helpers::wait;
+use noosphere_core::tracing::initialize_tracing;
+use serde_json::Value;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn orb_status_errors_on_empty_directory() -> Result<()> {
+ initialize_tracing(None);
+ let client = CliSimulator::new()?;
+
+ assert!(client.orb(&["sphere", "status"]).await.is_err());
+
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn orb_sphere_create_initializes_a_sphere() -> Result<()> {
+ initialize_tracing(None);
+ let client = CliSimulator::new()?;
+
+ client.orb(&["key", "create", "foobar"]).await?;
+ client
+ .orb(&["sphere", "create", "--owner-key", "foobar"])
+ .await?;
+
+ assert!(client.sphere_directory().join(SPHERE_DIRECTORY).exists());
+
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn orb_can_enable_multiple_replicas_to_synchronize() -> Result<()> {
+ initialize_tracing(None);
+
+ let first_replica = CliSimulator::new()?;
+ let second_replica = CliSimulator::new()?;
+
+ first_replica.orb(&["key", "create", "foo"]).await?;
+ first_replica
+ .orb(&["sphere", "create", "--owner-key", "foo"])
+ .await?;
+
+ let client_sphere_id = first_replica
+ .orb_with_output(&["sphere", "status", "--id"])
+ .await?
+ .join("\n");
+
+ let gateway = CliSimulator::new()?;
+
+ gateway.orb(&["key", "create", "gateway"]).await?;
+
+ gateway
+ .orb(&["sphere", "create", "--owner-key", "gateway"])
+ .await?;
+
+ gateway
+ .orb(&["sphere", "config", "set", "counterpart", &client_sphere_id])
+ .await?;
+
+ let gateway_task = tokio::task::spawn(async move { gateway.orb(&["serve"]).await });
+
+ wait(1).await;
+
+ first_replica
+ .orb(&[
+ "sphere",
+ "config",
+ "set",
+ "gateway-url",
+ "http://127.0.0.1:4433",
+ ])
+ .await?;
+
+ second_replica.orb(&["key", "create", "bar"]).await?;
+ let second_replica_id = match serde_json::from_str(
+ &second_replica
+ .orb_with_output(&["key", "list", "--as-json"])
+ .await?
+ .join("\n"),
+ )? {
+ Value::Object(keys) => keys.get("bar").unwrap().as_str().unwrap().to_owned(),
+ _ => panic!(),
+ };
+
+ tokio::fs::write(
+ first_replica.sphere_directory().join("foo.subtext"),
+ "foobar",
+ )
+ .await?;
+
+ first_replica.orb(&["sphere", "save"]).await?;
+
+ first_replica
+ .orb(&["sphere", "auth", "add", &second_replica_id])
+ .await?;
+
+ let second_replica_auth = match serde_json::from_str(
+ &first_replica
+ .orb_with_output(&["sphere", "auth", "list", "--as-json"])
+ .await?
+ .join("\n"),
+ )? {
+ Value::Array(auths) => match auths
+ .iter()
+ .filter(|auth| {
+ auth.as_object()
+ .unwrap()
+ .get("name")
+ .unwrap()
+ .as_str()
+ .unwrap()
+ != "(OWNER)"
+ })
+ .take(1)
+ .next()
+ .unwrap()
+ {
+ Value::Object(auth) => auth.get("link").unwrap().as_str().unwrap().to_owned(),
+ _ => panic!(),
+ },
+ _ => panic!(),
+ };
+
+ first_replica.orb(&["sphere", "sync"]).await?;
+
+ second_replica
+ .orb(&[
+ "sphere",
+ "join",
+ "--authorization",
+ &second_replica_auth,
+ "--local-key",
+ "bar",
+ "--gateway-url",
+ "http://127.0.0.1:4433",
+ &client_sphere_id,
+ ])
+ .await?;
+
+ let foo_contents =
+ tokio::fs::read_to_string(second_replica.sphere_directory().join("foo.subtext")).await?;
+
+ assert_eq!(foo_contents.as_str(), "foobar");
+
+ gateway_task.abort();
+
+ Ok(())
+}
diff --git a/rust/noosphere-cli/tests/peer_to_peer.rs b/rust/noosphere/tests/distributed_basic.rs
similarity index 87%
rename from rust/noosphere-cli/tests/peer_to_peer.rs
rename to rust/noosphere/tests/distributed_basic.rs
index 991e6b1e1..83efc3f35 100644
--- a/rust/noosphere-cli/tests/peer_to_peer.rs
+++ b/rust/noosphere/tests/distributed_basic.rs
@@ -1,28 +1,32 @@
-#![cfg(all(feature = "test_kubo", not(target_arch = "wasm32")))]
+#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))]
+
+//! Integration tests that expect "full stack" Noosphere to be available, including
+//! name system and block syndication backend (e.g., IPFS Kubo). The tests in this
+//! module represent basic distributed system scenarios.
#[macro_use]
extern crate tracing;
-mod helpers;
use anyhow::Result;
-use helpers::{start_name_system_server, wait, SpherePair};
-use noosphere_core::data::{ContentType, Did, Link, MemoIpld};
-use noosphere_core::tracing::initialize_tracing;
-use noosphere_ns::{server::HttpClient, NameResolver};
-use noosphere_sphere::{
- HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite, SphereCursor,
- SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, SphereSync, SphereWalker,
- SyncRecovery,
+use noosphere_cli::helpers::{start_name_system_server, SpherePair};
+use noosphere_common::helpers::wait;
+use noosphere_core::{
+ context::{
+ HasMutableSphereContext, HasSphereContext, SphereContentRead, SphereContentWrite,
+ SphereCursor, SpherePetnameRead, SpherePetnameWrite, SphereReplicaRead, SphereSync,
+ SphereWalker, SyncRecovery,
+ },
+ data::{ContentType, Did, Link, MemoIpld},
+ stream::memo_history_stream,
+ tracing::initialize_tracing,
};
-use rand::Rng;
+use noosphere_ns::{server::HttpClient, NameResolver};
use std::collections::BTreeSet;
use std::sync::Arc;
use tokio::io::AsyncReadExt;
use tokio_stream::StreamExt;
use url::Url;
-use crate::helpers::TestEntropy;
-
#[tokio::test]
async fn gateway_publishes_and_resolves_petnames_configured_by_the_client() -> Result<()> {
initialize_tracing(None);
@@ -673,15 +677,12 @@ async fn local_lineage_remains_sparse_as_graph_changes_accrue_over_time() -> Res
Ok(())
}
-#[tokio::test(flavor = "multi_thread")]
-async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> {
+#[tokio::test]
+async fn all_of_client_history_is_made_manifest_on_the_gateway_after_sync() -> Result<()> {
initialize_tracing(None);
- let entropy = TestEntropy::default();
- let rng = entropy.to_rng();
-
- let ipfs_url = Url::parse("http://127.0.0.1:5001").unwrap();
- let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await.unwrap();
+ let ipfs_url = Url::parse("http://127.0.0.1:5001")?;
+ let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await?;
let mut pair_1 = SpherePair::new("ONE", &ipfs_url, &ns_url).await?;
let mut pair_2 = SpherePair::new("TWO", &ipfs_url, &ns_url).await?;
@@ -689,95 +690,59 @@ async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> {
pair_1.start_gateway().await?;
pair_2.start_gateway().await?;
- let peer_2_identity = pair_2.client.workspace.sphere_identity().await?;
- let pair_2_rng = rng.clone();
-
- pair_2
+ let _ = pair_2
.spawn(|mut ctx| async move {
- let mut rng = pair_2_rng.lock().await;
-
- // Long history, small-ish files
- for _ in 0..1000 {
- let random_index = rng.gen_range(0..100);
- let mut random_bytes = Vec::from(rng.gen::<[u8; 32]>());
- let slug = format!("slug{}", random_index);
-
- let next_bytes = if let Some(mut file) = ctx.read(&slug).await? {
- let mut file_bytes = Vec::new();
- file.contents.read_to_end(&mut file_bytes).await?;
- file_bytes.append(&mut random_bytes);
- file_bytes
- } else {
- random_bytes
- };
-
- ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None)
- .await?;
- ctx.save(None).await?;
- }
-
+ ctx.write("foo", &ContentType::Text, "bar".as_bytes(), None)
+ .await?;
+ ctx.save(None).await?;
Ok(ctx.sync(SyncRecovery::Retry(3)).await?)
})
.await?;
- let pair_1_rng = rng.clone();
+ let sphere_2_identity = pair_2.client.identity.clone();
- pair_1
- .spawn(|mut ctx| async move {
- let mut rng = pair_1_rng.lock().await;
-
- // Modest history, large-ish files
- for _ in 0..100 {
- let mut random_bytes = (0..1000).fold(Vec::new(), |mut bytes, _| {
- bytes.append(&mut Vec::from(rng.gen::<[u8; 32]>()));
- bytes
- });
- let random_index = rng.gen_range(0..10);
- let slug = format!("slug{}", random_index);
-
- let next_bytes = if let Some(mut file) = ctx.read(&slug).await? {
- let mut file_bytes = Vec::new();
- file.contents.read_to_end(&mut file_bytes).await?;
- file_bytes.append(&mut random_bytes);
- file_bytes
- } else {
- random_bytes
- };
-
- ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None)
+ let final_client_version = pair_1
+ .spawn(move |mut ctx| async move {
+ for value in ["one", "two", "three"] {
+ ctx.write(value, &ContentType::Text, value.as_bytes(), None)
.await?;
-
ctx.save(None).await?;
}
ctx.sync(SyncRecovery::Retry(3)).await?;
- ctx.set_petname("peer2", Some(peer_2_identity)).await?;
+ ctx.set_petname("two", Some(sphere_2_identity)).await?;
ctx.save(None).await?;
ctx.sync(SyncRecovery::Retry(3)).await?;
- // TODO(#606): Implement this part of the test when we "fix" latency asymmetry between
- // name system and syndication workers. We should be able to test traversing to a peer
- // after a huge update as been added to the name system.
- /*
- wait(1).await;
+ for value in ["one", "two", "three"] {
+ ctx.set_petname(value, Some(Did(format!("did:key:{}", value))))
+ .await?;
+ ctx.save(None).await?;
+ }
ctx.sync(SyncRecovery::Retry(3)).await?;
wait(1).await;
- let cursor = SphereCursor::latest(ctx);
- let _peer2_ctx = cursor
- .traverse_by_petnames(&["peer2".into()])
- .await?
- .unwrap();
- */
- Ok(())
+ Ok(ctx.sync(SyncRecovery::Retry(3)).await?)
})
.await?;
+ // Stream all of the blocks of client history as represented in gateway's storage
+ let block_stream = memo_history_stream(
+ pair_1.gateway.workspace.db().await?,
+ &final_client_version,
+ None,
+ true,
+ );
+
+ tokio::pin!(block_stream);
+
+ while let Some(_) = block_stream.try_next().await? {}
+
ns_task.abort();
Ok(())
diff --git a/rust/noosphere-cli/tests/cli.rs b/rust/noosphere/tests/distributed_stress.rs
similarity index 67%
rename from rust/noosphere-cli/tests/cli.rs
rename to rust/noosphere/tests/distributed_stress.rs
index 426c8972f..0eb4d3e13 100644
--- a/rust/noosphere-cli/tests/cli.rs
+++ b/rust/noosphere/tests/distributed_stress.rs
@@ -1,168 +1,146 @@
-#![cfg(not(target_arch = "wasm32"))]
+#![cfg(all(feature = "test-kubo", not(target_arch = "wasm32")))]
-mod helpers;
+//! Integration tests that expect "full stack" Noosphere to be available, including
+//! name system and block syndication backend (e.g., IPFS Kubo). The tests in this
+//! module represent sophisticated, complicated, nuanced or high-latency scenarios.
-use anyhow::Result;
-use helpers::CliSimulator;
-use noosphere_cli::paths::SPHERE_DIRECTORY;
-use noosphere_core::tracing::initialize_tracing;
-use serde_json::Value;
-
-use crate::helpers::wait;
-
-#[tokio::test(flavor = "multi_thread")]
-async fn orb_status_errors_on_empty_directory() -> Result<()> {
- initialize_tracing(None);
- let client = CliSimulator::new()?;
-
- assert!(client.orb(&["sphere", "status"]).await.is_err());
-
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn orb_sphere_create_initializes_a_sphere() -> Result<()> {
- initialize_tracing(None);
- let client = CliSimulator::new()?;
-
- client.orb(&["key", "create", "foobar"]).await?;
- client
- .orb(&["sphere", "create", "--owner-key", "foobar"])
- .await?;
+mod latency {
+ use anyhow::Result;
+ use noosphere_cli::helpers::{start_name_system_server, SpherePair};
+ use noosphere_common::helpers::TestEntropy;
+ use noosphere_core::{
+ context::{
+ HasMutableSphereContext, SphereContentRead, SphereContentWrite, SpherePetnameWrite,
+ SphereSync, SyncRecovery,
+ },
+ data::ContentType,
+ tracing::initialize_tracing,
+ };
+ use rand::Rng;
+ use tokio::io::AsyncReadExt;
+ use url::Url;
- assert!(client.sphere_directory().join(SPHERE_DIRECTORY).exists());
+ #[tokio::test(flavor = "multi_thread")]
+ async fn clients_can_sync_when_there_is_a_lot_of_content() -> Result<()> {
+ initialize_tracing(None);
- Ok(())
-}
+ let entropy = TestEntropy::default();
+ let rng = entropy.to_rng();
-#[tokio::test(flavor = "multi_thread")]
-async fn orb_can_enable_multiple_replicas_to_synchronize() -> Result<()> {
- initialize_tracing(None);
+ let ipfs_url = Url::parse("http://127.0.0.1:5001").unwrap();
+ let (ns_url, ns_task) = start_name_system_server(&ipfs_url).await.unwrap();
- let first_replica = CliSimulator::new()?;
- let second_replica = CliSimulator::new()?;
+ let mut pair_1 = SpherePair::new("ONE", &ipfs_url, &ns_url).await?;
+ let mut pair_2 = SpherePair::new("TWO", &ipfs_url, &ns_url).await?;
- first_replica.orb(&["key", "create", "foo"]).await?;
- first_replica
- .orb(&["sphere", "create", "--owner-key", "foo"])
- .await?;
+ pair_1.start_gateway().await?;
+ pair_2.start_gateway().await?;
- let client_sphere_id = first_replica
- .orb_with_output(&["sphere", "status", "--id"])
- .await?
- .join("\n");
+ let peer_2_identity = pair_2.client.workspace.sphere_identity().await?;
+ let pair_2_rng = rng.clone();
- let gateway = CliSimulator::new()?;
+ pair_2
+ .spawn(|mut ctx| async move {
+ let mut rng = pair_2_rng.lock().await;
+
+ // Long history, small-ish files
+ for _ in 0..1000 {
+ let random_index = rng.gen_range(0..100);
+ let mut random_bytes = Vec::from(rng.gen::<[u8; 32]>());
+ let slug = format!("slug{}", random_index);
+
+ let next_bytes = if let Some(mut file) = ctx.read(&slug).await? {
+ let mut file_bytes = Vec::new();
+ file.contents.read_to_end(&mut file_bytes).await?;
+ file_bytes.append(&mut random_bytes);
+ file_bytes
+ } else {
+ random_bytes
+ };
+
+ ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None)
+ .await?;
+ ctx.save(None).await?;
+ }
- gateway.orb(&["key", "create", "gateway"]).await?;
+ Ok(ctx.sync(SyncRecovery::Retry(3)).await?)
+ })
+ .await?;
- gateway
- .orb(&["sphere", "create", "--owner-key", "gateway"])
- .await?;
+ let pair_1_rng = rng.clone();
+
+ pair_1
+ .spawn(|mut ctx| async move {
+ let mut rng = pair_1_rng.lock().await;
+
+ // Modest history, large-ish files
+ for _ in 0..100 {
+ let mut random_bytes = (0..1000).fold(Vec::new(), |mut bytes, _| {
+ bytes.append(&mut Vec::from(rng.gen::<[u8; 32]>()));
+ bytes
+ });
+ let random_index = rng.gen_range(0..10);
+ let slug = format!("slug{}", random_index);
+
+ let next_bytes = if let Some(mut file) = ctx.read(&slug).await? {
+ let mut file_bytes = Vec::new();
+ file.contents.read_to_end(&mut file_bytes).await?;
+ file_bytes.append(&mut random_bytes);
+ file_bytes
+ } else {
+ random_bytes
+ };
+
+ ctx.write(&slug, &ContentType::Bytes, next_bytes.as_ref(), None)
+ .await?;
- gateway
- .orb(&["sphere", "config", "set", "counterpart", &client_sphere_id])
- .await?;
+ ctx.save(None).await?;
+ }
- let gateway_task = tokio::task::spawn(async move { gateway.orb(&["serve"]).await });
+ ctx.sync(SyncRecovery::Retry(3)).await?;
- wait(1).await;
+ ctx.set_petname("peer2", Some(peer_2_identity)).await?;
- first_replica
- .orb(&[
- "sphere",
- "config",
- "set",
- "gateway-url",
- "http://127.0.0.1:4433",
- ])
- .await?;
+ ctx.save(None).await?;
- second_replica.orb(&["key", "create", "bar"]).await?;
- let second_replica_id = match serde_json::from_str(
- &second_replica
- .orb_with_output(&["key", "list", "--as-json"])
- .await?
- .join("\n"),
- )? {
- Value::Object(keys) => keys.get("bar").unwrap().as_str().unwrap().to_owned(),
- _ => panic!(),
- };
+ ctx.sync(SyncRecovery::Retry(3)).await?;
- tokio::fs::write(
- first_replica.sphere_directory().join("foo.subtext"),
- "foobar",
- )
- .await?;
+ // TODO(#606): Implement this part of the test when we "fix" latency asymmetry between
+ // name system and syndication workers. We should be able to test traversing to a peer
+ // after a huge update as been added to the name system.
+ // wait(1).await;
- first_replica.orb(&["sphere", "save"]).await?;
+ // ctx.sync(SyncRecovery::Retry(3)).await?;
- first_replica
- .orb(&["sphere", "auth", "add", &second_replica_id])
- .await?;
+ // wait(1).await;
- let second_replica_auth = match serde_json::from_str(
- &first_replica
- .orb_with_output(&["sphere", "auth", "list", "--as-json"])
- .await?
- .join("\n"),
- )? {
- Value::Array(auths) => match auths
- .iter()
- .filter(|auth| {
- auth.as_object()
- .unwrap()
- .get("name")
- .unwrap()
- .as_str()
- .unwrap()
- != "(OWNER)"
+ // let cursor = SphereCursor::latest(ctx);
+ // let _peer2_ctx = cursor
+ // .traverse_by_petnames(&["peer2".into()])
+ // .await?
+ // .unwrap();
+ Ok(())
})
- .take(1)
- .next()
- .unwrap()
- {
- Value::Object(auth) => auth.get("link").unwrap().as_str().unwrap().to_owned(),
- _ => panic!(),
- },
- _ => panic!(),
- };
-
- first_replica.orb(&["sphere", "sync"]).await?;
-
- second_replica
- .orb(&[
- "sphere",
- "join",
- "--authorization",
- &second_replica_auth,
- "--local-key",
- "bar",
- "--gateway-url",
- "http://127.0.0.1:4433",
- &client_sphere_id,
- ])
- .await?;
-
- let foo_contents =
- tokio::fs::read_to_string(second_replica.sphere_directory().join("foo.subtext")).await?;
-
- assert_eq!(foo_contents.as_str(), "foobar");
+ .await?;
- gateway_task.abort();
+ ns_task.abort();
- Ok(())
+ Ok(())
+ }
}
-#[cfg(feature = "test_kubo")]
mod multiplayer {
- use crate::helpers::{start_name_system_server, wait, CliSimulator, SpherePair};
use anyhow::Result;
- use noosphere_core::{data::Did, tracing::initialize_tracing};
- use noosphere_sphere::{
- HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SpherePetnameWrite,
- SphereSync, SyncRecovery,
+ use noosphere_cli::helpers::{start_name_system_server, CliSimulator, SpherePair};
+ use noosphere_common::helpers::wait;
+ use noosphere_core::{
+ context::{
+ HasMutableSphereContext, SphereAuthorityWrite, SphereContentWrite, SpherePetnameWrite,
+ SphereSync, SyncRecovery,
+ },
+ data::Did,
+ tracing::initialize_tracing,
};
use serde_json::Value;
use url::Url;
diff --git a/rust/noosphere-cli/tests/gateway.rs b/rust/noosphere/tests/gateway.rs
similarity index 87%
rename from rust/noosphere-cli/tests/gateway.rs
rename to rust/noosphere/tests/gateway.rs
index c594acb1c..637382774 100644
--- a/rust/noosphere-cli/tests/gateway.rs
+++ b/rust/noosphere/tests/gateway.rs
@@ -1,8 +1,12 @@
#![cfg(not(target_arch = "wasm32"))]
+//! Integration tests that span the distance between a client and a gateway;
+//! tests in this module should be able to run without an available backing
+//! IPFS-like block syndication layer.
+
use anyhow::Result;
use noosphere::key::KeyStorage;
-use noosphere_sphere::{
+use noosphere_core::context::{
HasMutableSphereContext, HasSphereContext, SphereAuthorityWrite, SphereContentRead,
SphereContentWrite, SphereCursor, SphereSync, SyncRecovery,
};
@@ -12,23 +16,21 @@ use tokio::io::AsyncReadExt;
use tokio_stream::StreamExt;
use url::Url;
-use noosphere_api::route::Route;
+use noosphere_core::api::v0alpha1;
use noosphere_core::data::{ContentType, Did};
use ucan::crypto::KeyMaterial;
-use noosphere_cli::commands::{
- key::key_create,
- sphere::{sphere_create, sphere_join},
+use noosphere_cli::{
+ commands::{
+ key::key_create,
+ sphere::{sphere_create, sphere_join},
+ },
+ helpers::{temporary_workspace, SpherePair},
};
use noosphere_core::tracing::initialize_tracing;
use noosphere_gateway::{start_gateway, GatewayScope};
-mod helpers;
-use helpers::temporary_workspace;
-
-use crate::helpers::SpherePair;
-
#[tokio::test]
async fn gateway_tells_you_its_identity() -> Result<()> {
initialize_tracing(None);
@@ -81,7 +83,7 @@ async fn gateway_tells_you_its_identity() -> Result<()> {
gateway_address.port()
))
.unwrap();
- url.set_path(&Route::Did.to_string());
+ url.set_path(&v0alpha1::Route::Did.to_string());
let did_response = client.get(url).send().await.unwrap().text().await.unwrap();
@@ -388,6 +390,60 @@ async fn gateway_receives_sphere_revisions_from_a_client() -> Result<()> {
Ok(())
}
+// #[tokio::test]
+// async fn all_of_client_history_is_made_manifest_on_the_gateway_after_sync() -> Result<()> {
+// initialize_tracing(None);
+
+// let mut sphere_pair = SpherePair::new(
+// "one",
+// &Url::parse("http://127.0.0.1:5001")?,
+// &Url::parse("http://127.0.0.1:6667")?,
+// )
+// .await?;
+
+// sphere_pair.start_gateway().await?;
+
+// let final_client_version = sphere_pair
+// .spawn(move |mut client_sphere_context| async move {
+// for value in ["one", "two", "three"] {
+// client_sphere_context
+// .write(value, &ContentType::Text, value.as_bytes(), None)
+// .await?;
+// client_sphere_context.save(None).await?;
+// }
+
+// client_sphere_context.sync(SyncRecovery::None).await?;
+
+// for value in ["one", "two", "three"] {
+// client_sphere_context
+// .set_petname(value, Some(Did(format!("did:key:{}", value))))
+// .await?;
+// client_sphere_context.save(None).await?;
+// }
+
+// client_sphere_context.sync(SyncRecovery::None).await?;
+
+// wait(1).await;
+
+// Ok(client_sphere_context.sync(SyncRecovery::None).await?)
+// })
+// .await?;
+
+// // Stream all of the blocks of client history as represented in gateway's storage
+// let block_stream = memo_history_stream(
+// sphere_pair.gateway.workspace.db().await?,
+// &final_client_version,
+// None,
+// true,
+// );
+
+// tokio::pin!(block_stream);
+
+// while let Some(_) = block_stream.try_next().await? {}
+
+// Ok(())
+// }
+
#[tokio::test]
async fn gateway_can_sync_an_authorized_sphere_across_multiple_replicas() -> Result<()> {
initialize_tracing(None);
diff --git a/rust/noosphere/tests/integration.rs b/rust/noosphere/tests/sphere_channel.rs
similarity index 97%
rename from rust/noosphere/tests/integration.rs
rename to rust/noosphere/tests/sphere_channel.rs
index ab7d3d38e..6631e601b 100644
--- a/rust/noosphere/tests/integration.rs
+++ b/rust/noosphere/tests/sphere_channel.rs
@@ -1,4 +1,5 @@
-#![cfg(test)]
+//! Integration tests to demonstrate that a [SphereChannel] works as expected in
+//! end-to-end flows.
use std::pin::Pin;
@@ -10,8 +11,8 @@ use std::time::Duration;
use anyhow::Result;
use async_stream::try_stream;
use bytes::Bytes;
+use noosphere_core::context::{HasMutableSphereContext, SphereContentRead, SphereContentWrite};
use noosphere_core::{data::ContentType, tracing::initialize_tracing};
-use noosphere_sphere::{HasMutableSphereContext, SphereContentRead, SphereContentWrite};
use tokio::io::AsyncReadExt;
use tokio_stream::Stream;