Merge branch 'main' into provider-events-refactor

Author: rklaehn

Cargo.toml

@@ -21,7 +21,6 @@ quinn = { package = "iroh-quinn", version = "0.14.0" }
 n0-future = "0.2.0"
 n0-snafu = "0.2.0"
 range-collections = { version = "0.4.6", features = ["serde"] }
-redb = { version = "=2.4" }
 smallvec = { version = "1", features = ["serde", "const_new"] }
 snafu = "0.8.5"
 tokio = { version = "1.43.0", features = ["full"] }
@@ -41,9 +40,10 @@ iroh = "0.91.1"
 self_cell = "1.1.0"
 genawaiter = { version = "0.99.1", features = ["futures03"] }
 iroh-base = "0.91.1"
-reflink-copy = "0.1.24"
 irpc = { version = "0.7.0", features = ["rpc", "quinn_endpoint_setup", "spans", "stream", "derive"], default-features = false }
 iroh-metrics = { version = "0.35" }
+redb = { version = "=2.4", optional = true }
+reflink-copy = { version = "0.1.24", optional = true }
 
 [dev-dependencies]
 clap = { version = "4.5.31", features = ["derive"] }
@@ -64,7 +64,8 @@ iroh = { version = "0.91.1", features = ["discovery-local-network"]}
 [features]
 hide-proto-docs = []
 metrics = []
-default = ["hide-proto-docs"]
+default = ["hide-proto-docs", "fs-store"]
+fs-store = ["dep:redb", "dep:reflink-copy"]
 
 [patch.crates-io]
 iroh = { git = "https://github.com/n0-computer/iroh", branch = "main" }

src/api/blobs.rs

@@ -144,13 +144,15 @@ impl Blobs {
     /// clears the protections before.
     ///
     /// Users should rely only on garbage collection for blob deletion.
+    #[cfg(feature = "fs-store")]
     pub(crate) async fn delete_with_opts(&self, options: DeleteOptions) -> RequestResult<()> {
         trace!("{options:?}");
         self.client.rpc(options).await??;
         Ok(())
     }
 
     /// See [`Self::delete_with_opts`].
+    #[cfg(feature = "fs-store")]
     pub(crate) async fn delete(
         &self,
         hashes: impl IntoIterator<Item = impl Into<Hash>>,
@@ -510,6 +512,7 @@ impl Blobs {
         }
     }
 
+    #[allow(dead_code)]
     pub(crate) async fn clear_protected(&self) -> RequestResult<()> {
         let msg = ClearProtectedRequest;
         self.client.rpc(msg).await??;

src/api/blobs/reader.rs

@@ -214,6 +214,7 @@ impl tokio::io::AsyncSeek for BlobReader {
 }
 
 #[cfg(test)]
+#[cfg(feature = "fs-store")]
 mod tests {
     use bao_tree::ChunkRanges;
     use testresult::TestResult;

src/api/downloader.rs

@@ -521,6 +521,7 @@ impl ContentDiscovery for Shuffled {
 }
 
 #[cfg(test)]
+#[cfg(feature = "fs-store")]
 mod tests {
     use std::ops::Deref;
 

src/api/proto.rs

@@ -40,6 +40,7 @@ pub use bitfield::Bitfield;
 
 use crate::{store::util::Tag, util::temp_tag::TempTag, BlobFormat, Hash, HashAndFormat};
 
+#[allow(dead_code)]
 pub(crate) trait HashSpecific {
     fn hash(&self) -> Hash;
 

src/api/proto/bitfield.rs

@@ -70,6 +70,7 @@ impl<'de> Deserialize<'de> for Bitfield {
 }
 
 impl Bitfield {
+    #[cfg(feature = "fs-store")]
     pub(crate) fn new_unchecked(ranges: ChunkRanges, size: u64) -> Self {
         Self { ranges, size }
     }

src/api/remote.rs

@@ -1063,6 +1063,7 @@ where
 }
 
 #[cfg(test)]
+#[cfg(feature = "fs-store")]
 mod tests {
     use bao_tree::{ChunkNum, ChunkRanges};
     use testresult::TestResult;

src/hash.rs

@@ -283,7 +283,7 @@ impl From<Hash> for HashAndFormat {
     }
 }
 
-// #[cfg(feature = "redb")]
+#[cfg(feature = "fs-store")]
 mod redb_support {
     use postcard::experimental::max_size::MaxSize;
     use redb::{Key as RedbKey, Value as RedbValue};
@@ -493,7 +493,7 @@ mod tests {
         assert_eq_hex!(serialized, expected);
     }
 
-    // #[cfg(feature = "redb")]
+    #[cfg(feature = "fs-store")]
     #[test]
     fn hash_redb() {
         use redb::Value as RedbValue;
@@ -518,7 +518,7 @@ mod tests {
         assert_eq_hex!(serialized, expected);
     }
 
-    // #[cfg(feature = "redb")]
+    #[cfg(feature = "fs-store")]
     #[test]
     fn hash_and_format_redb() {
         use redb::Value as RedbValue;

src/lib.rs

@@ -24,6 +24,11 @@
 //! The [downloader](api::downloader) module provides a component to download blobs from
 //! multiple sources and store them in a store.
 //!
+//! # Features:
+//!
+//! - `fs-store`: Enables the filesystem based store implementation. This comes with a few additional dependencies such as `redb` and `reflink-copy`.
+//! - `metrics`: Enables prometheus metrics for stores and the protocol.
+//!
 //! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf
 //! [iroh]: https://docs.rs/iroh
 mod hash;
@@ -46,6 +51,7 @@ pub mod test;
 pub mod util;
 
 #[cfg(test)]
+#[cfg(feature = "fs-store")]
 mod tests;
 
 pub use protocol::ALPN;

src/store/fs.rs

@@ -121,12 +121,13 @@ use crate::{
             util::entity_manager::{self, ActiveEntityState},
         },
         util::{BaoTreeSender, FixedSize, MemOrFile, ValueOrPoisioned},
-        Hash, IROH_BLOCK_SIZE,
+        IROH_BLOCK_SIZE,
     },
     util::{
         channel::oneshot,
         temp_tag::{TagDrop, TempTag, TempTagScope, TempTags},
     },
+    Hash,
 };
 mod bao_file;
 use bao_file::BaoFileHandle;
@@ -142,11 +143,13 @@ use options::Options;
 use tracing::Instrument;
 mod gc;
 
-use super::HashAndFormat;
-use crate::api::{
-    self,
-    blobs::{AddProgressItem, ExportMode, ExportProgressItem},
-    Store,
+use crate::{
+    api::{
+        self,
+        blobs::{AddProgressItem, ExportMode, ExportProgressItem},
+        Store,
+    },
+    HashAndFormat,
 };
 
 /// Create a 16 byte unique ID.
@@ -1477,7 +1480,7 @@ pub mod tests {
         api::blobs::Bitfield,
         store::{
             util::{read_checksummed, SliceInfoExt, Tag},
-            HashAndFormat, IROH_BLOCK_SIZE,
+            IROH_BLOCK_SIZE,
         },
     };