[storage & runtime] Switch to page-level checksums instead of record-level checksums

consensus/fuzz/src/lib.rs

@@ -28,14 +28,20 @@ use commonware_cryptography::{
 };
 use commonware_p2p::simulated::{Config as NetworkConfig, Link, Network};
 use commonware_runtime::{buffer::PoolRef, deterministic, Clock, Metrics, Runner, Spawner};
-use commonware_utils::{max_faults, NZUsize};
+use commonware_utils::{max_faults, NZUsize, NZU16};
 use futures::{channel::mpsc::Receiver, future::join_all, StreamExt};
 use rand::{rngs::StdRng, RngCore, SeedableRng};
-use std::{cell::RefCell, num::NonZeroUsize, panic, sync::Arc, time::Duration};
+use std::{
+    cell::RefCell,
+    num::{NonZeroU16, NonZeroUsize},
+    panic,
+    sync::Arc,
+    time::Duration,
+};
 
 pub const EPOCH: u64 = 333;
 
-const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
 const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
 const MIN_REQUIRED_CONTAINERS: u64 = 10;
 const MAX_REQUIRED_CONTAINERS: u64 = 50;

consensus/src/aggregation/mod.rs

@@ -103,19 +103,19 @@ mod tests {
         deterministic::{self, Context},
         Clock, Metrics, Quota, Runner, Spawner,
     };
-    use commonware_utils::{test_rng, NZUsize, NonZeroDuration};
+    use commonware_utils::{test_rng, NZUsize, NonZeroDuration, NZU16};
     use futures::{channel::oneshot, future::join_all};
     use rand::{rngs::StdRng, Rng};
     use std::{
         collections::BTreeMap,
-        num::{NonZeroU32, NonZeroUsize},
+        num::{NonZeroU16, NonZeroU32, NonZeroUsize},
         time::Duration,
     };
     use tracing::debug;
 
     type Registrations<P> = BTreeMap<P, (Sender<P, deterministic::Context>, Receiver<P>)>;
 
-    const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+    const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
     const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
     const TEST_QUOTA: Quota = Quota::per_second(NonZeroU32::MAX);
     const TEST_NAMESPACE: &[u8] = b"my testing namespace";

consensus/src/marshal/mod.rs

@@ -130,15 +130,15 @@ mod tests {
     };
     use commonware_runtime::{buffer::PoolRef, deterministic, Clock, Metrics, Quota, Runner};
     use commonware_storage::archive::immutable;
-    use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU64};
+    use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU16, NZU64};
     use futures::StreamExt;
     use rand::{
         seq::{IteratorRandom, SliceRandom},
         Rng,
     };
     use std::{
         collections::BTreeMap,
-        num::{NonZeroU32, NonZeroU64, NonZeroUsize},
+        num::{NonZeroU16, NonZeroU32, NonZeroU64, NonZeroUsize},
         time::{Duration, Instant},
     };
     use tracing::info;
@@ -150,7 +150,7 @@ mod tests {
     type S = bls12381_threshold::Scheme<K, V>;
     type P = ConstantProvider<S, Epoch>;
 
-    const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+    const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
     const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
     const NAMESPACE: &[u8] = b"test";
     const NUM_VALIDATORS: u32 = 4;

consensus/src/ordered_broadcast/mod.rs

@@ -97,16 +97,16 @@ mod tests {
         deterministic::{self, Context},
         Clock, Metrics, Quota, Runner, Spawner,
     };
-    use commonware_utils::NZUsize;
+    use commonware_utils::{NZUsize, NZU16};
     use futures::{channel::oneshot, future::join_all};
     use std::{
         collections::{BTreeMap, HashMap},
-        num::{NonZeroU32, NonZeroUsize},
+        num::{NonZeroU16, NonZeroU32, NonZeroUsize},
         time::Duration,
     };
     use tracing::debug;
 
-    const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+    const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
     const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
     const TEST_QUOTA: Quota = Quota::per_second(NonZeroU32::MAX);
     const TEST_NAMESPACE: &[u8] = b"ordered_broadcast_test";

consensus/src/simplex/actors/voter/mod.rs

@@ -71,15 +71,15 @@ mod tests {
     use commonware_macros::{select, test_traced};
     use commonware_p2p::simulated::{Config as NConfig, Network};
     use commonware_runtime::{deterministic, Clock, Metrics, Quota, Runner};
-    use commonware_utils::{quorum, NZUsize};
+    use commonware_utils::{quorum, NZUsize, NZU16};
     use futures::{channel::mpsc, FutureExt, StreamExt};
     use std::{
-        num::NonZeroU32,
+        num::{NonZeroU16, NonZeroU32},
         sync::{Arc, Mutex},
         time::Duration,
     };
 
-    const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+    const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
     const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
     const TEST_QUOTA: Quota = Quota::per_second(NonZeroU32::MAX);
 

consensus/src/simplex/mod.rs

@@ -320,20 +320,20 @@ mod tests {
     use commonware_runtime::{
         buffer::PoolRef, deterministic, Clock, Metrics, Quota, Runner, Spawner,
     };
-    use commonware_utils::{max_faults, quorum, test_rng, NZUsize};
+    use commonware_utils::{max_faults, quorum, test_rng, NZUsize, NZU16};
     use engine::Engine;
     use futures::{future::join_all, StreamExt};
     use rand::{rngs::StdRng, Rng as _};
     use std::{
         collections::{BTreeMap, HashMap},
-        num::{NonZeroU32, NonZeroUsize},
+        num::{NonZeroU16, NonZeroU32, NonZeroUsize},
         sync::{Arc, Mutex},
         time::Duration,
     };
     use tracing::{debug, info, warn};
     use types::Activity;
 
-    const PAGE_SIZE: NonZeroUsize = NZUsize!(1024);
+    const PAGE_SIZE: NonZeroU16 = NZU16!(1024);
     const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10);
     const TEST_QUOTA: Quota = Quota::per_second(NonZeroU32::MAX);
 

examples/bridge/src/bin/validator.rs

@@ -20,7 +20,7 @@ use commonware_runtime::{
     buffer::PoolRef, tokio, Metrics, Network, Quota, RayonPoolSpawner, Runner,
 };
 use commonware_stream::{dial, Config as StreamConfig};
-use commonware_utils::{from_hex, ordered::Set, union, NZUsize, TryCollect, NZU32};
+use commonware_utils::{from_hex, ordered::Set, union, NZUsize, TryCollect, NZU16, NZU32};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     str::FromStr,
@@ -264,7 +264,7 @@ fn main() {
                 activity_timeout: ViewDelta::new(10),
                 skip_timeout: ViewDelta::new(5),
                 fetch_concurrent: 32,
-                buffer_pool: PoolRef::new(NZUsize!(16_384), NZUsize!(10_000)),
+                buffer_pool: PoolRef::new(NZU16!(16_384), NZUsize!(10_000)),
             },
         );
 

examples/log/src/main.rs

@@ -55,7 +55,7 @@ use commonware_consensus::{
 use commonware_cryptography::{ed25519, Sha256, Signer as _};
 use commonware_p2p::{authenticated::discovery, Manager};
 use commonware_runtime::{buffer::PoolRef, tokio, Metrics, Quota, Runner};
-use commonware_utils::{ordered::Set, union, NZUsize, TryCollect, NZU32};
+use commonware_utils::{ordered::Set, union, NZUsize, TryCollect, NZU16, NZU32};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     str::FromStr,
@@ -223,7 +223,7 @@ fn main() {
             activity_timeout: ViewDelta::new(10),
             skip_timeout: ViewDelta::new(5),
             fetch_concurrent: 32,
-            buffer_pool: PoolRef::new(NZUsize!(16_384), NZUsize!(10_000)),
+            buffer_pool: PoolRef::new(NZU16!(16_384), NZUsize!(10_000)),
         };
         let engine = simplex::Engine::new(context.with_label("engine"), cfg);
 

examples/reshare/src/dkg/state.rs

@@ -30,15 +30,15 @@ use commonware_storage::journal::{
     contiguous::variable::{Config as CVConfig, Journal as CVJournal},
     segmented::variable::{Config as SVConfig, Journal as SVJournal},
 };
-use commonware_utils::{NZUsize, NZU64};
+use commonware_utils::{NZUsize, NZU16, NZU64};
 use futures::StreamExt;
 use std::{
     collections::BTreeMap,
-    num::{NonZeroU32, NonZeroUsize},
+    num::{NonZeroU16, NonZeroU32, NonZeroUsize},
 };
 use tracing::debug;
 
-const PAGE_SIZE: NonZeroUsize = NZUsize!(1 << 12);
+const PAGE_SIZE: NonZeroU16 = NZU16!(1 << 12);
 const POOL_CAPACITY: NonZeroUsize = NZUsize!(1 << 20);
 const WRITE_BUFFER: NonZeroUsize = NZUsize!(1 << 12);
 const READ_BUFFER: NonZeroUsize = NZUsize!(1 << 20);

examples/reshare/src/engine.rs

@@ -27,10 +27,14 @@ use commonware_runtime::{
     buffer::PoolRef, spawn_cell, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage,
 };
 use commonware_storage::archive::immutable;
-use commonware_utils::{ordered::Set, union, NZUsize, NZU32, NZU64};
+use commonware_utils::{ordered::Set, union, NZUsize, NZU16, NZU32, NZU64};
 use futures::{channel::mpsc, future::try_join_all};
 use rand_core::CryptoRngCore;
-use std::{marker::PhantomData, num::NonZero, time::Instant};
+use std::{
+    marker::PhantomData,
+    num::{NonZero, NonZeroU16},
+    time::Instant,
+};
 use tracing::{error, info, warn};
 
 const MAILBOX_SIZE: usize = 10;
@@ -45,7 +49,7 @@ const FREEZER_VALUE_TARGET_SIZE: u64 = 1024 * 1024 * 1024; // 1GB
 const FREEZER_VALUE_COMPRESSION: Option<u8> = Some(3);
 const REPLAY_BUFFER: NonZero<usize> = NZUsize!(8 * 1024 * 1024); // 8MB
 const WRITE_BUFFER: NonZero<usize> = NZUsize!(1024 * 1024); // 1MB
-const BUFFER_POOL_PAGE_SIZE: NonZero<usize> = NZUsize!(4_096); // 4KB
+const BUFFER_POOL_PAGE_SIZE: NonZeroU16 = NZU16!(4_096); // 4KB
 const BUFFER_POOL_CAPACITY: NonZero<usize> = NZUsize!(8_192); // 32MB
 const MAX_REPAIR: NonZero<usize> = NZUsize!(50);
 

examples/reshare/src/orchestrator/actor.rs

@@ -23,7 +23,7 @@ use commonware_parallel::Strategy;
 use commonware_runtime::{
     buffer::PoolRef, spawn_cell, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage,
 };
-use commonware_utils::{vec::NonEmptyVec, NZUsize};
+use commonware_utils::{vec::NonEmptyVec, NZUsize, NZU16};
 use futures::{channel::mpsc, StreamExt};
 use rand_core::CryptoRngCore;
 use std::{collections::BTreeMap, marker::PhantomData, time::Duration};
@@ -105,7 +105,7 @@ where
         config: Config<B, V, C, H, A, S, L, St>,
     ) -> (Self, Mailbox<V, C::PublicKey>) {
         let (sender, mailbox) = mpsc::channel(config.mailbox_size);
-        let pool_ref = PoolRef::new(NZUsize!(16_384), NZUsize!(10_000));
+        let pool_ref = PoolRef::new(NZU16!(16_384), NZUsize!(10_000));
 
         (
             Self {

examples/sync/src/databases/any.rs

@@ -17,7 +17,7 @@ use commonware_storage::{
         operation::Committable,
     },
 };
-use commonware_utils::{NZUsize, NZU64};
+use commonware_utils::{NZUsize, NZU16, NZU64};
 use std::{future::Future, num::NonZeroU64};
 use tracing::error;
 
@@ -39,7 +39,7 @@ pub fn create_config() -> Config<Translator> {
         log_write_buffer: NZUsize!(1024),
         translator: Translator::default(),
         thread_pool: None,
-        buffer_pool: buffer::PoolRef::new(NZUsize!(1024), NZUsize!(10)),
+        buffer_pool: buffer::PoolRef::new(NZU16!(1024), NZUsize!(10)),
     }
 }
 

examples/sync/src/databases/immutable.rs

@@ -11,7 +11,7 @@ use commonware_storage::{
         Durable, Merkleized,
     },
 };
-use commonware_utils::{NZUsize, NZU64};
+use commonware_utils::{NZUsize, NZU16, NZU64};
 use std::{future::Future, num::NonZeroU64};
 use tracing::error;
 
@@ -36,7 +36,7 @@ pub fn create_config() -> Config<Translator, ()> {
         log_write_buffer: NZUsize!(1024),
         translator: commonware_storage::translator::EightCap,
         thread_pool: None,
-        buffer_pool: commonware_runtime::buffer::PoolRef::new(NZUsize!(1024), NZUsize!(10)),
+        buffer_pool: commonware_runtime::buffer::PoolRef::new(NZU16!(1024), NZUsize!(10)),
     }
 }
 

runtime/Cargo.toml

@@ -23,6 +23,7 @@ commonware-conformance = { workspace = true, optional = true }
 commonware-macros.workspace = true
 commonware-parallel = { workspace = true, features = ["std"] }
 commonware-utils = { workspace = true, features = ["std"] }
+crc32fast.workspace = true
 futures.workspace = true
 governor.workspace = true
 io-uring = { workspace = true, optional = true }

runtime/conformance.toml

@@ -1,3 +1,7 @@
 ["commonware_runtime::storage::tests::conformance::CodecConformance<Header>"]
 n_cases = 65536
 hash = "541c356728d47b13f1d3ac800926ef3ae2396c82f5d4e043f5c7641c4c22b4b9"
+
+["commonware_runtime::utils::buffer::pool::tests::conformance::CodecConformance<Checksum>"]
+n_cases = 65536
+hash = "2ca927141b521b7cccc541ec0df8614e418d317fc864ce11f428aefb330cf256"

runtime/fuzz/Cargo.toml

@@ -19,3 +19,9 @@ name = "buffer"
 path = "fuzz_targets/buffer.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "blob_integrity"
+path = "fuzz_targets/blob_integrity.rs"
+test = false
+doc = false