diff --git a/Cargo.lock b/Cargo.lock index 66ac3dc8e9..6bdecdf269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1182,6 +1182,8 @@ dependencies = [ "commonware-math", "commonware-parallel", "commonware-utils", + "crc", + "crc-fast", "criterion", "ctutils", "ecdsa", @@ -1457,11 +1459,11 @@ dependencies = [ "cfg-if", "commonware-codec", "commonware-conformance", + "commonware-cryptography", "commonware-macros", "commonware-parallel", "commonware-utils", "console-subscriber", - "crc32fast", "criterion", "futures", "getrandom 0.2.16", @@ -1512,7 +1514,6 @@ dependencies = [ "commonware-runtime", "commonware-storage", "commonware-utils", - "crc32fast", "criterion", "futures", "futures-util", @@ -1760,6 +1761,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -4610,6 +4636,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spinning_top" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index b2d7d9eb6e..5bdbd5bc4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,8 @@ commonware-storage = { version = "0.0.64", path = "storage", default-features = commonware-stream = { version = "0.0.64", path = "stream" } commonware-utils = { version = "0.0.64", path = "utils", default-features = false } console-subscriber = "0.5.0" -crc32fast = "1.5.0" +crc = "3.4.0" +crc-fast = { version = "1.10.0", default-features = false } criterion = "0.7.0" crossterm = "0.29.0" ctutils = "0.3.1" diff --git a/coding/conformance.toml b/coding/conformance.toml index ed9ada281c..e5bcf2fe41 100644 --- a/coding/conformance.toml +++ b/coding/conformance.toml @@ -8,7 +8,7 @@ hash = "edc22446bb2952609d0c8daccf2d22f8ad2b71eedfcf45296c3f4db49d78404a" ["commonware_coding::reed_solomon::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "7d6e1b7080b182d7b63a261faea8acd77c528c7b0ab99af750d9695457c0f2e3" +hash = "aa8512bb8e86e967833edd1a6cc806280d5e7334e9dc8428a098de9204db12d1" ["commonware_coding::test::conformance::CodecConformance"] n_cases = 65536 @@ -16,8 +16,8 @@ hash = "1a412c5c279f981857081765537b85474184048d1b17053394f94fc42ac1dbf4" ["commonware_coding::zoda::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "e71cd9c877d6e43195d41a00e840e67f134ce425f2c89a1c252fd0131a35cb75" +hash = "ebbbe08eb9beb1c5215a5d67ad9deddaef7c54920e53a751b56a8261e60e0e52" ["commonware_coding::zoda::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "0f69b872f96f5ee24a7f707dc496a7a4721d7f48dd17cb6792d514612ab1376f" +hash = "929ce4f95f9d5784f995c52b7e5cde8b62663ab068848925314dc9f80eb27d34" diff --git a/consensus/conformance.toml b/consensus/conformance.toml index 824f13ff44..90ca23174d 100644 --- a/consensus/conformance.toml +++ b/consensus/conformance.toml @@ -1,54 +1,54 @@ ["commonware_consensus::aggregation::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "650f939707e2af1e00091f813437689bebd63e401952cd34ae967511e8802366" +hash = "f53d34477165ef9764e4f2859e42c837f778c088a91ab69a6a7605ca56ff77c9" ["commonware_consensus::aggregation::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "6498f5323f4bef33ab441bb988081591ac315f55cd78f85bad9f0a7dcd03f579" +hash = "1f25d8ef3c67a8741ee8f566410e6bad03afa1aa3ce8b569ad33a904403dc0d5" ["commonware_consensus::aggregation::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "6d62a98f635a930dcf01f2d5709f70190ef916cce774b7483eaf0092f0b5e1dc" +hash = "fde109e77996588a9cf8f9b723a9521d26148c0337b655f3c4cf0d92e9b12ff4" ["commonware_consensus::aggregation::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "69536ce6262449bc22fd256943545a3c9d8c5290b67a9a6e8396a5e5cf761961" +hash = "55b287b5530fc9ba77c45ec5030ada38f86b2ddf66a1595fde27d4699667b1a3" ["commonware_consensus::aggregation::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "d22d425105d9dda042400a4040dfd06ddcc61957cffebad4b2d5c527979b07f3" +hash = "924a77caeabb871baa8ac0ccd93a2e53da86f124b0dd5b9b0a383273fab42b6c" ["commonware_consensus::marshal::ingress::handler::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "01475715f7713d6b0e856d97aa5789e6546178dacc8319de7ce17fe79a41d2d8" +hash = "481cd68f2e6452c0dda512c75d74ddd2e19ac6c9fb19c642c1a152a0d830c1b2" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "d77c48b9ee8b47ab689c67eff36add26f5cb503f615e971d0695cf8f99730e54" +hash = "6aa909d1f8ad45b30cce5169dd05bb743a3259a9d6612997879fb1057db01fa9" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "902584ad8634ae1eea9551343feddaf0c8b19cc1ca9bafa949c5fe07f61397ca" +hash = "030e7f6899fbbfd76a7757cfdaa57b400b4d0bc7662e883b304514e2a9bfb404" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "b62ae4cd0dfe7ccd2193c2294be33784f37b1bdd9e4f8a6db640e745495f6e64" +hash = "43624c9e7ce355baebf32ab6e76db21cb5bf08005396ba43dae96b3a86f3a535" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "1ba3de6600c32bad42e2a70440a5fcb0ff4970866cd12fa955c1ccdec6ae0704" +hash = "32eac601e1ec919be9e8f502a47c8cd64e8feb61701689d2f9ff6f0a8ebafb03" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "ce5158fafa28b1d9f033a19c5d33fa9d91f6a8940bbd5d42312d0e88804cdb0e" +hash = "b2dfff5e68de39db6881cfd7c5f95397f4330193b8eca5155a716266e53ea921" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "f46dbf29df8571c17a69fbde128c4c62af0efdc3566b945f8add9938071c21db" +hash = "c830d0e3842c1f2ef26f08ee762995942bfbbc722ca9c5e8e6a87c3dbe05c70d" ["commonware_consensus::ordered_broadcast::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "9d2ee4f3c6bf86521c09f8858683de48d6310a987643ee379c143480fc7e9a34" +hash = "6fc6f17d8734a14fb841a8acc1a39cda7519bdd025700663e260d3b000c9c5c6" ["commonware_consensus::simplex::scheme::bls12381_threshold::tests::conformance::CodecConformance>"] n_cases = 65536 @@ -60,43 +60,43 @@ hash = "030caacb9ab46d08601977418808dc5ff729bddb9f6bb2ea685920539b69ecfb" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "f2da297d1d4f0a1bf238227a5cb3cad562a92ea00701960bfd3e5c3dae50d4bd" +hash = "15a849588cb0e5ea44d62c364c789aa564045c970b830b449b74a1f34720f799" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "36a54495de107b83b1e14ee8c0cf97199f175dad90d08ec272766bb0664296a0" +hash = "076241e74a3f2b0da47792d76277bd4f3b617ad429f298d887151d5dae2d42af" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "080be68d5a9b2b564b6c1593c438884daaf2e077c6a04dd9f33feac422e4e36a" +hash = "c81cf2317351865d6a9786a60934995b2693cc759350a853bab7e277f758b5e6" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "a78db937a58c2ba47a07e614a1b1d38c71b8529a9fd55ae8672ae54142b7d49c" +hash = "1422a1e4db078efe00f3f8a5b391d315dc4661f29d24515144a74c65d84a320f" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "e309da1bb5891f59a73fb6e8b9982ef2d98bdd064bf8f6d9d72851e257674100" +hash = "ae3440a50566a8e2108c272e2934e2bd28a39715c2d6077ccda329d5d19c5357" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "e309da1bb5891f59a73fb6e8b9982ef2d98bdd064bf8f6d9d72851e257674100" +hash = "ae3440a50566a8e2108c272e2934e2bd28a39715c2d6077ccda329d5d19c5357" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "aab95f3f91f700f11302e98d1e13198d8f6ff2c5d852a27fbe659b4bb45c7e0e" +hash = "efbd478fdec6e832ffa445bf9cf0282b496bb0cbc45d6e0cf15281cd733b9978" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "b1be7e36c66125871070a7465dde3d4510ccb6f46bbcaef46c585611eb3c7bf4" +hash = "182a3d295140ffd4ea33fe5bbfd2b09156c76c4dcde06e71707ac3479345fa0c" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "aab95f3f91f700f11302e98d1e13198d8f6ff2c5d852a27fbe659b4bb45c7e0e" +hash = "efbd478fdec6e832ffa445bf9cf0282b496bb0cbc45d6e0cf15281cd733b9978" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "b1be7e36c66125871070a7465dde3d4510ccb6f46bbcaef46c585611eb3c7bf4" +hash = "182a3d295140ffd4ea33fe5bbfd2b09156c76c4dcde06e71707ac3479345fa0c" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 @@ -108,11 +108,11 @@ hash = "1430b3fac9de5758b9d00c8793d56fc57d1eddbf5ce2fd4f24d2229e4b15c226" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "37fec8dca307c26dad1b6bb84960feb5239ccd739558db394a2a311176c1aab3" +hash = "8681786700e24b08366664c9c76a1a4b144ef7881b13c6468bb5ca78175baa09" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "c5274c98a6c936c91f380261cff61c2dc9988e5f28700f0d9cb1b25346d1ffc3" +hash = "4ba1f5b5cbf271ae45313d8ac48b05c010166ca743334b9b8051a4a610c58fe7" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance"] n_cases = 65536 @@ -120,11 +120,11 @@ hash = "6f852b6ad6e5dd77804d80781b03c808e186e7e7c5611752f124c625c5c3ee17" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "37c0c4f2dd328ed18a8f29fb73ca7c82c4008e49ea00243ffb3e024873a5b565" +hash = "6ad91c2b61600699c399c20abb6e9f948bf95ac413363c325226d327bbb37456" ["commonware_consensus::simplex::types::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "dec7851b91f56833b8f917c7e3ba16a28cd2683c68dda6c021c5d3c5c2400a37" +hash = "9eefedcb37bd0e098bcf47cf6135148900755f3eeca78f30b013433fe368678f" ["commonware_consensus::types::tests::conformance::CodecConformance"] n_cases = 65536 diff --git a/cryptography/Cargo.toml b/cryptography/Cargo.toml index 62896efd38..a6ed333390 100644 --- a/cryptography/Cargo.toml +++ b/cryptography/Cargo.toml @@ -25,6 +25,7 @@ commonware-codec.workspace = true commonware-math.workspace = true commonware-parallel.workspace = true commonware-utils.workspace = true +crc-fast = { workspace = true, features = ["panic-handler"] } ctutils.workspace = true ecdsa.workspace = true ed25519-consensus = { workspace = true, default-features = false } @@ -50,6 +51,7 @@ features = ["js"] anyhow.workspace = true commonware-conformance.workspace = true commonware-math = { workspace = true, features = ["test_strategies"] } +crc.workspace = true criterion.workspace = true proptest.workspace = true rayon.workspace = true @@ -81,6 +83,7 @@ std = [ "commonware-math/std", "commonware-parallel/std", "commonware-utils/std", + "crc-fast/std", "ecdsa/std", "ed25519-consensus/std", "getrandom/std", diff --git a/cryptography/conformance.toml b/cryptography/conformance.toml index 51b04492c7..f582d8ac2e 100644 --- a/cryptography/conformance.toml +++ b/cryptography/conformance.toml @@ -1,6 +1,6 @@ ["commonware_cryptography::blake3::tests::conformance::CodecConformance"] n_cases = 65536 -hash = "c0501d4a691d1fccec7c5906e8608228569d24164150edd215838593e3b77512" +hash = "d01e3ef0dd81919abd3f149649940ae2e8cff6dc4aa9f927d544aece387b12d1" ["commonware_cryptography::bloomfilter::tests::conformance::CodecConformance"] n_cases = 65536 @@ -102,6 +102,10 @@ hash = "42867052b1619b912d08f0a6bf5f964f587897a0eed8612732fd27813f6c03bd" n_cases = 65536 hash = "41ef7a306dd2032d7d1b73fe2799cff6f2e8fb1edcd02a0b9798f18f900a19fa" +["commonware_cryptography::crc32::tests::conformance::CodecConformance"] +n_cases = 65536 +hash = "d0101a0688d45c8f7428b3861ed6e7e77c2ed25033cb7d6b47b3fa58148a3f18" + ["commonware_cryptography::ed25519::certificate::tests::conformance::CodecConformance"] n_cases = 65536 hash = "eab4ff22bb7ed99fd73107c8b067e39bd25813b65614df95b4b5c46a312557ad" @@ -164,7 +168,7 @@ hash = "05880099d66c9e04ab2d4bf5dfa3fecfc4fb114264ca3b4a0c3b4816332414b9" ["commonware_cryptography::sha256::tests::conformance::CodecConformance"] n_cases = 65536 -hash = "c0501d4a691d1fccec7c5906e8608228569d24164150edd215838593e3b77512" +hash = "d65ced18cacd65769e8f9732131a06216e6600d5a115c4ed7483a8530813ddcd" ["commonware_cryptography::transcript::test::conformance::CodecConformance"] n_cases = 65536 diff --git a/cryptography/src/blake3/mod.rs b/cryptography/src/blake3/mod.rs index 53ee3d04f6..d73790642e 100644 --- a/cryptography/src/blake3/mod.rs +++ b/cryptography/src/blake3/mod.rs @@ -93,10 +93,19 @@ impl Hasher for Blake3 { /// Digest of a BLAKE3 hashing operation. #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[repr(transparent)] pub struct Digest(pub [u8; DIGEST_LENGTH]); +#[cfg(feature = "arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for Digest { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + // Generate random bytes and compute their Blake3 hash + let len = u.int_in_range(0..=256)?; + let data = u.bytes(len)?; + Ok(Blake3::hash(data)) + } +} + impl Write for Digest { fn write(&self, buf: &mut impl BufMut) { self.0.write(buf); diff --git a/cryptography/src/crc32/mod.rs b/cryptography/src/crc32/mod.rs new file mode 100644 index 0000000000..b56a3abc11 --- /dev/null +++ b/cryptography/src/crc32/mod.rs @@ -0,0 +1,443 @@ +//! CRC32C implementation of the `Hasher` trait. +//! +//! This implementation uses the `crc-fast` crate to generate CRC32C (iSCSI/Castagnoli) +//! checksums as specified in RFC 3720. CRC32C uses polynomial 0x1EDC6F41. +//! +//! # Warning +//! +//! CRC32 is not a cryptographic hash function. It is designed for error +//! detection, not security. Use SHA-256 or Blake3 for cryptographic purposes. +//! +//! # Example +//! +//! ```rust +//! use commonware_cryptography::{Hasher, Crc32}; +//! +//! // One-shot checksum (returns u32 directly) +//! let checksum: u32 = Crc32::checksum(b"hello world"); +//! +//! // Using the Hasher trait +//! let mut hasher = Crc32::new(); +//! hasher.update(b"hello "); +//! hasher.update(b"world"); +//! let digest = hasher.finalize(); +//! +//! // Convert digest to u32 +//! assert_eq!(digest.as_u32(), checksum); +//! ``` + +use crate::Hasher; +#[cfg(not(feature = "std"))] +use alloc::vec; +use bytes::{Buf, BufMut}; +use commonware_codec::{Error as CodecError, FixedSize, Read, ReadExt, Write}; +use commonware_math::algebra::Random; +use commonware_utils::{hex, Array, Span}; +use core::{ + fmt::{Debug, Display}, + ops::Deref, +}; +use rand_core::CryptoRngCore; + +/// Size of a CRC32 checksum in bytes. +const SIZE: usize = 4; + +/// The CRC32 algorithm used (CRC32C/iSCSI/Castagnoli). +const ALGORITHM: crc_fast::CrcAlgorithm = crc_fast::CrcAlgorithm::Crc32Iscsi; + +/// CRC32C hasher. +/// +/// Uses the iSCSI polynomial (0x1EDC6F41) as specified in RFC 3720. +#[derive(Debug)] +pub struct Crc32 { + inner: crc_fast::Digest, +} + +impl Default for Crc32 { + fn default() -> Self { + Self { + inner: crc_fast::Digest::new(ALGORITHM), + } + } +} + +impl Clone for Crc32 { + fn clone(&self) -> Self { + // We manually implement `Clone` to avoid cloning the hasher state. + Self::default() + } +} + +impl Crc32 { + /// Compute a CRC32 checksum of the given data (one-shot). + /// + /// Returns the checksum as a `u32` directly. + #[inline] + pub fn checksum(data: &[u8]) -> u32 { + crc_fast::checksum(ALGORITHM, data) as u32 + } +} + +impl Hasher for Crc32 { + type Digest = Digest; + + fn update(&mut self, message: &[u8]) -> &mut Self { + self.inner.update(message); + self + } + + fn finalize(&mut self) -> Self::Digest { + Self::Digest::from(self.inner.finalize_reset() as u32) + } + + fn reset(&mut self) -> &mut Self { + self.inner = crc_fast::Digest::new(ALGORITHM); + self + } +} + +/// Digest of a CRC32 hashing operation (4 bytes). +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[repr(transparent)] +pub struct Digest(pub [u8; SIZE]); + +#[cfg(feature = "arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for Digest { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + // Generate random bytes and compute their CRC32 checksum + let len = u.int_in_range(0..=256)?; + let data = u.bytes(len)?; + Ok(Crc32::hash(data)) + } +} + +impl Digest { + /// Get the digest as a `u32` value. + #[inline] + pub const fn as_u32(&self) -> u32 { + u32::from_be_bytes(self.0) + } +} + +impl Write for Digest { + fn write(&self, buf: &mut impl BufMut) { + self.0.write(buf); + } +} + +impl Read for Digest { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &()) -> Result { + let array = <[u8; SIZE]>::read(buf)?; + Ok(Self(array)) + } +} + +impl FixedSize for Digest { + const SIZE: usize = SIZE; +} + +impl Span for Digest {} + +impl Array for Digest {} + +impl From<[u8; SIZE]> for Digest { + fn from(value: [u8; SIZE]) -> Self { + Self(value) + } +} + +impl From for Digest { + fn from(value: u32) -> Self { + Self(value.to_be_bytes()) + } +} + +impl AsRef<[u8]> for Digest { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl Deref for Digest { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.0 + } +} + +impl Debug for Digest { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", hex(&self.0)) + } +} + +impl Display for Digest { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", hex(&self.0)) + } +} + +impl crate::Digest for Digest { + const EMPTY: Self = Self([0u8; SIZE]); +} + +impl Random for Digest { + fn random(mut rng: impl CryptoRngCore) -> Self { + let mut array = [0u8; SIZE]; + rng.fill_bytes(&mut array); + Self(array) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Hasher; + use commonware_codec::{DecodeExt, Encode}; + use crc::{Crc, CRC_32_ISCSI}; + + /// Reference CRC32C implementation from the [`crc`](https://crates.io/crates/crc) crate. + const CRC32C_REF: Crc = Crc::::new(&CRC_32_ISCSI); + + /// Verify checksum against both the reference `crc` crate and our implementation. + fn verify(data: &[u8], expected: u32) { + assert_eq!(CRC32C_REF.checksum(data), expected); + assert_eq!(Crc32::checksum(data), expected); + } + + /// Generate deterministic test data: sequential bytes wrapping at 256. + fn sequential_data(len: usize) -> Vec { + (0..len).map(|i| (i & 0xFF) as u8).collect() + } + + /// Test vectors from RFC 3720 Appendix B.4 "CRC Examples". + /// https://datatracker.ietf.org/doc/html/rfc3720#appendix-B.4 + #[test] + fn rfc3720_test_vectors() { + // 32 bytes of zeros -> CRC = aa 36 91 8a + verify(&[0x00; 32], 0x8A9136AA); + + // 32 bytes of 0xFF -> CRC = 43 ab a8 62 + verify(&[0xFF; 32], 0x62A8AB43); + + // 32 bytes ascending (0x00..0x1F) -> CRC = 4e 79 dd 46 + let ascending: Vec = (0x00..0x20).collect(); + verify(&ascending, 0x46DD794E); + + // 32 bytes descending (0x1F..0x00) -> CRC = 5c db 3f 11 + let descending: Vec = (0x00..0x20).rev().collect(); + verify(&descending, 0x113FDB5C); + + // iSCSI SCSI Read (10) Command PDU -> CRC = 56 3a 96 d9 + let iscsi_read_pdu: [u8; 48] = [ + 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + verify(&iscsi_read_pdu, 0xD9963A56); + } + + /// Additional test vectors from external sources. + /// https://reveng.sourceforge.io/crc-catalogue/17plus.htm#crc.cat.crc-32c + /// https://github.com/ICRAR/crc32c/blob/master/test/test_crc32c.py + /// https://github.com/google/leveldb/blob/main/util/crc32c_test.cc + #[test] + fn external_test_vectors() { + // CRC catalogue test vector + verify(b"", 0x00000000); + verify(b"123456789", 0xE3069283); + + // ICRAR test vectors + verify(b"23456789", 0xBFE92A83); + verify(b"The quick brown fox jumps over the lazy dog", 0x22620404); + + // LevelDB test vector: sequential 0x01-0xF0 (240 bytes) + let sequential_240: Vec = (0x01..=0xF0).collect(); + verify(&sequential_240, 0x24C5D375); + } + + /// SIMD boundary tests. + /// + /// SIMD implementations (PCLMULQDQ, ARM CRC) have different code paths + /// based on input size. These tests verify correctness at critical boundaries. + #[test] + fn simd_boundaries() { + // Critical sizes where SIMD implementations change code paths: + // - 16: single 128-bit register + // - 32: two 128-bit registers / one 256-bit register + // - 64: fold-by-4 block size + // - 128: large data threshold + // - 256, 512, 1024: power-of-2 boundaries + // - 4096: page boundary (common in storage) + const BOUNDARY_SIZES: &[usize] = &[ + 0, 1, 2, 3, 4, 7, 8, 9, // Small sizes + 15, 16, 17, // 128-bit boundary + 31, 32, 33, // 256-bit boundary + 63, 64, 65, // Fold-by-4 boundary + 127, 128, 129, // Large threshold + 255, 256, 257, // 256-byte boundary + 511, 512, 513, // 512-byte boundary + 1023, 1024, 1025, // 1KB boundary + 4095, 4096, 4097, // Page boundary + ]; + + // Pre-computed expected values for sequential data pattern. + // Generated with the [`crc`](https://crates.io/crates/crc) crate. + const EXPECTED: &[(usize, u32)] = &[ + (0, 0x00000000), + (1, 0x527D5351), + (2, 0x030AF4D1), + (3, 0x92FD4BFA), + (4, 0xD9331AA3), + (7, 0xA359ED4C), + (8, 0x8A2CBC3B), + (9, 0x7144C5A8), + (15, 0x68EF03F6), + (16, 0xD9C908EB), + (17, 0x38435E17), + (31, 0xE95CABCB), + (32, 0x46DD794E), // Matches RFC 3720 + (33, 0x9F85A26D), + (63, 0x7A873004), + (64, 0xFB6D36EB), + (65, 0x694420FA), + (127, 0x6C31BD0C), + (128, 0x30D9C515), + (129, 0xF514629F), + (255, 0x8953C482), + (256, 0x9C44184B), + (257, 0x8A13A1CE), + (511, 0x35348950), + (512, 0xAE10EE5A), + (513, 0x6814B154), + (1023, 0x0C8F24D0), + (1024, 0x2CDF6E8F), + (1025, 0x8EB48B63), + (4095, 0xBCB5BD82), + (4096, 0x9C71FE32), + (4097, 0x83391BE9), + ]; + + assert_eq!( + BOUNDARY_SIZES, + EXPECTED.iter().map(|(size, _)| *size).collect::>() + ); + + for &(size, expected) in EXPECTED { + let data = sequential_data(size); + verify(&data, expected); + } + } + + /// Verify incremental hashing produces the same result regardless of chunk size. + #[test] + fn chunk_size_independence() { + let data = sequential_data(1024); + let expected = CRC32C_REF.checksum(&data); + + // Test chunk sizes from 1 to 64 bytes + for chunk_size in 1..=64 { + let mut hasher = Crc32::new(); + for chunk in data.chunks(chunk_size) { + hasher.update(chunk); + } + assert_eq!(hasher.finalize().as_u32(), expected); + } + } + + /// Test with unaligned data by processing at different offsets within a buffer. + #[test] + fn alignment_independence() { + // Create a larger buffer and test CRC of a fixed-size window at different offsets + let base_data: Vec = (0..256).map(|i| i as u8).collect(); + let test_len = 64; + + // Get reference CRC for the first 64 bytes + let reference = CRC32C_REF.checksum(&base_data[..test_len]); + + // Verify the same 64-byte pattern produces the same CRC regardless of where + // it appears in the source buffer (tests alignment handling) + for offset in 0..16 { + let data = &base_data[offset..offset + test_len]; + let expected = CRC32C_REF.checksum(data); + assert_eq!(Crc32::checksum(data), expected); + } + + // Also verify that the first 64 bytes always produce the reference CRC + verify(&base_data[..test_len], reference); + } + + #[test] + fn test_crc32_hasher_trait() { + let msg = b"hello world"; + + // Generate initial hash using Hasher trait + let mut hasher = Crc32::new(); + hasher.update(msg); + let digest = hasher.finalize(); + assert!(Digest::decode(digest.as_ref()).is_ok()); + + // Verify against reference + let expected = CRC32C_REF.checksum(msg); + assert_eq!(digest.as_u32(), expected); + + // Reuse hasher (should auto-reset after finalize) + hasher.update(msg); + let digest2 = hasher.finalize(); + assert_eq!(digest, digest2); + + // Test Hasher::hash convenience method + let hash = Crc32::hash(msg); + assert_eq!(hash.as_u32(), expected); + } + + #[test] + fn test_crc32_len() { + assert_eq!(Digest::SIZE, SIZE); + assert_eq!(SIZE, 4); + } + + #[test] + fn test_codec() { + let msg = b"hello world"; + let mut hasher = Crc32::new(); + hasher.update(msg); + let digest = hasher.finalize(); + + let encoded = digest.encode(); + assert_eq!(encoded.len(), SIZE); + assert_eq!(encoded, digest.as_ref()); + + let decoded = Digest::decode(encoded).unwrap(); + assert_eq!(digest, decoded); + } + + #[test] + fn test_digest_from_u32() { + let value: u32 = 0xDEADBEEF; + let digest = Digest::from(value); + assert_eq!(digest.as_u32(), value); + assert_eq!(digest.0, [0xDE, 0xAD, 0xBE, 0xEF]); + } + + #[test] + fn test_checksum_returns_u32() { + // Verify the one-shot checksum returns u32 directly + let checksum: u32 = Crc32::checksum(b"test"); + let expected = CRC32C_REF.checksum(b"test"); + assert_eq!(checksum, expected); + } + + #[cfg(feature = "arbitrary")] + mod conformance { + use super::*; + use commonware_codec::conformance::CodecConformance; + + commonware_conformance::conformance_tests! { + CodecConformance, + } + } +} diff --git a/cryptography/src/lib.rs b/cryptography/src/lib.rs index 332876d0b8..d6992ec27f 100644 --- a/cryptography/src/lib.rs +++ b/cryptography/src/lib.rs @@ -28,6 +28,8 @@ use rand_core::CryptoRngCore; pub use sha256::{CoreSha256, Sha256}; pub mod blake3; pub use blake3::{Blake3, CoreBlake3}; +pub mod crc32; +pub use crc32::Crc32; pub mod bloomfilter; pub use bloomfilter::BloomFilter; #[cfg(feature = "std")] diff --git a/cryptography/src/sha256/mod.rs b/cryptography/src/sha256/mod.rs index a0c48a5a79..fe9bbcfdcd 100644 --- a/cryptography/src/sha256/mod.rs +++ b/cryptography/src/sha256/mod.rs @@ -83,10 +83,19 @@ impl Hasher for Sha256 { /// Digest of a SHA-256 hashing operation. #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[repr(transparent)] pub struct Digest(pub [u8; DIGEST_LENGTH]); +#[cfg(feature = "arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for Digest { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + // Generate random bytes and compute their Sha256 hash + let len = u.int_in_range(0..=256)?; + let data = u.bytes(len)?; + Ok(Sha256::hash(data)) + } +} + impl Write for Digest { fn write(&self, buf: &mut impl BufMut) { self.0.write(buf); diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 8ec609c8b8..e04975702b 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -20,10 +20,10 @@ bytes.workspace = true cfg-if.workspace = true commonware-codec.workspace = true commonware-conformance = { workspace = true, optional = true } +commonware-cryptography = { workspace = true, features = ["std"] } commonware-macros.workspace = true commonware-parallel = { workspace = true, features = ["std"] } commonware-utils = { workspace = true, features = ["std"] } -crc32fast.workspace = true futures.workspace = true governor.workspace = true io-uring = { workspace = true, optional = true } @@ -58,6 +58,7 @@ tokio = { workspace = true, features = ["full"] } default = [] arbitrary = [ "commonware-codec/arbitrary", + "commonware-cryptography/arbitrary", "commonware-utils/arbitrary", "dep:arbitrary", "dep:commonware-conformance", diff --git a/runtime/src/utils/buffer/pool/append.rs b/runtime/src/utils/buffer/pool/append.rs index ab1ec84687..18c077336a 100644 --- a/runtime/src/utils/buffer/pool/append.rs +++ b/runtime/src/utils/buffer/pool/append.rs @@ -33,6 +33,7 @@ use crate::{ }, Blob, Error, RwLock, RwLockWriteGuard, }; +use commonware_cryptography::Crc32; use commonware_utils::StableBuf; use std::{num::NonZeroUsize, sync::Arc}; use tracing::warn; @@ -628,7 +629,7 @@ impl Append { let logical_page = &buffer.data[start_read_idx..end_read_idx]; write_buffer.extend_from_slice(logical_page); - let crc = crc32fast::hash(logical_page); + let crc = Crc32::checksum(logical_page); let logical_page_size_u16 = u16::try_from(logical_page_size).expect("page size must fit in u16 for CRC record"); @@ -664,7 +665,7 @@ impl Append { } write_buffer.extend_from_slice(partial_page); let partial_len = partial_page.len(); - let crc = crc32fast::hash(partial_page); + let crc = Crc32::checksum(partial_page); // Pad with zeros to fill up to logical_page_size. write_buffer.resize(write_buffer.len() + (logical_page_size - partial_len), 0); diff --git a/runtime/src/utils/buffer/pool/mod.rs b/runtime/src/utils/buffer/pool/mod.rs index c7094b8e79..2a92c6331a 100644 --- a/runtime/src/utils/buffer/pool/mod.rs +++ b/runtime/src/utils/buffer/pool/mod.rs @@ -26,6 +26,7 @@ use crate::{Blob, Error}; use bytes::{Buf, BufMut}; use commonware_codec::{EncodeFixed, FixedSize, Read as CodecRead, ReadExt, Write}; +use commonware_cryptography::{crc32, Crc32}; use commonware_utils::StableBuf; mod append; @@ -38,7 +39,7 @@ pub use read::Read; use tracing::{debug, error}; // A checksum record contains two u16 lengths and two CRCs (each 4 bytes). -const CHECKSUM_SIZE: u64 = 12; +const CHECKSUM_SIZE: u64 = Checksum::SIZE as u64; /// Read the designated page from the underlying blob and return its logical bytes as a vector if it /// passes the integrity check, returning error otherwise. Safely handles partial pages. Caller can @@ -128,7 +129,7 @@ impl Checksum { return None; } - let computed_crc = crc32fast::hash(&buf[..len_usize]); + let computed_crc = Crc32::checksum(&buf[..len_usize]); if computed_crc != crc { debug!("Invalid CRC: doesn't match page contents. Using fallback CRC"); if crc_record.validate_fallback(buf, crc_start_idx) { @@ -159,7 +160,7 @@ impl Checksum { return false; } - let computed_crc = crc32fast::hash(&buf[..len_usize]); + let computed_crc = Crc32::checksum(&buf[..len_usize]); if computed_crc != crc { debug!("Invalid fallback CRC: doesn't match page contents."); return false; @@ -225,7 +226,7 @@ impl CodecRead for Checksum { } impl FixedSize for Checksum { - const SIZE: usize = CHECKSUM_SIZE as usize; + const SIZE: usize = 2 * u16::SIZE + 2 * crc32::Digest::SIZE; } #[cfg(feature = "arbitrary")] @@ -244,8 +245,6 @@ impl arbitrary::Arbitrary<'_> for Checksum { mod tests { use super::*; - const CHECKSUM_SIZE_USIZE: usize = CHECKSUM_SIZE as usize; - #[test] fn test_crc_record_encode_read_roundtrip() { let record = Checksum { @@ -327,7 +326,7 @@ mod tests { #[test] fn test_validate_page_valid() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write some data @@ -335,11 +334,11 @@ mod tests { page[..data.len()].copy_from_slice(data); // Compute CRC of the data portion - let crc = crc32fast::hash(&page[..data.len()]); + let crc = Crc32::checksum(&page[..data.len()]); let record = Checksum::new(data.len() as u16, crc); // Write the CRC record at the end - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Validate - should return Some with the Checksum @@ -352,7 +351,7 @@ mod tests { #[test] fn test_validate_page_invalid_crc() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write some data @@ -363,7 +362,7 @@ mod tests { let wrong_crc = 0xBADBADBA; let record = Checksum::new(data.len() as u16, wrong_crc); - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Should fail validation (return None) @@ -374,16 +373,16 @@ mod tests { #[test] fn test_validate_page_corrupted_data() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write some data and compute correct CRC let data = b"hello world"; page[..data.len()].copy_from_slice(data); - let crc = crc32fast::hash(&page[..data.len()]); + let crc = Crc32::checksum(&page[..data.len()]); let record = Checksum::new(data.len() as u16, crc); - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Corrupt the data @@ -397,13 +396,13 @@ mod tests { #[test] fn test_validate_page_uses_larger_len() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write data and compute CRC for the larger portion let data = b"hello world, this is longer"; page[..data.len()].copy_from_slice(data); - let crc = crc32fast::hash(&page[..data.len()]); + let crc = Crc32::checksum(&page[..data.len()]); // Create a record where len2 has the valid CRC for longer data let record = Checksum { @@ -413,7 +412,7 @@ mod tests { crc2: crc, }; - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Should validate using len2/crc2 since len2 > len1 @@ -426,13 +425,13 @@ mod tests { #[test] fn test_validate_page_uses_fallback() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write data let data = b"fallback data"; page[..data.len()].copy_from_slice(data); - let valid_crc = crc32fast::hash(&page[..data.len()]); + let valid_crc = Crc32::checksum(&page[..data.len()]); let valid_len = data.len() as u16; // Create a record where: @@ -445,7 +444,7 @@ mod tests { crc2: valid_crc, // Valid CRC }; - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Should validate using the fallback (len2) @@ -465,7 +464,7 @@ mod tests { #[test] fn test_validate_page_no_fallback_available() { let logical_page_size = 64usize; - let physical_page_size = logical_page_size + CHECKSUM_SIZE_USIZE; + let physical_page_size = logical_page_size + Checksum::SIZE; let mut page = vec![0u8; physical_page_size]; // Write some data @@ -482,7 +481,7 @@ mod tests { crc2: 0, }; - let crc_start = physical_page_size - CHECKSUM_SIZE_USIZE; + let crc_start = physical_page_size - Checksum::SIZE; page[crc_start..].copy_from_slice(&record.to_bytes()); // Should fail validation since primary is invalid and no fallback exists diff --git a/runtime/src/utils/buffer/pool/page_cache.rs b/runtime/src/utils/buffer/pool/page_cache.rs index c7ddaf31ba..ca7e92af2b 100644 --- a/runtime/src/utils/buffer/pool/page_cache.rs +++ b/runtime/src/utils/buffer/pool/page_cache.rs @@ -418,6 +418,7 @@ impl Pool { mod tests { use super::{super::Checksum, *}; use crate::{buffer::pool::CHECKSUM_SIZE, deterministic, Runner as _, Storage as _}; + use commonware_cryptography::Crc32; use commonware_macros::test_traced; use commonware_utils::{NZUsize, NZU16}; use std::num::NonZeroU16; @@ -489,7 +490,7 @@ mod tests { for i in 0..11 { // Write logical data followed by Checksum. let logical_data = vec![i as u8; PAGE_SIZE.get() as usize]; - let crc = crc32fast::hash(&logical_data); + let crc = Crc32::checksum(&logical_data); let record = Checksum::new(PAGE_SIZE.get(), crc); let mut page_data = logical_data; page_data.extend_from_slice(&record.to_bytes()); diff --git a/runtime/src/utils/buffer/pool/read.rs b/runtime/src/utils/buffer/pool/read.rs index 95a13d1647..6b82f090c4 100644 --- a/runtime/src/utils/buffer/pool/read.rs +++ b/runtime/src/utils/buffer/pool/read.rs @@ -1,11 +1,10 @@ -use super::{Checksum, CHECKSUM_SIZE}; +use super::Checksum; use crate::{Blob, Error}; +use commonware_codec::FixedSize; use commonware_utils::StableBuf; use std::num::NonZeroUsize; use tracing::{debug, error}; -const CHECKSUM_SIZE_USIZE: usize = CHECKSUM_SIZE as usize; - /// A reader that buffers content from a [Blob] with page-level CRCs to optimize the performance of /// a full scan of contents. pub struct Read { @@ -45,7 +44,7 @@ impl Read { capacity: NonZeroUsize, logical_page_size: NonZeroUsize, ) -> Self { - let page_size = logical_page_size.get() + CHECKSUM_SIZE_USIZE; + let page_size = logical_page_size.get() + Checksum::SIZE; let mut capacity = capacity.get(); if !capacity.is_multiple_of(page_size) { capacity += page_size - capacity % page_size; @@ -74,7 +73,7 @@ impl Read { /// Returns the current logical position in the blob. pub const fn position(&self) -> u64 { - let logical_page_size = (self.page_size - CHECKSUM_SIZE_USIZE) as u64; + let logical_page_size = (self.page_size - Checksum::SIZE) as u64; self.blob_page * logical_page_size + self.buffer_position as u64 } @@ -141,7 +140,7 @@ impl Read { /// Fills the buffer from the blob starting at the current physical position and verifies the /// CRC of each page (including any trailing partial page). async fn fill_buffer(&mut self) -> Result<(), Error> { - let logical_page_size = self.page_size - CHECKSUM_SIZE_USIZE; + let logical_page_size = self.page_size - Checksum::SIZE; // Advance blob_page based on how much of the buffer we've consumed. We use ceiling division // because even a partial page counts as a "page" read from the blob. @@ -222,7 +221,7 @@ impl Read { } // Partial page - must have at least CHECKSUM_SIZE bytes - if remaining < CHECKSUM_SIZE_USIZE { + if remaining < Checksum::SIZE { error!( page = self.blob_page + (read_offset / self.page_size) as u64, "short page" @@ -261,7 +260,7 @@ impl Read { /// Repositions the buffer to read from the specified logical position in the blob. pub fn seek_to(&mut self, position: u64) -> Result<(), Error> { - let logical_page_size = (self.page_size - CHECKSUM_SIZE_USIZE) as u64; + let logical_page_size = (self.page_size - Checksum::SIZE) as u64; // Check if the position is within the current buffer. let buffer_start = self.blob_page * logical_page_size; @@ -283,6 +282,7 @@ impl Read { mod tests { use super::super::{append::Append, PoolRef}; use crate::{deterministic, Blob, Error, Runner as _, Storage as _}; + use commonware_cryptography::Crc32; use commonware_macros::test_traced; use commonware_utils::{NZUsize, NZU16}; use std::num::NonZeroU16; @@ -448,7 +448,7 @@ mod tests { // Corrupt page 0 to claim a shorter (partial) length with a valid CRC. let page_size = PAGE_SIZE.get() as u64; let short_len = page_size / 2; - let crc = crc32fast::hash(&data[..short_len as usize]); + let crc = Crc32::checksum(&data[..short_len as usize]); let record = super::Checksum::new(short_len as u16, crc); let crc_offset = page_size; // CRC record starts after logical page bytes blob.write_at(record.to_bytes().to_vec(), crc_offset) diff --git a/storage/Cargo.toml b/storage/Cargo.toml index db57ea1a7b..068d99caca 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -24,7 +24,6 @@ commonware-macros.workspace = true commonware-parallel = { workspace = true, optional = true, features = ["std"] } commonware-runtime = { workspace = true, optional = true } commonware-utils = { workspace = true, default-features = false } -crc32fast = { workspace = true, optional = true } futures = { workspace = true, optional = true } futures-util = { workspace = true, optional = true } prometheus-client = { workspace = true, optional = true } @@ -53,7 +52,6 @@ std = [ "commonware-parallel?/std", "commonware-runtime", "commonware-utils/std", - "crc32fast/std", "futures", "futures-util", "prometheus-client", diff --git a/storage/conformance.toml b/storage/conformance.toml index ab29dff0ac..5d7f488019 100644 --- a/storage/conformance.toml +++ b/storage/conformance.toml @@ -1,10 +1,10 @@ ["commonware_storage::archive::conformance::ArchiveImmutable"] n_cases = 128 -hash = "6acfa1bc0c17920b5c0e0437af106e09ee57dcd37459091402192f2c146afdb5" +hash = "45b84846b33d2769f05c4118b6388e944361e778cc320c4d94d2297b46faf816" ["commonware_storage::archive::conformance::ArchivePrunable"] n_cases = 128 -hash = "cb063a05c6a75902893f790e9802b4906be506f2f7e5d10b46dff90a92e40819" +hash = "590d5e27608bed5c2945a3347b04f13bbeb706ba28ca38f6c307c4428a621ca7" ["commonware_storage::archive::immutable::storage::conformance::CodecConformance"] n_cases = 65536 @@ -16,15 +16,15 @@ hash = "3cb6882637c1c1a929a50b3ab425311f3ef342184dc46a80b1eae616ca7b64a4" ["commonware_storage::bmt::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "927da8204f16522162168efcae8ca126b927c96fb08289e805859071647b9333" +hash = "20f5ef35a4bbd3a40852e907df519c724e5ce24d9f929e84947fd971a2256d02" ["commonware_storage::bmt::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "fcb0baa781f374ca04f2c967abc3c82c7b99d26239b387472de23c85aebd2790" +hash = "c1f1d4c35fcd50931d7c36cbcddbb1c0a93afef9a93945cdd3efadf68ff53328" ["commonware_storage::bmt::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "035babcc7aa34c3d4b263924865c4da8206b226e7c372484b89e5430166068c3" +hash = "6ecb0491b09443f1f93c178af5472f138ddc71b3e8c0c106f32eafca617b56af" ["commonware_storage::cache::storage::conformance::CodecConformance>"] n_cases = 65536 @@ -48,35 +48,35 @@ hash = "13b3e99a8c74b50dc18150194a92306de670b94e6642758feb6d9b6e9881f827" ["commonware_storage::journal::conformance::ContiguousFixed"] n_cases = 512 -hash = "4c786b6b7f91b9924a62a7b9a1c32a8d47398f1c8a3d5bf06fe1a90998e86aab" +hash = "cac48d3c84becf5567c7ee1eb3495809f48610ab400200f748815af4ccceb5c1" ["commonware_storage::journal::conformance::ContiguousVariable"] n_cases = 512 -hash = "973ebd77804d2ea346574d377f39bd29350063098f2e6fab9596783ba43664e5" +hash = "9debd18a1e9b8e8844a57bcbc9546a80503a8caad5c23816206d623ed230bd7e" ["commonware_storage::journal::conformance::SegmentedFixed"] n_cases = 512 -hash = "e077ce8c6d9a79c87cf9b48866c65f387ffbec9d8e8c65dd40c46b4296cfc050" +hash = "fb5d8f684b1a03c0bbaf8c4fd7b67043fa73496446fff115e75d7521154b6ad1" ["commonware_storage::journal::conformance::SegmentedGlob"] n_cases = 512 -hash = "adb1efeef12c203c05879ce4d1d03ef443c767737a6c6b57433189100eec9197" +hash = "bcc3e67945fe5a97a644e839aeff277bb29711503da65a20e84077e69ae9dcef" ["commonware_storage::journal::conformance::SegmentedOversized"] n_cases = 512 -hash = "b815138329a06cbe235cf547ed62774165bd2108e68c65bf15ae152bedf84b3a" +hash = "ada5da872b95b6369905c2fca454f3276935a380d3da91190bc04f1602e7c501" ["commonware_storage::journal::conformance::SegmentedVariable"] n_cases = 512 -hash = "418dafd67008cb74d34fe58b9be8747cfaf86e345a71eb4c35ae0e43a4c077ef" +hash = "506b5a02d1a7c3e249f514b889e6099c27bb0723ebc711514263f0c71f0676ef" ["commonware_storage::mmr::proof::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "e3e6735a810f1002164333013fbff442c91a690483e75fe7a78618a96d5afd62" +hash = "9658f1ff018c2339049b7bcba191d3202844e28d3bf3001da851c9be505232ed" ["commonware_storage::ordinal::storage::conformance::CodecConformance>"] n_cases = 65536 -hash = "07a88b442e9f86b5395a73584211cb6abbb58e51c6f3954b29095c56d77d370c" +hash = "f93b1dda40f6d9f1ccb3f05994be56189205dcb558551e056c2f3db03a79182d" ["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance>>"] n_cases = 65536 @@ -120,4 +120,4 @@ hash = "b41d6c6ec560bde9caf2e206526864c618a0721af367585a1719617ca7ce9291" ["commonware_storage::qmdb::sync::target::tests::conformance::CodecConformance>"] n_cases = 65536 -hash = "48e2131a0dffcb40a0a29581aa7bb0e39160212117eadb661b629030a58373d6" +hash = "252e8350e4c38a2e0e5aa8679e474cb9a788fe88bfa5811a59c3220bff4afca0" diff --git a/storage/src/freezer/mod.rs b/storage/src/freezer/mod.rs index 4ebbeb848b..0a3d942a6c 100644 --- a/storage/src/freezer/mod.rs +++ b/storage/src/freezer/mod.rs @@ -1016,19 +1016,22 @@ mod tests { .unwrap(); // Insert keys to trigger resize + // key0 -> entry 0, key2 -> entry 1 freezer.put(test_key("key0"), 0).await.unwrap(); - freezer.put(test_key("key1"), 1).await.unwrap(); + freezer.put(test_key("key2"), 1).await.unwrap(); freezer.sync().await.unwrap(); // should start resize // Verify resize started assert!(freezer.resizing().is_some()); // Insert during resize (to first entry) - freezer.put(test_key("key2"), 2).await.unwrap(); + // key6 -> entry 0 + freezer.put(test_key("key6"), 2).await.unwrap(); assert!(context.encode().contains("unnecessary_writes_total 1")); assert_eq!(freezer.resizable(), 3); // Insert another key (to unmodified entry) + // key3 -> entry 1 freezer.put(test_key("key3"), 3).await.unwrap(); assert!(context.encode().contains("unnecessary_writes_total 1")); assert_eq!(freezer.resizable(), 3); @@ -1039,17 +1042,21 @@ mod tests { assert_eq!(freezer.resizable(), 2); // More inserts + // key4 -> entry 1, key7 -> entry 0 freezer.put(test_key("key4"), 4).await.unwrap(); - freezer.put(test_key("key5"), 5).await.unwrap(); + freezer.put(test_key("key7"), 5).await.unwrap(); freezer.sync().await.unwrap(); // Another resize should've started assert!(freezer.resizing().is_some()); // Verify all can be retrieved during resize - for i in 0..6 { - let key = test_key(&format!("key{i}")); - assert_eq!(freezer.get(Identifier::Key(&key)).await.unwrap(), Some(i)); + let keys = ["key0", "key2", "key6", "key3", "key4", "key7"]; + for (i, k) in keys.iter().enumerate() { + assert_eq!( + freezer.get(Identifier::Key(&test_key(k))).await.unwrap(), + Some(i as i32) + ); } // Sync until resize completes @@ -1090,8 +1097,9 @@ mod tests { .unwrap(); // Insert keys to trigger resize + // key0 -> entry 0, key2 -> entry 1 freezer.put(test_key("key0"), 0).await.unwrap(); - freezer.put(test_key("key1"), 1).await.unwrap(); + freezer.put(test_key("key2"), 1).await.unwrap(); let checkpoint = freezer.sync().await.unwrap(); // Verify resize started diff --git a/storage/src/freezer/storage.rs b/storage/src/freezer/storage.rs index 5b1c133f8f..b434200965 100644 --- a/storage/src/freezer/storage.rs +++ b/storage/src/freezer/storage.rs @@ -7,6 +7,7 @@ use crate::{ }; use bytes::{Buf, BufMut}; use commonware_codec::{Codec, Encode, FixedSize, Read, ReadExt, Write as CodecWrite}; +use commonware_cryptography::{crc32, Crc32, Hasher}; use commonware_runtime::{buffer, Blob, Clock, Metrics, Storage}; use commonware_utils::{Array, Span}; use futures::future::{try_join, try_join_all}; @@ -197,12 +198,12 @@ impl Entry { /// Compute a checksum for [Entry]. fn compute_crc(epoch: u64, section: u64, position: u64, added: u8) -> u32 { - let mut hasher = crc32fast::Hasher::new(); + let mut hasher = Crc32::new(); hasher.update(&epoch.to_be_bytes()); hasher.update(§ion.to_be_bytes()); hasher.update(&position.to_be_bytes()); hasher.update(&added.to_be_bytes()); - hasher.finalize() + hasher.finalize().as_u32() } /// Create a new [Entry]. @@ -228,7 +229,7 @@ impl Entry { } impl FixedSize for Entry { - const SIZE: usize = u64::SIZE + u64::SIZE + u64::SIZE + u8::SIZE + u32::SIZE; + const SIZE: usize = u64::SIZE + u64::SIZE + u64::SIZE + u8::SIZE + crc32::Digest::SIZE; } impl CodecWrite for Entry { @@ -799,7 +800,7 @@ impl Freezer { /// /// To determine the appropriate entry, we AND the key's hash with the current table size. fn table_index(&self, key: &K) -> u32 { - let hash = crc32fast::hash(key.as_ref()); + let hash = Crc32::checksum(key.as_ref()); hash & (self.table_size - 1) } diff --git a/storage/src/journal/segmented/glob.rs b/storage/src/journal/segmented/glob.rs index 19c66c2a84..50229cbb23 100644 --- a/storage/src/journal/segmented/glob.rs +++ b/storage/src/journal/segmented/glob.rs @@ -30,6 +30,7 @@ use super::manager::{Config as ManagerConfig, Manager, WriteFactory}; use crate::journal::Error; use bytes::BufMut; use commonware_codec::{Codec, FixedSize}; +use commonware_cryptography::{crc32, Crc32}; use commonware_runtime::{Blob as _, Error as RError, Metrics, Storage}; use std::{io::Cursor, num::NonZeroUsize}; use zstd::{bulk::compress, decode_all}; @@ -95,15 +96,15 @@ impl Glob { let encoded = value.encode(); let mut compressed = compress(&encoded, level as i32).map_err(|_| Error::CompressionFailed)?; - let checksum = crc32fast::hash(&compressed); + let checksum = Crc32::checksum(&compressed); compressed.put_u32(checksum); compressed } else { // Uncompressed: pre-allocate exact size to avoid copying - let entry_size = value.encode_size() + u32::SIZE; + let entry_size = value.encode_size() + crc32::Digest::SIZE; let mut buf = Vec::with_capacity(entry_size); value.write(&mut buf); - let checksum = crc32fast::hash(&buf); + let checksum = Crc32::checksum(&buf); buf.put_u32(checksum); buf }; @@ -134,17 +135,17 @@ impl Glob { let buf = buf.as_ref(); // Entry format: [compressed_data] [crc32 (4 bytes)] - if buf.len() < u32::SIZE { + if buf.len() < crc32::Digest::SIZE { return Err(Error::Runtime(RError::BlobInsufficientLength)); } - let data_len = buf.len() - u32::SIZE; + let data_len = buf.len() - crc32::Digest::SIZE; let compressed_data = &buf[..data_len]; let stored_checksum = u32::from_be_bytes(buf[data_len..].try_into().expect("checksum is 4 bytes")); // Verify checksum - let checksum = crc32fast::hash(compressed_data); + let checksum = Crc32::checksum(compressed_data); if checksum != stored_checksum { return Err(Error::ChecksumMismatch(stored_checksum, checksum)); } diff --git a/storage/src/journal/segmented/oversized.rs b/storage/src/journal/segmented/oversized.rs index baa8c864df..26711b5a12 100644 --- a/storage/src/journal/segmented/oversized.rs +++ b/storage/src/journal/segmented/oversized.rs @@ -442,6 +442,7 @@ mod tests { use super::*; use bytes::{Buf, BufMut}; use commonware_codec::{FixedSize, Read, ReadExt, Write}; + use commonware_cryptography::Crc32; use commonware_macros::test_traced; use commonware_runtime::{buffer::PoolRef, deterministic, Blob as _, Runner}; use commonware_utils::{NZUsize, NZU16}; @@ -2487,7 +2488,7 @@ mod tests { // Build page-level CRC record (12 bytes): // len1 (2) + crc1 (4) + len2 (2) + crc2 (4) - let crc = crc32fast::hash(&entry_data); + let crc = Crc32::checksum(&entry_data); let len1 = TestEntry::SIZE as u16; let mut crc_record = Vec::new(); crc_record.extend_from_slice(&len1.to_be_bytes()); // len1 diff --git a/storage/src/metadata/storage.rs b/storage/src/metadata/storage.rs index d4cee20853..31a51a7838 100644 --- a/storage/src/metadata/storage.rs +++ b/storage/src/metadata/storage.rs @@ -1,6 +1,7 @@ use super::{Config, Error}; use bytes::BufMut; use commonware_codec::{Codec, FixedSize, ReadExt}; +use commonware_cryptography::{crc32, Crc32}; use commonware_runtime::{ telemetry::metrics::status::GaugeExt, Blob, Clock, Error as RError, Metrics, Storage, }; @@ -152,7 +153,7 @@ impl Metadata { // Verify integrity. // // 8 bytes for version + 4 bytes for checksum. - if buf.len() < 12 { + if buf.len() < 8 + crc32::Digest::SIZE { // Truncate and return none warn!( blob = index, @@ -165,10 +166,10 @@ impl Metadata { } // Extract checksum - let checksum_index = buf.len() - 4; + let checksum_index = buf.len() - crc32::Digest::SIZE; let stored_checksum = u32::from_be_bytes(buf.as_ref()[checksum_index..].try_into().unwrap()); - let computed_checksum = crc32fast::hash(&buf.as_ref()[..checksum_index]); + let computed_checksum = Crc32::checksum(&buf.as_ref()[..checksum_index]); if stored_checksum != computed_checksum { // Truncate and return none warn!( @@ -371,8 +372,8 @@ impl Metadata { writes.push(target.blob.write_at(version.as_slice().into(), 0)); // Update checksum - let checksum_index = target.data.len() - 4; - let checksum = crc32fast::hash(&target.data[..checksum_index]).to_be_bytes(); + let checksum_index = target.data.len() - crc32::Digest::SIZE; + let checksum = Crc32::checksum(&target.data[..checksum_index]).to_be_bytes(); target.data[checksum_index..].copy_from_slice(&checksum); writes.push( target @@ -402,7 +403,7 @@ impl Metadata { value.write(&mut next_data); lengths.insert(key.clone(), Info::new(start, value.encode_size())); } - next_data.put_u32(crc32fast::hash(&next_data[..])); + next_data.put_u32(Crc32::checksum(&next_data[..])); // Persist changes target.blob.write_at(next_data.clone(), 0).await?; diff --git a/storage/src/ordinal/mod.rs b/storage/src/ordinal/mod.rs index 193dc609c5..e840dfc4e7 100644 --- a/storage/src/ordinal/mod.rs +++ b/storage/src/ordinal/mod.rs @@ -135,6 +135,7 @@ mod tests { use super::*; use bytes::{Buf, BufMut}; use commonware_codec::{FixedSize, Read, ReadExt, Write}; + use commonware_cryptography::Crc32; use commonware_macros::{test_group, test_traced}; use commonware_runtime::{deterministic, Blob, Metrics, Runner, Storage}; use commonware_utils::{bitmap::BitMap, hex, sequence::FixedBytes, NZUsize, NZU64}; @@ -911,7 +912,7 @@ mod tests { // Write a valid record after the zeros let mut valid_record = vec![44u8; 32]; - let crc = crc32fast::hash(&valid_record); + let crc = Crc32::checksum(&valid_record); valid_record.extend_from_slice(&crc.to_be_bytes()); blob.write_at(valid_record, 36 * 5).await.unwrap(); diff --git a/storage/src/ordinal/storage.rs b/storage/src/ordinal/storage.rs index 68d62d0539..56c67fb095 100644 --- a/storage/src/ordinal/storage.rs +++ b/storage/src/ordinal/storage.rs @@ -2,6 +2,7 @@ use super::{Config, Error}; use crate::{kv, rmap::RMap, Persistable}; use bytes::{Buf, BufMut}; use commonware_codec::{CodecFixed, Encode, FixedSize, Read, ReadExt, Write as CodecWrite}; +use commonware_cryptography::{crc32, Crc32}; use commonware_runtime::{ buffer::{Read as ReadBuffer, Write}, Blob, Clock, Error as RError, Metrics, Storage, @@ -24,17 +25,17 @@ struct Record> { impl> Record { fn new(value: V) -> Self { - let crc = crc32fast::hash(&value.encode()); + let crc = Crc32::checksum(&value.encode()); Self { value, crc } } fn is_valid(&self) -> bool { - self.crc == crc32fast::hash(&self.value.encode()) + self.crc == Crc32::checksum(&self.value.encode()) } } impl> FixedSize for Record { - const SIZE: usize = V::SIZE + u32::SIZE; + const SIZE: usize = V::SIZE + crc32::Digest::SIZE; } impl> CodecWrite for Record {