diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28a63499..df3bc7b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,6 +62,11 @@ jobs: - run: cargo fmt -- --check - run: cargo doc --all-features - run: cargo clippy --all-features -- -D warnings + - run: | + if cargo tree --features zlib-rs --no-default-features | grep -q crc32fast; then + echo "zlib-rs uses its own crc implementation, and should not build crc32fast" + exit 1 + fi audit: name: Security Audit diff --git a/Cargo.toml b/Cargo.toml index 8acfa8b5..e6347ba1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ cloudflare-zlib-sys = { version = "0.3.6", optional = true } ## This implementation uses only safe Rust code and doesn't require a C compiler. ## It provides good performance for most use cases while being completely portable. miniz_oxide = { version = "0.8.5", optional = true, default-features = false, features = ["with-alloc", "simd"] } -crc32fast = "1.2.0" +crc32fast = { version = "1.2.0", optional = true } document-features = { version = "0.2", optional = true } [target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies] @@ -48,11 +48,6 @@ default = ["rust_backend"] #! Only one backend should be enabled at a time, or else one will see an unstable order which is currently #! `zlib-ng`, `zlib-rs`, `cloudflare_zlib`, `miniz_oxide` and which may change at any time. -## Use the zlib-rs backend, a pure Rust rewrite of zlib. -## This is the fastest backend overall, providing excellent performance with some `unsafe` code. -## It does not require a C compiler but uses `unsafe` Rust for optimization. -zlib-rs = ["any_zlib", "dep:zlib-rs"] - ## Use the pure Rust `miniz_oxide` backend (default). ## This implementation uses only safe Rust code and doesn't require a C compiler. ## It provides good performance for most use cases while being completely portable. @@ -60,14 +55,22 @@ zlib-rs = ["any_zlib", "dep:zlib-rs"] ## Note that this feature at some point may be switched to use `zlib-rs` instead. rust_backend = ["miniz_oxide", "any_impl"] +## Use the zlib-rs backend, a pure Rust rewrite of zlib. +## This is the fastest backend overall, providing excellent performance with some `unsafe` code. +## It does not require a C compiler but uses `unsafe` Rust for optimization. +zlib-rs = ["any_zlib", "dep:zlib-rs"] # zlib-rs uses its own crc32 implementation. + +## Use the pure Rust `miniz_oxide` backend. +miniz_oxide = ["any_impl", "dep:miniz_oxide", "dep:crc32fast"] + ## Use the system's installed zlib library. ## This is useful when you need compatibility with other C code that uses zlib, ## or when you want to use the system-provided zlib for consistency. -zlib = ["any_c_zlib", "libz-sys"] +zlib = ["any_c_zlib", "libz-sys", "dep:crc32fast"] ## Use the system's installed zlib library with default features enabled. ## Similar to `zlib` but enables additional features from libz-sys. -zlib-default = ["any_c_zlib", "libz-sys/default"] +zlib-default = ["any_c_zlib", "libz-sys/default", "dep:crc32fast"] ## Use zlib-ng in zlib-compat mode via libz-sys. ## This provides zlib-ng's performance improvements while maintaining compatibility. @@ -78,18 +81,18 @@ zlib-default = ["any_c_zlib", "libz-sys/default"] ## See [the libz-sys README](https://github.com/rust-lang/libz-sys/blob/main/README.md) for details. ## To avoid that, use the `"zlib-ng"` feature instead. -zlib-ng-compat = ["zlib", "libz-sys/zlib-ng"] +zlib-ng-compat = ["zlib", "libz-sys/zlib-ng", "dep:crc32fast"] ## Use the high-performance zlib-ng library directly. ## This typically provides better performance than stock zlib and works even when ## other dependencies use zlib. Requires a C compiler. -zlib-ng = ["any_c_zlib", "libz-ng-sys"] +zlib-ng = ["any_c_zlib", "libz-ng-sys", "dep:crc32fast"] ## Use Cloudflare's optimized zlib implementation. ## This provides better performance than stock zlib on x86-64 (with SSE 4.2) and ARM64 (with NEON & CRC). ## * ⚠ Does not support 32-bit CPUs and is incompatible with mingw. ## * ⚠ May cause conflicts if other crates use different zlib versions. -cloudflare_zlib = ["any_c_zlib", "cloudflare-zlib-sys"] +cloudflare_zlib = ["any_c_zlib", "cloudflare-zlib-sys", "dep:crc32fast"] ## Deprecated alias for `rust_backend`, provided for backwards compatibility. ## Use `rust_backend` instead. diff --git a/src/crc.rs b/src/crc.rs index 991cc957..0466bda3 100644 --- a/src/crc.rs +++ b/src/crc.rs @@ -3,62 +3,123 @@ use std::io; use std::io::prelude::*; -use crc32fast::Hasher; +#[cfg(not(feature = "zlib-rs"))] +pub use impl_crc32fast::Crc; -/// The CRC calculated by a [`CrcReader`]. -/// -/// [`CrcReader`]: struct.CrcReader.html -#[derive(Debug, Default)] -pub struct Crc { - amt: u32, - hasher: Hasher, -} +#[cfg(feature = "zlib-rs")] +pub use impl_zlib_rs::Crc; -/// A wrapper around a [`Read`] that calculates the CRC. -/// -/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html -#[derive(Debug)] -pub struct CrcReader { - inner: R, - crc: Crc, -} +#[cfg(not(feature = "zlib-rs"))] +mod impl_crc32fast { + use crc32fast::Hasher; -impl Crc { - /// Create a new CRC. - pub fn new() -> Self { - Self::default() + /// The CRC calculated by a [`CrcReader`]. + /// + /// [`CrcReader`]: struct.CrcReader.html + #[derive(Debug, Default)] + pub struct Crc { + amt: u32, + hasher: Hasher, } - /// Returns the current crc32 checksum. - pub fn sum(&self) -> u32 { - self.hasher.clone().finalize() - } + impl Crc { + /// Create a new CRC. + pub fn new() -> Self { + Self::default() + } - /// The number of bytes that have been used to calculate the CRC. - /// This value is only accurate if the amount is lower than 232. - pub fn amount(&self) -> u32 { - self.amt - } + /// Returns the current crc32 checksum. + pub fn sum(&self) -> u32 { + self.hasher.clone().finalize() + } + + /// The number of bytes that have been used to calculate the CRC. + /// This value is only accurate if the amount is lower than 232. + pub fn amount(&self) -> u32 { + self.amt + } + + /// Update the CRC with the bytes in `data`. + pub fn update(&mut self, data: &[u8]) { + self.amt = self.amt.wrapping_add(data.len() as u32); + self.hasher.update(data); + } + + /// Reset the CRC. + pub fn reset(&mut self) { + self.amt = 0; + self.hasher.reset(); + } - /// Update the CRC with the bytes in `data`. - pub fn update(&mut self, data: &[u8]) { - self.amt = self.amt.wrapping_add(data.len() as u32); - self.hasher.update(data); + /// Combine the CRC with the CRC for the subsequent block of bytes. + pub fn combine(&mut self, additional_crc: &Self) { + self.amt = self.amt.wrapping_add(additional_crc.amt); + self.hasher.combine(&additional_crc.hasher); + } } +} - /// Reset the CRC. - pub fn reset(&mut self) { - self.amt = 0; - self.hasher.reset(); +#[cfg(feature = "zlib-rs")] +mod impl_zlib_rs { + /// The CRC calculated by a [`CrcReader`]. + /// + /// [`CrcReader`]: struct.CrcReader.html + #[derive(Debug, Default)] + pub struct Crc { + consumed: u64, + state: u32, } - /// Combine the CRC with the CRC for the subsequent block of bytes. - pub fn combine(&mut self, additional_crc: &Crc) { - self.amt = self.amt.wrapping_add(additional_crc.amt); - self.hasher.combine(&additional_crc.hasher); + impl Crc { + /// Create a new CRC. + pub fn new() -> Self { + Self::default() + } + + /// Returns the current crc32 checksum. + pub fn sum(&self) -> u32 { + self.state + } + + /// The number of bytes that have been used to calculate the CRC. + /// This value is only accurate if the amount is lower than 232. + pub fn amount(&self) -> u32 { + self.consumed as u32 + } + + /// Update the CRC with the bytes in `data`. + pub fn update(&mut self, data: &[u8]) { + self.consumed = self.consumed.wrapping_add(data.len() as u64); + self.state = zlib_rs::crc32::crc32(self.state, data); + } + + /// Reset the CRC. + pub fn reset(&mut self) { + self.consumed = 0; + self.state = 0 + } + + /// Combine the CRC with the CRC for the subsequent block of bytes. + pub fn combine(&mut self, additional_crc: &Self) { + self.consumed = self.consumed.wrapping_add(additional_crc.consumed); + self.state = zlib_rs::crc32::crc32_combine( + self.state, + additional_crc.state, + additional_crc.consumed, + ); + } } } +/// A wrapper around a [`Read`] that calculates the CRC. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +#[derive(Debug)] +pub struct CrcReader { + inner: R, + crc: Crc, +} + impl CrcReader { /// Create a new `CrcReader`. pub fn new(r: R) -> CrcReader { @@ -173,3 +234,77 @@ impl Write for CrcWriter { self.inner.flush() } } + +#[cfg(test)] +mod tests { + use super::Crc; + + fn crc_of(data: &[u8]) -> Crc { + let mut c = Crc::new(); + c.update(data); + c + } + + fn sum_of(data: &[u8]) -> u32 { + crc_of(data).sum() + } + + #[test] + fn new_is_empty() { + let c = Crc::new(); + assert_eq!(c.amount(), 0); + assert_eq!(c.sum(), 0); + } + + #[test] + fn known_vector_hello() { + assert_eq!(sum_of(b"hello"), 0x3610_A686); + } + + #[test] + fn known_vector_quick_brown_fox() { + assert_eq!( + sum_of(b"The quick brown fox jumps over the lazy dog"), + 0x414F_A339 + ); + } + + #[test] + fn update_is_streaming() { + let mut c = Crc::new(); + c.update(b"hello"); + c.update(b" "); + c.update(b"world"); + + assert_eq!(c.amount(), 11); + assert_eq!(c.sum(), sum_of(b"hello world")); + } + + #[test] + fn reset_restores_initial_state() { + let mut c = Crc::new(); + c.update(b"abc"); + assert_ne!(c.sum(), 0); + assert_eq!(c.amount(), 3); + + c.reset(); + assert_eq!(c.amount(), 0); + assert_eq!(c.sum(), 0); + } + + #[test] + fn combine_matches_concatenation() { + let a = b"hello "; + let b = b"world"; + + let mut ca = crc_of(a); + let cb = crc_of(b); + + ca.combine(&cb); + + dbg!(&ca); + + assert_eq!(ca.amount(), 11); + assert_eq!(ca.sum(), sum_of(b"hello world")); + } +}