feat(lib)!: change compression from lz4 to brotli (#1064)

lib/Cargo.toml

@@ -21,9 +21,6 @@ name = "lib"
 harness = false
 
 [dependencies]
-lz4_flex = { version = "0.11.3", default-features = false, features = [
-  "frame",
-] }
 byteorder = "1.5.0"
 quick-xml = { version = "0.37.0", features = ["serialize"] }
 rayon = "1.10.0"
@@ -46,6 +43,7 @@ dirs = { version = "5.0.1", optional = true }
 sea-query = { version = "0.32.0", optional = true }
 thiserror = "2.0.7"
 anyhow = "1.0.94"
+brotli = "7.0.0"
 
 [dev-dependencies]
 insta = "1.38.0"

lib/src/compression.rs

@@ -0,0 +1,25 @@
+use std::io::{Read, Write};
+
+const LG_WINDOW_SIZE: u32 = 21;
+const QUALITY: u32 = 11;
+
+pub fn compress(bytes: &[u8]) -> crate::Result<Vec<u8>> {
+    let mut buf = Vec::new();
+    let mut writer = brotli::CompressorWriter::new(&mut buf, 4096, QUALITY, LG_WINDOW_SIZE);
+
+    writer.write_all(bytes).expect("Failed to write data");
+    writer.flush().expect("Failed to flush data");
+
+    drop(writer);
+
+    Ok(buf)
+}
+
+pub fn decompress(bytes: &[u8]) -> crate::Result<Vec<u8>> {
+    let mut buf = Vec::new();
+    let mut reader = brotli::Decompressor::new(bytes, 4096);
+
+    reader.read_to_end(&mut buf).expect("Failed to read data");
+
+    Ok(buf)
+}

lib/src/core/read.rs

@@ -8,9 +8,9 @@ use byteorder::{LittleEndian, ReadBytesExt};
 use rkyv::access_unchecked;
 
 use crate::{
+    compression::decompress,
     constants::{SIGNATURE, VERSION},
     error::Error,
-    lz4::decompress,
     ArchivedDictionary, Dictionary,
 };
 

lib/src/core/write.rs

@@ -2,8 +2,8 @@ use std::fs::read_to_string;
 use std::path::Path;
 use std::{fs::File, io::Write};
 
+use crate::compression::compress;
 use crate::error::Error;
-use crate::lz4::compress;
 use crate::{Dictionary, ToDictionary};
 
 use super::constants::*;

lib/src/error.rs

@@ -10,7 +10,7 @@ pub enum Error {
     FromUtf8(#[from] std::string::FromUtf8Error),
 
     #[error("Failed to compress: {0}")]
-    Compression(#[from] lz4_flex::frame::Error),
+    Compression(String),
 
     #[error("Failed to write the current dictionary: {0}")]
     Write(String),

lib/src/lib.rs

@@ -11,10 +11,10 @@ pub mod json;
 #[cfg(feature = "sql")]
 pub mod sql;
 
+mod compression;
 mod core;
 mod error;
 mod ext;
-mod lz4;
 mod md;
 mod models;