libdeflate (#12741)

.gitmodules

@@ -82,3 +82,7 @@ url = https://github.com/oven-sh/zig
 depth = 1
 shallow = true
 fetchRecurseSubmodules = false
+[submodule "src/deps/libdeflate"]
+path = src/deps/libdeflate
+url = https://github.com/ebiggers/libdeflate
+ignore = "dirty"

CMakeLists.txt

@@ -310,6 +310,7 @@ endif()
 # -- Build Flags --
 option(USE_STATIC_SQLITE "Statically link SQLite?" ${DEFAULT_ON_UNLESS_APPLE})
 option(USE_CUSTOM_ZLIB "Use Bun's recommended version of zlib" ON)
+option(USE_CUSTOM_LIBDEFLATE "Use Bun's recommended version of libdeflate" ON)
 option(USE_CUSTOM_BORINGSSL "Use Bun's recommended version of BoringSSL" ON)
 option(USE_CUSTOM_LIBARCHIVE "Use Bun's recommended version of libarchive" ON)
 option(USE_CUSTOM_MIMALLOC "Use Bun's recommended version of Mimalloc" ON)
@@ -1358,6 +1359,19 @@ else()
     target_link_libraries(${bun} PRIVATE LibArchive::LibArchive)
 endif()
 
+if(USE_CUSTOM_LIBDEFLATE)
+    include_directories(${BUN_DEPS_DIR}/libdeflate)
+
+    if(WIN32)
+        target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/deflate.lib")
+    else()
+        target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/libdeflate.a")
+    endif()
+else()
+    find_package(LibDeflate REQUIRED)
+    target_link_libraries(${bun} PRIVATE LibDeflate::LibDeflate)
+endif()
+
 if(USE_CUSTOM_MIMALLOC)
     include_directories(${BUN_DEPS_DIR}/mimalloc/include)
 

Dockerfile

@@ -263,6 +263,27 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \
   && bash ./scripts/build-zlib.sh && rm -rf src/deps/zlib scripts
 
 
+FROM bun-base as libdeflate
+
+ARG BUN_DIR
+ARG CPU_TARGET
+ENV CPU_TARGET=${CPU_TARGET}
+ARG CCACHE_DIR=/ccache
+ENV CCACHE_DIR=${CCACHE_DIR}
+
+COPY Makefile ${BUN_DIR}/Makefile
+COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt
+COPY scripts ${BUN_DIR}/scripts
+COPY src/deps/libdeflate ${BUN_DIR}/src/deps/libdeflate
+COPY package.json bun.lockb Makefile .gitmodules ${BUN_DIR}/
+
+WORKDIR $BUN_DIR
+
+RUN --mount=type=cache,target=${CCACHE_DIR} \
+  cd $BUN_DIR \
+  && bash ./scripts/build-libdeflate.sh && rm -rf src/deps/libdeflate scripts
+
+
 FROM bun-base as libarchive
 
 ARG BUN_DIR
@@ -412,6 +433,9 @@ COPY src ${BUN_DIR}/src
 COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt
 COPY src/deps/boringssl/include ${BUN_DIR}/src/deps/boringssl/include
 
+# for uWebSockets
+COPY src/deps/libdeflate ${BUN_DIR}/src/deps/libdeflate
+
 ARG CCACHE_DIR=/ccache
 ENV CCACHE_DIR=${CCACHE_DIR}
 
@@ -516,6 +540,7 @@ COPY src/symbols.dyn src/linker.lds ${BUN_DIR}/src/
 
 COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt
 COPY --from=zlib ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/
+COPY --from=libdeflate ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/
 COPY --from=libarchive ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/
 COPY --from=boringssl ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/
 COPY --from=lolhtml ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/

LICENSE.md

@@ -34,6 +34,8 @@ Bun statically links these libraries:
 | [`c-ares`](https://github.com/c-ares/c-ares) | MIT licensed |
 | [`libicu`](https://github.com/unicode-org/icu) 72 | [license here](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE) |
 | [`libbase64`](https://github.com/aklomp/base64/blob/master/LICENSE) | BSD 2-Clause |
+| [`libuv`](https://github.com/libuv/libuv) (on Windows) | MIT |
+| [`libdeflate`](https://github.com/ebiggers/libdeflate) | MIT |
 | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets) | Apache 2.0 licensed |
 | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed |
 

bench/gzip/bun.js

@@ -1,20 +1,43 @@
-import { run, bench } from "mitata";
+import { run, bench, group } from "mitata";
 import { gzipSync, gunzipSync } from "bun";
 
-const data = new TextEncoder().encode("Hello World!".repeat(9999));
+const data = await Bun.file(require.resolve("@babel/standalone/babel.min.js")).arrayBuffer();
 
 const compressed = gzipSync(data);
 
-bench(`roundtrip - "Hello World!".repeat(9999))`, () => {
-  gunzipSync(gzipSync(data));
+const libraries = ["zlib"];
+if (Bun.semver.satisfies(Bun.version.replaceAll("-debug", ""), ">=1.1.21")) {
+  libraries.push("libdeflate");
+}
+const options = { library: undefined };
+const benchFn = (name, fn) => {
+  if (libraries.length > 1) {
+    group(name, () => {
+      for (const library of libraries) {
+        bench(library, () => {
+          options.library = library;
+          fn();
+        });
+      }
+    });
+  } else {
+    options.library = libraries[0];
+    bench(name, () => {
+      fn();
+    });
+  }
+};
+
+benchFn(`roundtrip - @babel/standalone/babel.min.js`, () => {
+  gunzipSync(gzipSync(data, options), options);
 });
 
-bench(`gzipSync("Hello World!".repeat(9999)))`, () => {
-  gzipSync(data);
+benchFn(`gzipSync(@babel/standalone/babel.min.js`, () => {
+  gzipSync(data, options);
 });
 
-bench(`gunzipSync("Hello World!".repeat(9999)))`, () => {
-  gunzipSync(compressed);
+benchFn(`gunzipSync(@babel/standalone/babel.min.js`, () => {
+  gunzipSync(compressed, options);
 });
 
 await run();

bench/gzip/node.mjs

@@ -1,19 +1,22 @@
 import { run, bench } from "mitata";
 import { gzipSync, gunzipSync } from "zlib";
+import { createRequire } from "module";
+import { readFileSync } from "fs";
 
-const data = new TextEncoder().encode("Hello World!".repeat(9999));
+const require = createRequire(import.meta.url);
+const data = readFileSync(require.resolve("@babel/standalone/babel.min.js"));
 
 const compressed = gzipSync(data);
 
-bench(`roundtrip - "Hello World!".repeat(9999))`, () => {
+bench(`roundtrip - @babel/standalone/babel.min.js)`, () => {
   gunzipSync(gzipSync(data));
 });
 
-bench(`gzipSync("Hello World!".repeat(9999)))`, () => {
+bench(`gzipSync(@babel/standalone/babel.min.js))`, () => {
   gzipSync(data);
 });
 
-bench(`gunzipSync("Hello World!".repeat(9999)))`, () => {
+bench(`gunzipSync(@babel/standalone/babel.min.js))`, () => {
   gunzipSync(compressed);
 });
 

bench/gzip/package.json

@@ -7,5 +7,8 @@
     "bench:node": "$NODE node.mjs",
     "bench:deno": "$DENO run -A --unstable deno.js",
     "bench": "bun run bench:bun && bun run bench:node && bun run bench:deno"
+  },
+  "dependencies": {
+    "@babel/standalone": "7.24.10"
   }
 }

packages/bun-types/bun.d.ts

@@ -3481,6 +3481,13 @@ declare module "bun" {
      * Filtered data consists mostly of small values with a somewhat random distribution.
      */
     strategy?: number;
+
+    library?: "zlib";
+  }
+
+  interface LibdeflateCompressionOptions {
+    level?: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12;
+    library?: "libdeflate";
   }
 
   /**
@@ -3489,26 +3496,38 @@ declare module "bun" {
    * @param options Compression options to use
    * @returns The output buffer with the compressed data
    */
-  function deflateSync(data: Uint8Array | string | ArrayBuffer, options?: ZlibCompressionOptions): Uint8Array;
+  function deflateSync(
+    data: Uint8Array | string | ArrayBuffer,
+    options?: ZlibCompressionOptions | LibdeflateCompressionOptions,
+  ): Uint8Array;
   /**
    * Compresses a chunk of data with `zlib` GZIP algorithm.
    * @param data The buffer of data to compress
    * @param options Compression options to use
    * @returns The output buffer with the compressed data
    */
-  function gzipSync(data: Uint8Array | string | ArrayBuffer, options?: ZlibCompressionOptions): Uint8Array;
+  function gzipSync(
+    data: Uint8Array | string | ArrayBuffer,
+    options?: ZlibCompressionOptions | LibdeflateCompressionOptions,
+  ): Uint8Array;
   /**
    * Decompresses a chunk of data with `zlib` INFLATE algorithm.
    * @param data The buffer of data to decompress
    * @returns The output buffer with the decompressed data
    */
-  function inflateSync(data: Uint8Array | string | ArrayBuffer): Uint8Array;
+  function inflateSync(
+    data: Uint8Array | string | ArrayBuffer,
+    options?: ZlibCompressionOptions | LibdeflateCompressionOptions,
+  ): Uint8Array;
   /**
    * Decompresses a chunk of data with `zlib` GUNZIP algorithm.
    * @param data The buffer of data to decompress
    * @returns The output buffer with the decompressed data
    */
-  function gunzipSync(data: Uint8Array | string | ArrayBuffer): Uint8Array;
+  function gunzipSync(
+    data: Uint8Array | string | ArrayBuffer,
+    options?: ZlibCompressionOptions | LibdeflateCompressionOptions,
+  ): Uint8Array;
 
   type Target =
     /**

packages/bun-uws/src/PerMessageDeflate.h

@@ -20,6 +20,8 @@
 #ifndef UWS_PERMESSAGEDEFLATE_H
 #define UWS_PERMESSAGEDEFLATE_H
 
+#define UWS_USE_LIBDEFLATE 1
+
 #include <cstdint>
 #include <cstring>
 
@@ -134,6 +136,9 @@ struct ZlibContext {
 
 struct DeflationStream {
     z_stream deflationStream = {};
+#ifdef UWS_USE_LIBDEFLATE
+    unsigned char reset_buffer[4096 + 1];
+#endif
 
     DeflationStream(CompressOptions compressOptions) {
 
@@ -154,13 +159,11 @@ struct DeflationStream {
         /* Run a fast path in case of shared_compressor */
         if (reset) {
             size_t written = 0;
-            static unsigned char buf[1024 + 1];
-
-            written = libdeflate_deflate_compress(zlibContext->compressor, raw.data(), raw.length(), buf, 1024);
+            written = libdeflate_deflate_compress(zlibContext->compressor, raw.data(), raw.length(), reset_buffer, 4096);
 
             if (written) {
-                memcpy(&buf[written], "\x00", 1);
-                return std::string_view((char *) buf, written + 1);
+                memcpy(&reset_buffer[written], "\x00", 1);
+                return std::string_view((char *) reset_buffer, written + 1);
             }
         }
 #endif
@@ -214,6 +217,9 @@ struct DeflationStream {
 
 struct InflationStream {
     z_stream inflationStream = {};
+#ifdef UWS_USE_LIBDEFLATE
+    char buf[4096];
+#endif
 
     InflationStream(CompressOptions compressOptions) {
         /* Inflation windowBits are the top 8 bits of the 16 bit compressOptions */
@@ -230,13 +236,12 @@ struct InflationStream {
 #ifdef UWS_USE_LIBDEFLATE
         /* Try fast path first */
         size_t written = 0;
-        static char buf[1024];
 
         /* We have to pad 9 bytes and restore those bytes when done since 9 is more than 6 of next WebSocket message */
         char tmp[9];
         memcpy(tmp, (char *) compressed.data() + compressed.length(), 9);
         memcpy((char *) compressed.data() + compressed.length(), "\x00\x00\xff\xff\x01\x00\x00\xff\xff", 9);
-        libdeflate_result res = libdeflate_deflate_decompress(zlibContext->decompressor, compressed.data(), compressed.length() + 9, buf, 1024, &written);
+        libdeflate_result res = libdeflate_deflate_decompress(zlibContext->decompressor, compressed.data(), compressed.length() + 9, buf, 4096, &written);
         memcpy((char *) compressed.data() + compressed.length(), tmp, 9);
 
         if (res == 0) {

scripts/all-dependencies.ps1

@@ -79,6 +79,10 @@ Build-Dependency `
   -Script "lshpack" `
   -Outputs @("lshpack.lib")
 
+Build-Dependency `
+  -Script "libdeflate" `
+  -Outputs @("deflate.lib")
+
 if (!($Script:DidAnything)) {
   Write-Host "(run with -Force to rebuild all)"
 }

scripts/all-dependencies.sh

@@ -3,7 +3,7 @@ set -eo pipefail
 source "$(dirname -- "${BASH_SOURCE[0]}")/env.sh"
 
 if [[ "$CI" ]]; then
-  $(dirname -- "${BASH_SOURCE[0]}")/update-submodules.sh
+    $(dirname -- "${BASH_SOURCE[0]}")/update-submodules.sh
 fi
 
 FORCE=
@@ -92,6 +92,7 @@ dep mimalloc mimalloc libmimalloc.a libmimalloc.o
 dep tinycc tinycc libtcc.a
 dep zlib zlib libz.a
 dep zstd zstd libzstd.a
+dep libdeflate libdeflate libdeflate.a
 dep ls-hpack lshpack liblshpack.a
 
 if [ "$BUILT_ANY" -eq 0 ]; then

scripts/build-libdeflate.ps1

@@ -0,0 +1,16 @@
+$ErrorActionPreference = 'Stop'  # Setting strict mode, similar to 'set -euo pipefail' in bash
+. (Join-Path $PSScriptRoot "env.ps1")
+
+Push-Location (Join-Path $BUN_DEPS_DIR 'libdeflate')
+try {
+  Remove-Item CMakeCache.txt, CMakeFiles, build -Recurse -ErrorAction SilentlyContinue
+  mkdir -Force build
+
+  Run cmake -S "." -B build @CMAKE_FLAGS -DLIBDEFLATE_BUILD_STATIC_LIB=ON -DLIBDEFLATE_BUILD_SHARED_LIB=OFF -DLIBDEFLATE_BUILD_GZIP=OFF
+  Run cmake --build build --clean-first --config Release
+
+  # In https://github.com/ebiggers/libdeflate/releases/tag/v1.20, it's outputting libdeflate.a even on Windows
+  Copy-Item build/deflatestatic.lib $BUN_DEPS_OUT_DIR/deflate.lib
+  Write-Host "-> deflate.lib"
+} finally { Pop-Location }
+

scripts/build-libdeflate.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -exo pipefail
+source $(dirname -- "${BASH_SOURCE[0]}")/env.sh
+
+mkdir -p $BUN_DEPS_OUT_DIR
+cd $BUN_DEPS_DIR/libdeflate
+rm -rf build CMakeCache.txt CMakeFiles
+cmake "${CMAKE_FLAGS[@]}" -DLIBDEFLATE_BUILD_STATIC_LIB=ON -DLIBDEFLATE_BUILD_SHARED_LIB=OFF -DLIBDEFLATE_BUILD_GZIP=OFF -B build -S . -G Ninja
+ninja libdeflate.a -C build
+cp build/libdeflate.a $BUN_DEPS_OUT_DIR/libdeflate.a

scripts/write-versions.sh

@@ -12,6 +12,7 @@ TINYCC=$(git rev-parse HEAD:./src/deps/tinycc)
 C_ARES=$(git rev-parse HEAD:./src/deps/c-ares)
 ZSTD=$(git rev-parse HEAD:./src/deps/zstd)
 LSHPACK=$(git rev-parse HEAD:./src/deps/ls-hpack)
+LIBDEFLATE=$(git rev-parse HEAD:./src/deps/libdeflate)
 
 rm -rf src/generated_versions_list.zig
 echo "// AUTO-GENERATED FILE. Created via .scripts/write-versions.sh" >src/generated_versions_list.zig
@@ -26,6 +27,7 @@ echo "pub const zlib = \"$ZLIB_VERSION\";" >>src/generated_versions_list.zig
 echo "pub const tinycc = \"$TINYCC\";" >>src/generated_versions_list.zig
 echo "pub const lolhtml = \"$LOLHTML\";" >>src/generated_versions_list.zig
 echo "pub const c_ares = \"$C_ARES\";" >>src/generated_versions_list.zig
+echo "pub const libdeflate = \"$LIBDEFLATE\";" >>src/generated_versions_list.zig
 echo "pub const zstd = \"$ZSTD\";" >>src/generated_versions_list.zig
 echo "pub const lshpack = \"$LSHPACK\";" >>src/generated_versions_list.zig
 echo "" >>src/generated_versions_list.zig