Merge pull request #135 from pycompression/zlib-ng

Add zlib-ng

.github/workflows/ci.yml

@@ -36,23 +36,23 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9"]
         optional-deps: [true]
-        with-python-isal: [true]
+        with-libs: [true]
         include:
         - os: macos-latest
           python-version: "3.10"
           optional-deps: true
         - os: ubuntu-20.04
           python-version: "3.10"
-          with-python-isal: false
+          with-libs: false
           optional-deps: false
         - os: ubuntu-20.04
           python-version: "3.10"
-          with-python-isal: false
+          with-libs: false
           optional-deps: true
         - os: ubuntu-20.04
           python-version: "3.10"
           optional-deps: false
-          with-python-isal: false
+          with-libs: false
           with-zstandard: true
         - os: windows-latest
           python-version: "3.10"
@@ -77,10 +77,10 @@ jobs:
       run: python -m pip install tox
     - name: Test
       run: tox -e py
-      if: matrix.with-python-isal
-    - name: Test without python-isal
-      run: tox -e no-isal
-      if: true && !matrix.with-python-isal
+      if: matrix.with-libs
+    - name: Test without python-isal and python-zlib-ng
+      run: tox -e no-libs
+      if: true && !matrix.with-libs
     - name: Test with zstandard
       if: matrix.with-zstandard
       run: tox -e zstd

README.rst

@@ -111,6 +111,10 @@ To ensure that you get the correct ``zstandard`` version, you can specify the ``
 Changelog
 ---------
 
+in-development
+~~~~~~~~~~~~~~~~~~~
+* #135: xopen now uses zlib-ng when available and applicable.
+
 v1.8.0 (2023-11-03)
 ~~~~~~~~~~~~~~~~~~~
 * #131: xopen now defers to the ``isal.igzip_threaded`` module rather than

pyproject.toml

@@ -19,7 +19,8 @@ classifiers = [
 requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = [
-    'isal>=1.4.1; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"'
+    'isal>=1.4.1; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"',
+    'zlib-ng>=0.4.0; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"'
 ]
 
 [project.urls]

src/xopen/__init__.py

@@ -33,6 +33,7 @@
 import subprocess
 import tempfile
 import time
+import zlib
 from abc import ABC, abstractmethod
 from subprocess import Popen, PIPE, DEVNULL
 from typing import (
@@ -57,6 +58,9 @@
 igzip: Optional[ModuleType]
 isal_zlib: Optional[ModuleType]
 igzip_threaded: Optional[ModuleType]
+zlib_ng: Optional[ModuleType]
+gzip_ng: Optional[ModuleType]
+gzip_ng_threaded: Optional[ModuleType]
 
 try:
     from isal import igzip, igzip_threaded, isal_zlib
@@ -65,6 +69,13 @@
     isal_zlib = None
     igzip_threaded = None
 
+try:
+    from zlib_ng import gzip_ng, gzip_ng_threaded, zlib_ng
+except ImportError:
+    gzip_ng = None
+    gzip_ng_threaded = None
+    zlib_ng = None
+
 try:
     import zstandard  # type: ignore
 except ImportError:
@@ -1062,6 +1073,27 @@ def _open_gz(  # noqa: C901
             )
         except ValueError:  # Wrong compression level
             pass
+    if gzip_ng_threaded and zlib_ng and threads != 0:
+        try:
+            if compresslevel is None:
+                level = zlib_ng.Z_DEFAULT_COMPRESSION
+            elif compresslevel == 1:
+                # zlib-ng level 1 is 50% bigger than zlib level 1.
+                # This will be wildly outside user ballpark expectations, so
+                # increase the level
+                level = 2
+            else:
+                level = compresslevel
+
+            return gzip_ng_threaded.open(
+                filename,
+                mode,
+                level,
+                **text_mode_kwargs,
+                threads=1 if threads is None else threads,
+            )
+        except zlib_ng.error:  # Bad compression level
+            pass
     if threads != 0:
         try:
             if "r" in mode:
@@ -1078,6 +1110,8 @@ def _open_gz(  # noqa: C901
     if "r" in mode:
         if igzip is not None:
             return igzip.open(filename, mode, **text_mode_kwargs)
+        elif gzip_ng is not None:
+            return gzip_ng.open(filename, mode, **text_mode_kwargs)
         return gzip.open(filename, mode, **text_mode_kwargs)
 
     g = _open_reproducible_gzip(
@@ -1119,12 +1153,23 @@ def _open_reproducible_gzip(filename, mode, compresslevel):
         except ValueError:
             # Compression level not supported, move to built-in gzip.
             pass
+    elif gzip_ng is not None:
+        if compresslevel == 1:
+            level = 2
+        elif compresslevel is None:
+            level = zlib_ng.Z_DEFAULT_COMPRESSION
+        else:
+            level = compresslevel
+        gzip_file = gzip_ng.GzipNGFile(**kwargs, compresslevel=level)
+
     if gzip_file is None:
         gzip_file = gzip.GzipFile(
             **kwargs,
             # Override gzip.open's default of 9 for consistency
             # with command-line gzip.
-            compresslevel=6 if compresslevel is None else compresslevel,
+            compresslevel=zlib.Z_DEFAULT_COMPRESSION
+            if compresslevel is None
+            else compresslevel,
         )
     # When (I)GzipFile is created with a fileobj instead of a filename,
     # the passed file object is not closed when (I)GzipFile.close()

tests/test_piped.py

@@ -261,7 +261,7 @@ def test_concatenated_gzip_function():
 )
 def test_pipesize_changed(tmp_path, monkeypatch):
     # Higher compression level to avoid opening with threaded opener
-    with xopen(tmp_path / "hello.gz", "wb", compresslevel=5) as f:
+    with PipedGzipWriter(tmp_path / "hello.gz", "wb", compresslevel=5) as f:
         assert isinstance(f, PipedCompressionWriter)
         assert fcntl.fcntl(f._file.fileno(), fcntl.F_GETPIPE_SZ) == _MAX_PIPE_SIZE
 

tox.ini

@@ -21,9 +21,9 @@ deps =
     {[testenv]deps}
     zstandard
 
-[testenv:no-isal]
+[testenv:no-libs]
 commands=
-    pip uninstall -y isal
+    pip uninstall -y isal zlib-ng
     {[testenv]commands}
 
 [testenv:black]