From 0b33d8151ecfde115220914bf18db10e1ba0c411 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Sat, 23 Dec 2023 21:01:51 +0100 Subject: [PATCH] Integrate zlib-ng --- pyproject.toml | 3 ++- src/xopen/__init__.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6f27495..02a005e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,8 @@ classifiers = [ requires-python = ">=3.8" dynamic = ["version"] dependencies = [ - 'isal>=1.4.1; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"' + 'isal>=1.4.1; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"', + 'zlib-ng>=0.4.0; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"' ] [project.urls] diff --git a/src/xopen/__init__.py b/src/xopen/__init__.py index 4c317ab..c06ac56 100644 --- a/src/xopen/__init__.py +++ b/src/xopen/__init__.py @@ -65,6 +65,13 @@ isal_zlib = None igzip_threaded = None +try: + from zlib_ng import gzip_ng, gzip_ng_threaded, zlib_ng +except ImportError: + gzip_ng = None + gzip_ng_threaded = None + zlib_ng = None + try: import zstandard # type: ignore except ImportError: @@ -1062,6 +1069,14 @@ def _open_gz( # noqa: C901 ) except ValueError: # Wrong compression level pass + if gzip_ng_threaded and threads != 0: + return gzip_ng_threaded.open( + filename, + mode, + zlib_ng.Z_DEFAULT_COMPRESSION if compresslevel is None else compresslevel, + **text_mode_kwargs, + threads=1 if threads is None else threads, + ) if threads != 0: try: if "r" in mode: @@ -1078,6 +1093,8 @@ def _open_gz( # noqa: C901 if "r" in mode: if igzip is not None: return igzip.open(filename, mode, **text_mode_kwargs) + elif gzip_ng is not None: + return gzip_ng.open(filename, mode, **text_mode_kwargs) return gzip.open(filename, mode, **text_mode_kwargs) g = _open_reproducible_gzip( @@ -1119,6 +1136,14 @@ def _open_reproducible_gzip(filename, mode, compresslevel): except ValueError: # Compression level not supported, move to built-in gzip. pass + elif gzip_ng is not None: + gzip_file = gzip_ng.GzipNGFile( + **kwargs, + compresslevel=zlib_ng.Z_DEFAULT_COMPRESSION + if compresslevel is None + else compresslevel, + ) + if gzip_file is None: gzip_file = gzip.GzipFile( **kwargs,