Add nthreads param && support compressing array of images

.github/workflows/cibuildwheels.yml

@@ -60,7 +60,7 @@ jobs:
           CIBW_BUILD: 'cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64'
           CIBW_BEFORE_TEST: python -m pip install --upgrade pip && python -m pip install -r requirements-test.txt
           CIBW_TEST_COMMAND: >
-            cmd /V /C "set BLOSC_TRACE=1 && python -m pytest"
+            cmd /V /C "set BLOSC_TRACE=1 && python -m pytest {project}/tests"
 
       - name: Build wheels (Linux / Mac OSX x86_64)
         if: ${{ matrix.os != 'windows-latest'}}
@@ -72,7 +72,7 @@ jobs:
           CIBW_BUILD: 'cp39-* cp310-* cp311-* cp312-*'
           CIBW_BEFORE_TEST: python -m pip install --upgrade pip && python -m pip install -r requirements-test.txt
           CIBW_TEST_COMMAND: >
-            BLOSC_TRACE=1 python -m pytest
+            BLOSC_TRACE=1 python -m pytest {project}/tests
           CIBW_ARCHS_LINUX: ${{ matrix.arch }}
           CIBW_ARCHS_MACOS: "x86_64"
 

README.md

@@ -63,30 +63,30 @@ The following parameters are available for compression for grok, with its defaul
     'tile_offset': (0, 0),
     'quality_mode': None,
     'quality_layers': np.zeros(0, dtype=np.float64),
-    * 'numgbits': 2,  # Equivalent to -N, -guard_bits
     'progression': "LRCP",
     'num_resolutions': 6,
     'codeblock_size': (64, 64),
-    'irreversible':
-    * 'roi_compno': -1,  # Together with 'roi_shift' it is equivalent to -R, -ROI
-    * 'roi_shift': 0,
+    'irreversible': False,
     'precinct_size': (0, 0),
     'offset': (0, 0),
+    'mct': 0,
+    * 'numgbits': 2,  # Equivalent to -N, -guard_bits
+    * 'roi_compno': -1,  # Together with 'roi_shift' it is equivalent to -R, -ROI
+    * 'roi_shift': 0,
     * 'decod_format': GrkFileFmt.GRK_FMT_UNK,
     * 'cod_format': GrkFileFmt.GRK_FMT_UNK,
-    * 'enableTilePartGeneration': False,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
-    'mct': 0,
-    * 'max_cs_size': 0,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
-    * 'max_comp_size': 0,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
     * 'rsiz': GrkProfile.GRK_PROFILE_NONE,  # Equivalent to -Z, -rsiz
     * 'framerate': 0,
     * 'apply_icc_': False,  # Equivalent to -f, -apply_icc
     * 'rateControlAlgorithm': GrkRateControl.BISECT,
+    * 'num_threads': 0,
     * 'deviceId': 0,  # Equivalent to -G, -device_id
     * 'duration': 0,  # Equivalent to -J, -duration
     * 'repeats': 1,  # Equivalent to -e, -repetitions
     * 'verbose': False,
-    * 'sharedMemoryInterface': False,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
+    * 'enableTilePartGeneration': False,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
+    * 'max_cs_size': 0,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
+    * 'max_comp_size': 0,  # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975
 
 ## More examples
 

blosc2_grok/__init__.py

@@ -129,17 +129,16 @@ def destroy():
     'mct': 0,
     'max_cs_size': 0,
     'max_comp_size': 0,
-    # 20 - 29
+    # 20 - 28
     'rsiz': GrkProfile.GRK_PROFILE_NONE,
     'framerate': 0,
     'apply_icc_': False,
     'rateControlAlgorithm': GrkRateControl.BISECT,
-    # 'numThreads': 0,  # C func will still receive this param
+    'num_threads': 0,
     'deviceId': 0,
     'duration': 0,
     'repeats': 1,
     'verbose': False,
-    'sharedMemoryInterface': False,
 }
 
 
@@ -163,9 +162,6 @@ def set_params_defaults(**kwargs):
 
     args[6] = args[6].encode('utf-8')
 
-    # Insert numThreads
-    args.insert(24, 0)
-
     # Convert tuples to desired NumPy arrays
     args[0] = np.array(args[0], dtype=np.int64)
     args[1] = np.array(args[1], dtype=np.int64)
@@ -188,7 +184,7 @@ def set_params_defaults(**kwargs):
                                                    [ctypes.c_int] +
                                                    [ctypes.c_int] + [ctypes.c_bool] +
                                                    [ctypes.c_int] * 5 + [ctypes.c_bool] +
-                                                   [ctypes.c_int] * 5 + [ctypes.c_bool] * 2)
+                                                   [ctypes.c_int] * 5 + [ctypes.c_bool])
     lib.blosc2_grok_set_default_params(*args)
 
 

examples/params.py

@@ -11,6 +11,7 @@
 import argparse
 import numpy as np
 from PIL import Image
+import time
 
 
 def compress(im, urlpath=None, **kwargs):
@@ -39,6 +40,7 @@ def compress(im, urlpath=None, **kwargs):
 
     # Transform the numpy array to a blosc2 array. This is where compression happens, and
     # the HTJ2K codec is called.
+    start = time.time()
     bl_array = blosc2.asarray(
         np_array,
         chunks=np_array.shape,
@@ -47,6 +49,8 @@ def compress(im, urlpath=None, **kwargs):
         urlpath=urlpath,
         mode="w",
     )
+    stop = time.time()
+    print("Time for compressing: ", stop - start)
     # Print information about the array, see the compression ratio (cratio)
     print(bl_array.info)
 
@@ -81,7 +85,12 @@ def compress(im, urlpath=None, **kwargs):
     # Register codec locally for now
     blosc2.register_codec('grok', 160)
 
-    print("Performing lossless compression ...")
+    print("Performing lossless compression with 1 thread ...")
+    kwargs['num_threads'] = 1
+    _ = compress(im, "lossless.b2nd", **kwargs)
+
+    print("Performing lossless compression with default threads ...")
+    kwargs['num_threads'] = 0
     _ = compress(im, "lossless.b2nd", **kwargs)
 
     print("Performing rates compression ...")

src/blosc2_grok.cpp

@@ -46,9 +46,16 @@ int blosc2_grok_encoder(
     uint32_t dimX = blockshape[0];
     uint32_t dimY = blockshape[1];
     uint32_t numComps = 1;
-    if (ndim == 3) {
+    if (dimX == 1) {
+        // Array formed by different images
+        dimX = blockshape[1];
+        dimY = blockshape[2];
+    }
+    else if (ndim == 3) {
+        // Single image with more than 1 component
         numComps = blockshape[2];
     }
+
     const uint32_t typesize = ((blosc2_schunk*)cparams->schunk)->typesize;
     const uint32_t precision = 8 * typesize;
     // const uint32_t precision = 8 * typesize - 7;
@@ -259,9 +266,9 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til
                                     int mct, int max_cs_size,
                                     int max_comp_size, int rsiz, int framerate,
                                     bool apply_icc_,
-                                    GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int numThreads, int deviceId,
+                                    GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int num_threads, int deviceId,
                                     int duration, int repeats,
-                                    bool verbose, bool sharedMemoryInterface) {
+                                    bool verbose) {
     if (tile_size[0] == 0 && tile_size[1] == 0) {
         GRK_CPARAMETERS_DEFAULTS.tile_size_on = false;
     } else {
@@ -363,7 +370,7 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til
     GRK_CPARAMETERS_DEFAULTS.write_display_resolution = write_display_resolution;*/
     GRK_CPARAMETERS_DEFAULTS.apply_icc_ = apply_icc_;
     GRK_CPARAMETERS_DEFAULTS.rateControlAlgorithm = rateControlAlgorithm;
-    GRK_CPARAMETERS_DEFAULTS.numThreads = numThreads;
+    GRK_CPARAMETERS_DEFAULTS.numThreads = num_threads;
     GRK_CPARAMETERS_DEFAULTS.deviceId = deviceId;
 
     GRK_CPARAMETERS_DEFAULTS.duration = duration;
@@ -373,7 +380,10 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til
     // GRK_CPARAMETERS_DEFAULTS.writeTLM = writeTLM;
 
     GRK_CPARAMETERS_DEFAULTS.verbose = verbose;
-    GRK_CPARAMETERS_DEFAULTS.sharedMemoryInterface = sharedMemoryInterface;
+    // GRK_CPARAMETERS_DEFAULTS.sharedMemoryInterface = sharedMemoryInterface;
+
+    // Initialize threads and verbose
+    grk_initialize(nullptr, GRK_CPARAMETERS_DEFAULTS.numThreads, GRK_CPARAMETERS_DEFAULTS.verbose);
 }
 
 void blosc2_grok_destroy() {

src/blosc2_grok.h

@@ -40,9 +40,9 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til
                                     int mct, int max_cs_size,
                                     int max_comp_size, int rsiz, int framerate,
                                     bool apply_icc_,
-                                    GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int numThreads, int deviceId,
+                                    GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int num_threads, int deviceId,
                                     int duration, int repeats,
-                                    bool verbose, bool sharedMemoryInterface);
+                                    bool verbose);
 
 
 #ifdef __cplusplus

tests/test_jp2.py

@@ -56,14 +56,14 @@
         ({'max_comp_size': 10**9}),
         ({'rsiz': blosc2_grok.GrkProfile.GRK_PROFILE_0}),
         ({'rsiz': blosc2_grok.GrkProfile.GRK_PROFILE_1}),
-        # ({'framerate': 8}), # Would make sense if we had more than one frame
+        # # ({'framerate': 8}), # Would make sense if we had more than one frame
         ({'apply_icc_': True}),
-        # ({'numThreads': 4}),  # Deactivated until we can actually use it
+        ({'num_threads': 4}),
+        ({'num_threads': 1}),
         # ({'deviceId': 8}),  # Meant for multi-GPU systems
         ({'duration': 1}),
         ({'repeats': 2}),
         ({'repeats': 0}),
-        ({'sharedMemoryInterface': True}),
     ],
 )
 def test_jp2(image, args):
@@ -102,6 +102,18 @@ def test_jp2(image, args):
     stop = time.time()
     if kwargs.get('duration', 0) > 0:
         assert stop - start < kwargs['duration']
+    if kwargs.get('num_threads', 0) == 1:
+        kwargs['num_threads'] = 0
+        blosc2_grok.set_params_defaults(**kwargs)
+        start2 = time.time()
+        _ = blosc2.asarray(
+            np_array,
+            chunks=np_array.shape,
+            blocks=np_array.shape,
+            cparams=cparams,
+        )
+        stop2 = time.time()
+        assert stop2 - start2 < stop - start
 
     print(bl_array.schunk.cratio)
     if kwargs.get('quality_mode', None) is None: