diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml index 8e32a18..c37c486 100644 --- a/.github/workflows/cibuildwheels.yml +++ b/.github/workflows/cibuildwheels.yml @@ -60,7 +60,7 @@ jobs: CIBW_BUILD: 'cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64' CIBW_BEFORE_TEST: python -m pip install --upgrade pip && python -m pip install -r requirements-test.txt CIBW_TEST_COMMAND: > - cmd /V /C "set BLOSC_TRACE=1 && python -m pytest" + cmd /V /C "set BLOSC_TRACE=1 && python -m pytest {project}/tests" - name: Build wheels (Linux / Mac OSX x86_64) if: ${{ matrix.os != 'windows-latest'}} @@ -72,7 +72,7 @@ jobs: CIBW_BUILD: 'cp39-* cp310-* cp311-* cp312-*' CIBW_BEFORE_TEST: python -m pip install --upgrade pip && python -m pip install -r requirements-test.txt CIBW_TEST_COMMAND: > - BLOSC_TRACE=1 python -m pytest + BLOSC_TRACE=1 python -m pytest {project}/tests CIBW_ARCHS_LINUX: ${{ matrix.arch }} CIBW_ARCHS_MACOS: "x86_64" diff --git a/README.md b/README.md index 422617a..281d9ce 100644 --- a/README.md +++ b/README.md @@ -63,30 +63,30 @@ The following parameters are available for compression for grok, with its defaul 'tile_offset': (0, 0), 'quality_mode': None, 'quality_layers': np.zeros(0, dtype=np.float64), - * 'numgbits': 2, # Equivalent to -N, -guard_bits 'progression': "LRCP", 'num_resolutions': 6, 'codeblock_size': (64, 64), - 'irreversible': - * 'roi_compno': -1, # Together with 'roi_shift' it is equivalent to -R, -ROI - * 'roi_shift': 0, + 'irreversible': False, 'precinct_size': (0, 0), 'offset': (0, 0), + 'mct': 0, + * 'numgbits': 2, # Equivalent to -N, -guard_bits + * 'roi_compno': -1, # Together with 'roi_shift' it is equivalent to -R, -ROI + * 'roi_shift': 0, * 'decod_format': GrkFileFmt.GRK_FMT_UNK, * 'cod_format': GrkFileFmt.GRK_FMT_UNK, - * 'enableTilePartGeneration': False, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 - 'mct': 0, - * 'max_cs_size': 0, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 - * 'max_comp_size': 0, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 * 'rsiz': GrkProfile.GRK_PROFILE_NONE, # Equivalent to -Z, -rsiz * 'framerate': 0, * 'apply_icc_': False, # Equivalent to -f, -apply_icc * 'rateControlAlgorithm': GrkRateControl.BISECT, + * 'num_threads': 0, * 'deviceId': 0, # Equivalent to -G, -device_id * 'duration': 0, # Equivalent to -J, -duration * 'repeats': 1, # Equivalent to -e, -repetitions * 'verbose': False, - * 'sharedMemoryInterface': False, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 + * 'enableTilePartGeneration': False, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 + * 'max_cs_size': 0, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 + * 'max_comp_size': 0, # See https://github.com/GrokImageCompression/grok/blob/a84ac2592e581405a976a00cf9e6f03cab7e2481/src/lib/core/grok.h#L975 ## More examples diff --git a/blosc2_grok/__init__.py b/blosc2_grok/__init__.py index dd8060d..f0b78ae 100644 --- a/blosc2_grok/__init__.py +++ b/blosc2_grok/__init__.py @@ -129,17 +129,16 @@ def destroy(): 'mct': 0, 'max_cs_size': 0, 'max_comp_size': 0, - # 20 - 29 + # 20 - 28 'rsiz': GrkProfile.GRK_PROFILE_NONE, 'framerate': 0, 'apply_icc_': False, 'rateControlAlgorithm': GrkRateControl.BISECT, - # 'numThreads': 0, # C func will still receive this param + 'num_threads': 0, 'deviceId': 0, 'duration': 0, 'repeats': 1, 'verbose': False, - 'sharedMemoryInterface': False, } @@ -163,9 +162,6 @@ def set_params_defaults(**kwargs): args[6] = args[6].encode('utf-8') - # Insert numThreads - args.insert(24, 0) - # Convert tuples to desired NumPy arrays args[0] = np.array(args[0], dtype=np.int64) args[1] = np.array(args[1], dtype=np.int64) @@ -188,7 +184,7 @@ def set_params_defaults(**kwargs): [ctypes.c_int] + [ctypes.c_int] + [ctypes.c_bool] + [ctypes.c_int] * 5 + [ctypes.c_bool] + - [ctypes.c_int] * 5 + [ctypes.c_bool] * 2) + [ctypes.c_int] * 5 + [ctypes.c_bool]) lib.blosc2_grok_set_default_params(*args) diff --git a/examples/params.py b/examples/params.py index 399e6d9..9cf7ccd 100644 --- a/examples/params.py +++ b/examples/params.py @@ -11,6 +11,7 @@ import argparse import numpy as np from PIL import Image +import time def compress(im, urlpath=None, **kwargs): @@ -39,6 +40,7 @@ def compress(im, urlpath=None, **kwargs): # Transform the numpy array to a blosc2 array. This is where compression happens, and # the HTJ2K codec is called. + start = time.time() bl_array = blosc2.asarray( np_array, chunks=np_array.shape, @@ -47,6 +49,8 @@ def compress(im, urlpath=None, **kwargs): urlpath=urlpath, mode="w", ) + stop = time.time() + print("Time for compressing: ", stop - start) # Print information about the array, see the compression ratio (cratio) print(bl_array.info) @@ -81,7 +85,12 @@ def compress(im, urlpath=None, **kwargs): # Register codec locally for now blosc2.register_codec('grok', 160) - print("Performing lossless compression ...") + print("Performing lossless compression with 1 thread ...") + kwargs['num_threads'] = 1 + _ = compress(im, "lossless.b2nd", **kwargs) + + print("Performing lossless compression with default threads ...") + kwargs['num_threads'] = 0 _ = compress(im, "lossless.b2nd", **kwargs) print("Performing rates compression ...") diff --git a/src/blosc2_grok.cpp b/src/blosc2_grok.cpp index e4bd3a1..dd15c51 100644 --- a/src/blosc2_grok.cpp +++ b/src/blosc2_grok.cpp @@ -46,9 +46,16 @@ int blosc2_grok_encoder( uint32_t dimX = blockshape[0]; uint32_t dimY = blockshape[1]; uint32_t numComps = 1; - if (ndim == 3) { + if (dimX == 1) { + // Array formed by different images + dimX = blockshape[1]; + dimY = blockshape[2]; + } + else if (ndim == 3) { + // Single image with more than 1 component numComps = blockshape[2]; } + const uint32_t typesize = ((blosc2_schunk*)cparams->schunk)->typesize; const uint32_t precision = 8 * typesize; // const uint32_t precision = 8 * typesize - 7; @@ -259,9 +266,9 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til int mct, int max_cs_size, int max_comp_size, int rsiz, int framerate, bool apply_icc_, - GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int numThreads, int deviceId, + GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int num_threads, int deviceId, int duration, int repeats, - bool verbose, bool sharedMemoryInterface) { + bool verbose) { if (tile_size[0] == 0 && tile_size[1] == 0) { GRK_CPARAMETERS_DEFAULTS.tile_size_on = false; } else { @@ -363,7 +370,7 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til GRK_CPARAMETERS_DEFAULTS.write_display_resolution = write_display_resolution;*/ GRK_CPARAMETERS_DEFAULTS.apply_icc_ = apply_icc_; GRK_CPARAMETERS_DEFAULTS.rateControlAlgorithm = rateControlAlgorithm; - GRK_CPARAMETERS_DEFAULTS.numThreads = numThreads; + GRK_CPARAMETERS_DEFAULTS.numThreads = num_threads; GRK_CPARAMETERS_DEFAULTS.deviceId = deviceId; GRK_CPARAMETERS_DEFAULTS.duration = duration; @@ -373,7 +380,10 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til // GRK_CPARAMETERS_DEFAULTS.writeTLM = writeTLM; GRK_CPARAMETERS_DEFAULTS.verbose = verbose; - GRK_CPARAMETERS_DEFAULTS.sharedMemoryInterface = sharedMemoryInterface; + // GRK_CPARAMETERS_DEFAULTS.sharedMemoryInterface = sharedMemoryInterface; + + // Initialize threads and verbose + grk_initialize(nullptr, GRK_CPARAMETERS_DEFAULTS.numThreads, GRK_CPARAMETERS_DEFAULTS.verbose); } void blosc2_grok_destroy() { diff --git a/src/blosc2_grok.h b/src/blosc2_grok.h index dd84037..536fb76 100644 --- a/src/blosc2_grok.h +++ b/src/blosc2_grok.h @@ -40,9 +40,9 @@ void blosc2_grok_set_default_params(const int64_t *tile_size, const int64_t *til int mct, int max_cs_size, int max_comp_size, int rsiz, int framerate, bool apply_icc_, - GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int numThreads, int deviceId, + GRK_RATE_CONTROL_ALGORITHM rateControlAlgorithm, int num_threads, int deviceId, int duration, int repeats, - bool verbose, bool sharedMemoryInterface); + bool verbose); #ifdef __cplusplus diff --git a/tests/test_jp2.py b/tests/test_jp2.py index 3651d96..199dcd7 100644 --- a/tests/test_jp2.py +++ b/tests/test_jp2.py @@ -56,14 +56,14 @@ ({'max_comp_size': 10**9}), ({'rsiz': blosc2_grok.GrkProfile.GRK_PROFILE_0}), ({'rsiz': blosc2_grok.GrkProfile.GRK_PROFILE_1}), - # ({'framerate': 8}), # Would make sense if we had more than one frame + # # ({'framerate': 8}), # Would make sense if we had more than one frame ({'apply_icc_': True}), - # ({'numThreads': 4}), # Deactivated until we can actually use it + ({'num_threads': 4}), + ({'num_threads': 1}), # ({'deviceId': 8}), # Meant for multi-GPU systems ({'duration': 1}), ({'repeats': 2}), ({'repeats': 0}), - ({'sharedMemoryInterface': True}), ], ) def test_jp2(image, args): @@ -102,6 +102,18 @@ def test_jp2(image, args): stop = time.time() if kwargs.get('duration', 0) > 0: assert stop - start < kwargs['duration'] + if kwargs.get('num_threads', 0) == 1: + kwargs['num_threads'] = 0 + blosc2_grok.set_params_defaults(**kwargs) + start2 = time.time() + _ = blosc2.asarray( + np_array, + chunks=np_array.shape, + blocks=np_array.shape, + cparams=cparams, + ) + stop2 = time.time() + assert stop2 - start2 < stop - start print(bl_array.schunk.cratio) if kwargs.get('quality_mode', None) is None: