From aa6303d6dc798e91776e358c772331c9ae5b645c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 30 Mar 2024 20:26:18 +0100 Subject: [PATCH] Fixed b2nd_append() so that it works with empty arrays too. Add a new bench, and completed existing example. --- bench/b2nd/bench_stack_append.c | 100 +++++++++++++++++++++++++++ blosc/b2nd.c | 37 ++++++++-- examples/b2nd/example_stack_images.c | 76 ++++++++++++++++++-- tests/b2nd/test_b2nd_append.c | 5 ++ 4 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 bench/b2nd/bench_stack_append.c diff --git a/bench/b2nd/bench_stack_append.c b/bench/b2nd/bench_stack_append.c new file mode 100644 index 000000000..aef229668 --- /dev/null +++ b/bench/b2nd/bench_stack_append.c @@ -0,0 +1,100 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 The Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +// Benchmark for appending data to a b2nd array. A new accelerated path has been +// added to b2nd_append() that allows for faster appending of data to a b2nd array +// when data to append is of the same size as the chunkshape. This benchmark +// compares the performance of the new accelerated path with the old one. + +#include "blosc2.h" +#include "b2nd.h" + + +int main() { + blosc2_init(); + const int width = 4 * 512; + const int height = 4 * 272; + const int64_t buffershape[] = {1, height, width}; + int64_t N_images = 1000; + + // Shapes of the b2nd array + int64_t shape[] = {N_images, height, width}; + int32_t chunkshape[] = {1, height, width}; + int32_t blockshape[] = {1, height, width}; + + // Determine the buffer size of the image (in bytes) + const int64_t buffersize = height * width * (int64_t)sizeof(uint16_t); + uint16_t* image = malloc(buffersize); + + // Generate data + for (int j = 0; j < height * width; j++) { + image[j] = j; + } + + char *urlpath = "bench_stack_append.b2nd"; + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + cparams.typesize = sizeof(image[0]); + blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; + //storage.contiguous = true; // for a single file in output + //storage.urlpath = urlpath; + storage.cparams = &cparams; + + char *accel_str = "non-accel"; + for (int accel=0; accel < 2; accel++) { + if (accel) { + shape[0] = 0; + accel_str = "accel"; + } + + blosc2_remove_urlpath(urlpath); + + b2nd_context_t *ctx = b2nd_create_ctx(&storage, 3, + shape, chunkshape, blockshape, + "|u2", DTYPE_NUMPY_FORMAT, + NULL, 0); + b2nd_array_t *src; + if (b2nd_empty(ctx, &src) < 0) { + printf("Error in b2nd_empty\n"); + return -1; + } + + // loop through all images + blosc_timestamp_t t0, t1; + blosc_set_timestamp(&t0); + for (int i = 0; i < N_images; i++) { + //printf("Saving image #: %d\n", i); + + if (accel) { + if (b2nd_append(src, image, buffersize, 0) < 0) { + printf("Error in b2nd_append\n"); + return -1; + } + } else { + int64_t start[] = {i, 0, 0}; + int64_t stop[] = {i + 1, height, width}; + if (b2nd_set_slice_cbuffer(image, buffershape, buffersize, start, stop, src) < 0) { + printf("Error in b2nd_append\n"); + return -1; + } + } + } + blosc_set_timestamp(&t1); + printf("Time to append (%s): %.4f s\n", accel_str, blosc_elapsed_secs(t0, t1)); + printf("Number of chunks: %lld\n", src->sc->nchunks); + printf("Shape of array: (%lld, %lld, %lld)\n", src->shape[0], src->shape[1], src->shape[2]); + + b2nd_free(src); + b2nd_free_ctx(ctx); + } + free(image); + + blosc2_destroy(); + return 0; +} diff --git a/blosc/b2nd.c b/blosc/b2nd.c index 7b0c895fa..6e7ae15ec 100644 --- a/blosc/b2nd.c +++ b/blosc/b2nd.c @@ -121,7 +121,7 @@ int update_shape(b2nd_array_t *array, int8_t ndim, const int64_t *shape, chunkshape[i] + blockshape[i] - chunkshape[i] % blockshape[i]; } } else { - array->extchunkshape[i] = 0; + array->extchunkshape[i] = chunkshape[i]; array->extshape[i] = 0; } } else { @@ -302,8 +302,7 @@ int b2nd_empty(b2nd_context_t *ctx, b2nd_array_t **array) { BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER); BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER); - // BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_UNINIT, array)); - // Avoid variable cratios + // Fill with zeros to avoid variable cratios BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_ZERO, array)); return BLOSC2_ERROR_SUCCESS; @@ -1305,8 +1304,7 @@ int shrink_shape(b2nd_array_t *array, const int64_t *new_shape, const int64_t *s BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM); } if (array->shape[i] == 0) { - BLOSC_TRACE_ERROR("Cannot shrink array with shape[%d] = 0", i); - BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM); + continue; } } if (diffs_sum == 0) { @@ -1451,7 +1449,34 @@ int b2nd_append(b2nd_array_t *array, const void *buffer, int64_t buffersize, BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER); BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER); - BLOSC_ERROR(b2nd_insert(array, buffer, buffersize, axis, array->shape[axis])); + int32_t chunksize = array->sc->chunksize; + int64_t nchunks_append = buffersize / chunksize; + // Check whether chunkshape and blockshape are equals + bool equal_chunks_blocks = true; + for (int i = 0; i < array->ndim; ++i) { + if (array->chunkshape[i] != array->blockshape[i]) { + equal_chunks_blocks = false; + break; + } + } + // General case where a buffer has a different size than the chunksize + if (!equal_chunks_blocks || buffersize % chunksize != 0 || nchunks_append != 1) { + BLOSC_ERROR(b2nd_insert(array, buffer, buffersize, axis, array->shape[axis])); + return BLOSC2_ERROR_SUCCESS; + } + + // Accelerated path for buffers that are of the same size as the chunksize + // printf("accelerated path\n"); + + // Append the buffer to the underlying schunk. This is very fast, as + // it doesn't need to do internal partitioning. + BLOSC_ERROR(blosc2_schunk_append_buffer(array->sc, (void*)buffer, buffersize)); + + // Finally, resize the array + int64_t newshape[B2ND_MAX_DIM]; + memcpy(newshape, array->shape, array->ndim * sizeof(int64_t)); + newshape[axis] += nchunks_append * array->chunkshape[axis]; + BLOSC_ERROR(b2nd_resize(array, newshape, NULL)); return BLOSC2_ERROR_SUCCESS; } diff --git a/examples/b2nd/example_stack_images.c b/examples/b2nd/example_stack_images.c index 6d210db2c..d53ecab67 100644 --- a/examples/b2nd/example_stack_images.c +++ b/examples/b2nd/example_stack_images.c @@ -9,6 +9,10 @@ **********************************************************************/ // This is an example that saves a stack of images in a b2nd frame. +// The images are generated randomly and saved in two different ways: +// 1) Using the b2nd_set_slice_cbuffer method. +// 2) Using the b2nd_append method. + #include #include @@ -24,9 +28,6 @@ int main() { uint16_t* image = malloc(buffersize); int64_t N_images = 10; - char *urlpath = "test_image_dataset.b2nd"; - blosc2_remove_urlpath(urlpath); - blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; cparams.typesize = sizeof(uint16_t); cparams.compcode = BLOSC_BLOSCLZ; @@ -35,8 +36,10 @@ int main() { blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; storage.contiguous = true; // for a single file in output - storage.cparams = &cparams; + char *urlpath = "example_stack_images_set_slice.b2nd"; + blosc2_remove_urlpath(urlpath); storage.urlpath = urlpath; + storage.cparams = &cparams; // shape, chunkshape and blockshape of the ndarray int64_t shape[] = {N_images, height, width}; @@ -53,9 +56,9 @@ int main() { return -1; } - // loop through all images + // Loop through all images + printf("Saving images (set_slice version)...\n"); for (int64_t i = 0; i < N_images; i++) { - printf("Saving image #: %lld\n", i); int64_t start[] = {i, 0, 0}; int64_t stop[] = {i + 1, height, width}; // Generate random image data @@ -64,10 +67,71 @@ int main() { } if (b2nd_set_slice_cbuffer(image, buffershape, buffersize, start, stop, src) < 0) { + printf("Error in b2nd_set_slice_cbuffer\n"); + return -1; + } + } + printf("Adding vlmetalayer data\n"); + uint8_t msgpack[1024]; + // Pack the message using the recommended msgpack format + // The Python wrapper can do this automatically + char *content = "Using b2nd_set_slice_cbuffer()"; + msgpack[0] = 0xd9; + msgpack[1] = strlen(content); + memcpy(msgpack + 2, content, strlen(content) + 1); + int metalen = blosc2_vlmeta_add(src->sc, "method", + msgpack, strlen(content) + 2, NULL); + if (metalen < 0) { + printf("Cannot write vlmetalayer"); + return metalen; + } + b2nd_free_ctx(ctx); + printf("Images saved successfully in %s\n", urlpath); + + // Use the append method to add more images + urlpath = "example_stack_images_append.b2nd"; + blosc2_remove_urlpath(urlpath); + storage.urlpath = urlpath; + // shape can start with 0 now + int64_t shape2[] = {0, height, width}; + + ctx = b2nd_create_ctx(&storage, 3, + shape2, chunkshape, blockshape, + "|u2", DTYPE_NUMPY_FORMAT, + NULL, 0); + b2nd_free(src); + if (b2nd_empty(ctx, &src) < 0) { + printf("Error in b2nd_empty\n"); + return -1; + } + + // loop through all images + printf("Saving images (append version)...\n"); + for (int64_t i = 0; i < N_images; i++) { + // Generate random image data + for (int j = 0; j < width * height; j++) { + image[j] = rand() % 65536; // generate random pixels (uncompressible data) + } + + if (b2nd_append(src, image, buffersize, 0) < 0) { printf("Error in b2nd_append\n"); return -1; } } + printf("Adding vlmetalayer data\n"); + // Pack the message using the recommended msgpack format + // The Python wrapper can do this automatically + content = "Using b2nd_append()"; + msgpack[0] = 0xd9; + msgpack[1] = strlen(content); + memcpy(msgpack + 2, content, strlen(content) + 1); + metalen = blosc2_vlmeta_add(src->sc, "method", + msgpack, strlen(content) + 2, NULL); + if (metalen < 0) { + printf("Cannot write vlmetalayer"); + return metalen; + } + printf("Images saved successfully in %s\n", urlpath); // Clean resources b2nd_free(src); diff --git a/tests/b2nd/test_b2nd_append.c b/tests/b2nd/test_b2nd_append.c index 2a9bfd9df..210941ab7 100644 --- a/tests/b2nd/test_b2nd_append.c +++ b/tests/b2nd/test_b2nd_append.c @@ -45,6 +45,11 @@ CUTEST_TEST_SETUP(append) { {2, {18, 6}, {6, 6}, {3, 3}, {18, 12}, 1}, {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {12, 10, 18}, 2}, {4, {10, 10, 5, 5}, {5, 7, 3, 3}, {2, 2, 1, 1}, {10, 10, 5, 30}, 3}, + // Append to empty arrays + {1, {0}, {3}, {3}, {10}, 0}, + {2, {0, 6}, {6, 6}, {3, 3}, {6, 6}, 0}, + // Accelerated path with chunkshape and blockshape equal to buffershape + {2, {0, 6}, {6, 6}, {6, 6}, {6, 6}, 0}, )); }