Fixed b2nd_append() so that it works with empty arrays too.

Add a new bench, and completed existing example.

bench/b2nd/bench_stack_append.c

@@ -0,0 +1,100 @@
+/*********************************************************************
+  Blosc - Blocked Shuffling and Compression Library
+
+  Copyright (c) 2021  The Blosc Development Team <[email protected]>
+  https://blosc.org
+  License: BSD 3-Clause (see LICENSE.txt)
+
+  See LICENSE.txt for details about copyright and rights to use.
+**********************************************************************/
+
+// Benchmark for appending data to a b2nd array.  A new accelerated path has been
+// added to b2nd_append() that allows for faster appending of data to a b2nd array
+// when data to append is of the same size as the chunkshape.  This benchmark
+// compares the performance of the new accelerated path with the old one.
+
+#include "blosc2.h"
+#include "b2nd.h"
+
+
+int main() {
+  blosc2_init();
+  const int width = 4 * 512;
+  const int height = 4 * 272;
+  const int64_t buffershape[] = {1, height, width};
+  int64_t N_images = 1000;
+
+  // Shapes of the b2nd array
+  int64_t shape[] = {N_images, height, width};
+  int32_t chunkshape[] = {1, height, width};
+  int32_t blockshape[] = {1, height, width};
+
+  // Determine the buffer size of the image (in bytes)
+  const int64_t buffersize = height * width * (int64_t)sizeof(uint16_t);
+  uint16_t* image = malloc(buffersize);
+
+  // Generate data
+  for (int j = 0; j < height * width; j++) {
+    image[j] = j;
+  }
+
+  char *urlpath = "bench_stack_append.b2nd";
+  blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
+  cparams.typesize = sizeof(image[0]);
+  blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS;
+  //storage.contiguous = true;  // for a single file in output
+  //storage.urlpath = urlpath;
+  storage.cparams = &cparams;
+
+  char *accel_str = "non-accel";
+  for (int accel=0; accel < 2; accel++) {
+    if (accel) {
+      shape[0] = 0;
+      accel_str = "accel";
+    }
+
+    blosc2_remove_urlpath(urlpath);
+
+    b2nd_context_t *ctx = b2nd_create_ctx(&storage, 3,
+                                          shape, chunkshape, blockshape,
+                                          "|u2", DTYPE_NUMPY_FORMAT,
+                                          NULL, 0);
+    b2nd_array_t *src;
+    if (b2nd_empty(ctx, &src) < 0) {
+      printf("Error in b2nd_empty\n");
+      return -1;
+    }
+
+    // loop through all images
+    blosc_timestamp_t t0, t1;
+    blosc_set_timestamp(&t0);
+    for (int i = 0; i < N_images; i++) {
+      //printf("Saving image #: %d\n", i);
+
+      if (accel) {
+        if (b2nd_append(src, image, buffersize, 0) < 0) {
+          printf("Error in b2nd_append\n");
+          return -1;
+        }
+      } else {
+        int64_t start[] = {i, 0, 0};
+        int64_t stop[] = {i + 1, height, width};
+        if (b2nd_set_slice_cbuffer(image, buffershape, buffersize, start, stop, src) < 0) {
+          printf("Error in b2nd_append\n");
+          return -1;
+        }
+      }
+    }
+    blosc_set_timestamp(&t1);
+    printf("Time to append (%s): %.4f s\n", accel_str, blosc_elapsed_secs(t0, t1));
+    printf("Number of chunks: %lld\n", src->sc->nchunks);
+    printf("Shape of array: (%lld, %lld, %lld)\n", src->shape[0], src->shape[1], src->shape[2]);
+
+    b2nd_free(src);
+    b2nd_free_ctx(ctx);
+  }
+  free(image);
+
+  blosc2_destroy();
+  return 0;
+}

blosc/b2nd.c

@@ -121,7 +121,7 @@ int update_shape(b2nd_array_t *array, int8_t ndim, const int64_t *shape,
                   chunkshape[i] + blockshape[i] - chunkshape[i] % blockshape[i];
         }
       } else {
-        array->extchunkshape[i] = 0;
+        array->extchunkshape[i] = chunkshape[i];
         array->extshape[i] = 0;
       }
     } else {
@@ -302,8 +302,7 @@ int b2nd_empty(b2nd_context_t *ctx, b2nd_array_t **array) {
   BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
   BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
 
-  // BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_UNINIT, array));
-  // Avoid variable cratios
+  // Fill with zeros to avoid variable cratios
   BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_ZERO, array));
 
   return BLOSC2_ERROR_SUCCESS;
@@ -1305,8 +1304,7 @@ int shrink_shape(b2nd_array_t *array, const int64_t *new_shape, const int64_t *s
       BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
     }
     if (array->shape[i] == 0) {
-      BLOSC_TRACE_ERROR("Cannot shrink array with shape[%d] = 0", i);
-      BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
+      continue;
     }
   }
   if (diffs_sum == 0) {
@@ -1451,7 +1449,34 @@ int b2nd_append(b2nd_array_t *array, const void *buffer, int64_t buffersize,
   BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
   BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
 
-  BLOSC_ERROR(b2nd_insert(array, buffer, buffersize, axis, array->shape[axis]));
+  int32_t chunksize = array->sc->chunksize;
+  int64_t nchunks_append = buffersize / chunksize;
+  // Check whether chunkshape and blockshape are equals
+  bool equal_chunks_blocks = true;
+  for (int i = 0; i < array->ndim; ++i) {
+    if (array->chunkshape[i] != array->blockshape[i]) {
+      equal_chunks_blocks = false;
+      break;
+    }
+  }
+  // General case where a buffer has a different size than the chunksize
+  if (!equal_chunks_blocks || buffersize % chunksize != 0 || nchunks_append != 1) {
+    BLOSC_ERROR(b2nd_insert(array, buffer, buffersize, axis, array->shape[axis]));
+    return BLOSC2_ERROR_SUCCESS;
+  }
+
+  // Accelerated path for buffers that are of the same size as the chunksize
+  // printf("accelerated path\n");
+
+  // Append the buffer to the underlying schunk. This is very fast, as
+  // it doesn't need to do internal partitioning.
+  BLOSC_ERROR(blosc2_schunk_append_buffer(array->sc, (void*)buffer, buffersize));
+
+  // Finally, resize the array
+  int64_t newshape[B2ND_MAX_DIM];
+  memcpy(newshape, array->shape, array->ndim * sizeof(int64_t));
+  newshape[axis] += nchunks_append * array->chunkshape[axis];
+  BLOSC_ERROR(b2nd_resize(array, newshape, NULL));
 
   return BLOSC2_ERROR_SUCCESS;
 }

examples/b2nd/example_stack_images.c

@@ -9,6 +9,10 @@
 **********************************************************************/
 
 // This is an example that saves a stack of images in a b2nd frame.
+// The images are generated randomly and saved in two different ways:
+// 1) Using the b2nd_set_slice_cbuffer method.
+// 2) Using the b2nd_append method.
+
 
 #include <blosc2.h>
 #include <b2nd.h>
@@ -24,9 +28,6 @@ int main() {
   uint16_t* image = malloc(buffersize);
   int64_t N_images = 10;
 
-  char *urlpath = "test_image_dataset.b2nd";
-  blosc2_remove_urlpath(urlpath);
-
   blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
   cparams.typesize = sizeof(uint16_t);
   cparams.compcode = BLOSC_BLOSCLZ;
@@ -35,8 +36,10 @@ int main() {
 
   blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS;
   storage.contiguous = true;  // for a single file in output
-  storage.cparams = &cparams;
+  char *urlpath = "example_stack_images_set_slice.b2nd";
+  blosc2_remove_urlpath(urlpath);
   storage.urlpath = urlpath;
+  storage.cparams = &cparams;
 
   // shape, chunkshape and blockshape of the ndarray
   int64_t shape[] = {N_images, height, width};
@@ -53,9 +56,9 @@ int main() {
     return -1;
   }
 
-  // loop through all images
+  // Loop through all images
+  printf("Saving images (set_slice version)...\n");
   for (int64_t i = 0; i < N_images; i++) {
-    printf("Saving image #: %lld\n", i);
     int64_t start[] = {i, 0, 0};
     int64_t stop[] = {i + 1, height, width};
     // Generate random image data
@@ -64,10 +67,71 @@ int main() {
     }
 
     if (b2nd_set_slice_cbuffer(image, buffershape, buffersize, start, stop, src) < 0) {
+      printf("Error in b2nd_set_slice_cbuffer\n");
+      return -1;
+    }
+  }
+  printf("Adding vlmetalayer data\n");
+  uint8_t msgpack[1024];
+  // Pack the message using the recommended msgpack format
+  // The Python wrapper can do this automatically
+  char *content = "Using b2nd_set_slice_cbuffer()";
+  msgpack[0] = 0xd9;
+  msgpack[1] = strlen(content);
+  memcpy(msgpack + 2, content, strlen(content) + 1);
+  int metalen = blosc2_vlmeta_add(src->sc, "method",
+                                 msgpack, strlen(content) + 2, NULL);
+  if (metalen < 0) {
+    printf("Cannot write vlmetalayer");
+    return metalen;
+  }
+  b2nd_free_ctx(ctx);
+  printf("Images saved successfully in %s\n", urlpath);
+
+  // Use the append method to add more images
+  urlpath = "example_stack_images_append.b2nd";
+  blosc2_remove_urlpath(urlpath);
+  storage.urlpath = urlpath;
+  // shape can start with 0 now
+  int64_t shape2[] = {0, height, width};
+
+  ctx = b2nd_create_ctx(&storage, 3,
+                         shape2, chunkshape, blockshape,
+                         "|u2", DTYPE_NUMPY_FORMAT,
+                         NULL, 0);
+  b2nd_free(src);
+  if (b2nd_empty(ctx, &src) < 0) {
+    printf("Error in b2nd_empty\n");
+    return -1;
+  }
+
+  // loop through all images
+  printf("Saving images (append version)...\n");
+  for (int64_t i = 0; i < N_images; i++) {
+    // Generate random image data
+    for (int j = 0; j < width * height; j++) {
+      image[j] = rand() % 65536; // generate random pixels (uncompressible data)
+    }
+
+    if (b2nd_append(src, image, buffersize, 0) < 0) {
       printf("Error in b2nd_append\n");
       return -1;
     }
   }
+  printf("Adding vlmetalayer data\n");
+  // Pack the message using the recommended msgpack format
+  // The Python wrapper can do this automatically
+  content = "Using b2nd_append()";
+  msgpack[0] = 0xd9;
+  msgpack[1] = strlen(content);
+  memcpy(msgpack + 2, content, strlen(content) + 1);
+  metalen = blosc2_vlmeta_add(src->sc, "method",
+                              msgpack, strlen(content) + 2, NULL);
+  if (metalen < 0) {
+    printf("Cannot write vlmetalayer");
+    return metalen;
+  }
+  printf("Images saved successfully in %s\n", urlpath);
 
   // Clean resources
   b2nd_free(src);

tests/b2nd/test_b2nd_append.c

@@ -45,6 +45,11 @@ CUTEST_TEST_SETUP(append) {
       {2, {18, 6}, {6, 6}, {3, 3}, {18, 12}, 1},
       {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {12, 10, 18}, 2},
       {4, {10, 10, 5, 5}, {5, 7, 3, 3}, {2, 2, 1, 1}, {10, 10, 5, 30}, 3},
+      // Append to empty arrays
+      {1, {0}, {3}, {3}, {10}, 0},
+      {2, {0, 6}, {6, 6}, {3, 3}, {6, 6}, 0},
+      // Accelerated path with chunkshape and blockshape equal to buffershape
+      {2, {0, 6}, {6, 6}, {6, 6}, {6, 6}, 0},
 
   ));
 }