From db122dbf69fecc96e6205adf5272146f403a992c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 22 May 2024 12:35:34 +0200 Subject: [PATCH 1/7] add testCodecTranspose --- .../v3/codec/core/TransposeCodec.java | 2 + src/test/java/dev/zarr/zarrjava/ZarrTest.java | 71 +++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index 69fb6fe..f1b1675 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -54,6 +54,8 @@ public long computeEncodedSize(long inputByteLength, public static final class Configuration { + // TODO: order 'C' and 'F' are deprecated + // https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/v1.0.html#transpose-codec-v1 public final String order; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index e6d65b6..59f2a4b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -10,10 +10,8 @@ import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -36,12 +34,12 @@ public class ZarrTest { final static Path TESTDATA = Paths.get("testdata"); final static Path TESTOUTPUT = Paths.get("testoutput"); - //TODO: is the Path with / instead of \ readable in Windows? final static Path ZARRITA_WRITE_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_write.py"); final static Path ZARRITA_READ_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_read.py"); - final static String CONDA_ENVIRONMENT = "zarrita_env"; + static ucar.ma2.Array testData; + static ucar.ma2.Array testDataTransposed; @BeforeAll public static void clearTestoutputFolder() throws IOException { @@ -96,6 +94,52 @@ public static void installZarritaInCondaEnv() throws IOException { } } + @BeforeEach + public void fillTestData() { + testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{4, 4, 4}, new int[]{ + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63 + }); + testDataTransposed = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{4, 4, 4}, new int[]{ + 0, 16, 32, 48, + 4, 20, 36, 52, + 8, 24, 40, 56, + 12, 28, 44, 60, + + 1, 17, 33, 49, + 5, 21, 37, 53, + 9, 25, 41, 57, + 13, 29, 45, 61, + + 2, 18, 34, 50, + 6, 22, 38, 54, + 10, 26, 42, 58, + 14, 30, 46, 62, + + 3, 19, 35, 51, + 7, 23, 39, 55, + 11, 27, 43, 59, + 15, 31, 47, 63 + }); + } + @ParameterizedTest @ValueSource(strings = {"blosc", "gzip", "zstd", "bytes", "transpose", "sharding", "crc32c"}) public void testReadFromZarrita(String codec) throws IOException, ZarrException, InterruptedException { @@ -251,6 +295,21 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException, Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); } + @ParameterizedTest + @ValueSource(strings = {"F", "C"}) + public void testCodecTranspose(String order) throws IOException, ZarrException, InterruptedException { + ArrayMetadata.CoreArrayMetadata metadata = new ArrayMetadata.CoreArrayMetadata( + new long[]{4, 4, 4}, + new int[]{4, 4, 4}, + DataType.UINT32, + null); + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(order)); + + //TODO is that what is expected? + assertEquals(testDataTransposed, transposeCodec.encode(testData, metadata)); + assertEquals(testData, transposeCodec.decode(testDataTransposed, metadata)); + } + @Test public void testFileSystemStores() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTDATA); From 842ad54e25a95a3639263e6e3edf342c4b3672ef Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 22 May 2024 13:09:41 +0200 Subject: [PATCH 2/7] start remove constants "C" adn "F" from Transpose Codec's order --- .../v3/codec/core/TransposeCodec.java | 8 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 85 ++++++------------- 2 files changed, 26 insertions(+), 67 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index f1b1675..a65d9b5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -56,14 +56,10 @@ public static final class Configuration { // TODO: order 'C' and 'F' are deprecated // https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/v1.0.html#transpose-codec-v1 - public final String order; + public final int[] order; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "order", defaultValue = "C") String order) - throws ZarrException { - if (!order.equals("C") && !order.equals("F")) { - throw new ZarrException("Only 'C' or 'F' are supported."); - } + public Configuration(@JsonProperty(value = "order") int[] order){ this.order = order; } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 59f2a4b..e90ba28 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -11,7 +11,10 @@ import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -27,8 +30,7 @@ import java.util.Map; import java.util.stream.Stream; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class ZarrTest { @@ -38,8 +40,6 @@ public class ZarrTest { final static Path ZARRITA_WRITE_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_write.py"); final static Path ZARRITA_READ_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_read.py"); final static String CONDA_ENVIRONMENT = "zarrita_env"; - static ucar.ma2.Array testData; - static ucar.ma2.Array testDataTransposed; @BeforeAll public static void clearTestoutputFolder() throws IOException { @@ -94,52 +94,6 @@ public static void installZarritaInCondaEnv() throws IOException { } } - @BeforeEach - public void fillTestData() { - testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{4, 4, 4}, new int[]{ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 15, - - 16, 17, 18, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, - 28, 29, 30, 31, - - 32, 33, 34, 35, - 36, 37, 38, 39, - 40, 41, 42, 43, - 44, 45, 46, 47, - - 48, 49, 50, 51, - 52, 53, 54, 55, - 56, 57, 58, 59, - 60, 61, 62, 63 - }); - testDataTransposed = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{4, 4, 4}, new int[]{ - 0, 16, 32, 48, - 4, 20, 36, 52, - 8, 24, 40, 56, - 12, 28, 44, 60, - - 1, 17, 33, 49, - 5, 21, 37, 53, - 9, 25, 41, 57, - 13, 29, 45, 61, - - 2, 18, 34, 50, - 6, 22, 38, 54, - 10, 26, 42, 58, - 14, 30, 46, 62, - - 3, 19, 35, 51, - 7, 23, 39, 55, - 11, 27, 43, 59, - 15, 31, 47, 63 - }); - } - @ParameterizedTest @ValueSource(strings = {"blosc", "gzip", "zstd", "bytes", "transpose", "sharding", "crc32c"}) public void testReadFromZarrita(String codec) throws IOException, ZarrException, InterruptedException { @@ -295,19 +249,28 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException, Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); } - @ParameterizedTest - @ValueSource(strings = {"F", "C"}) - public void testCodecTranspose(String order) throws IOException, ZarrException, InterruptedException { + @Test + public void testCodecTranspose() throws IOException, ZarrException, InterruptedException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + ArrayMetadata.CoreArrayMetadata metadata = new ArrayMetadata.CoreArrayMetadata( - new long[]{4, 4, 4}, - new int[]{4, 4, 4}, + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, DataType.UINT32, null); - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(order)); - - //TODO is that what is expected? - assertEquals(testDataTransposed, transposeCodec.encode(testData, metadata)); - assertEquals(testData, transposeCodec.decode(testDataTransposed, metadata)); + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + TransposeCodec transposeCodecWrongOrder1 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 2})); + TransposeCodec transposeCodecWrongOrder2 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3})); + TransposeCodec transposeCodecWrongOrder3 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3, 0})); + + assertEquals(testDataTransposed120, transposeCodec.encode(testData, metadata)); + assertEquals(testData, transposeCodec.decode(testDataTransposed120, metadata)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder1.encode(testData, metadata)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder2.encode(testData, metadata)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder3.encode(testData, metadata)); } @Test From cf0207df4c35446dc206b900a2b178ee54ab6854 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 24 May 2024 16:13:09 +0200 Subject: [PATCH 3/7] add CoreArrayMetadata to codec object instead of passing as argument in encode and decode remove transpose order "F" and "C" --- .../java/dev/zarr/zarrjava/utils/Utils.java | 20 ++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 9 +- .../zarrjava/v3/codec/ArrayArrayCodec.java | 7 +- .../zarrjava/v3/codec/ArrayBytesCodec.java | 17 +-- .../zarrjava/v3/codec/BytesBytesCodec.java | 10 +- .../dev/zarr/zarrjava/v3/codec/Codec.java | 19 ++- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 8 +- .../zarr/zarrjava/v3/codec/CodecPipeline.java | 29 +++-- .../zarrjava/v3/codec/core/BloscCodec.java | 6 +- .../zarrjava/v3/codec/core/BytesCodec.java | 8 +- .../zarrjava/v3/codec/core/Crc32cCodec.java | 10 +- .../zarrjava/v3/codec/core/GzipCodec.java | 6 +- .../v3/codec/core/ShardingIndexedCodec.java | 62 ++++------ .../v3/codec/core/TransposeCodec.java | 111 +++++++++++------- .../zarrjava/v3/codec/core/ZstdCodec.java | 6 +- .../java/dev/zarr/zarrjava/TestUtils.java | 31 +++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 22 ++-- .../java/dev/zarr/zarrjava/zarrita_read.py | 2 +- .../java/dev/zarr/zarrjava/zarrita_write.py | 2 +- 19 files changed, 231 insertions(+), 154 deletions(-) create mode 100644 src/test/java/dev/zarr/zarrjava/TestUtils.java diff --git a/src/main/java/dev/zarr/zarrjava/utils/Utils.java b/src/main/java/dev/zarr/zarrjava/utils/Utils.java index 93b7cf1..3ea4bfd 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/Utils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/Utils.java @@ -77,4 +77,24 @@ public static T[] concatArrays(T[] array1, T[]... arrays) { } return result; } + + public static boolean isPermutation(int[] array) { + if (array.length==0){ + return false; + } + int[] arange = new int[array.length]; + Arrays.setAll(arange, i -> i); + int[] orderSorted = array.clone(); + Arrays.sort(orderSorted); + return Arrays.equals(orderSorted, arange); + } + + public static int[] inversePermutation(int[] origin){ + assert isPermutation(origin); + int[] inverse = new int[origin.length]; + for (int i = 0; i < origin.length; i++) { + inverse[origin[i]] = i; + } + return inverse; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 631ef70..85e0457 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -27,7 +27,7 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { super(storeHandle); this.metadata = arrayMetadata; - this.codecPipeline = new CodecPipeline(arrayMetadata.codecs); + this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); } /** @@ -171,8 +171,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx if (codecPipeline.supportsPartialDecode()) { final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, - Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape, - metadata.coreArrayMetadata); + Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, chunkProjection.outOffset, chunkProjection.shape ); @@ -223,7 +222,7 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) return metadata.allocateFillValueChunk(); } - return codecPipeline.decode(chunkBytes, metadata.coreArrayMetadata); + return codecPipeline.decode(chunkBytes); } /** @@ -299,7 +298,7 @@ public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Zar if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue)) { chunkHandle.delete(); } else { - ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, metadata.coreArrayMetadata); + ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); chunkHandle.set(chunkBytes); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java index 868183a..a488d30 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java @@ -1,15 +1,14 @@ package dev.zarr.zarrjava.v3.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import ucar.ma2.Array; -public interface ArrayArrayCodec extends Codec { +public abstract class ArrayArrayCodec extends Codec { - Array encode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract Array encode(Array chunkArray) throws ZarrException; - Array decode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract Array decode(Array chunkArray) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java index ad042ac..361ae61 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java @@ -2,23 +2,24 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import java.nio.ByteBuffer; import ucar.ma2.Array; -public interface ArrayBytesCodec extends Codec { +public abstract class ArrayBytesCodec extends Codec { - ByteBuffer encode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract ByteBuffer encode(Array chunkArray) throws ZarrException; - Array decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) + protected abstract Array decode(ByteBuffer chunkBytes) throws ZarrException; - interface WithPartialDecode extends ArrayBytesCodec { + public abstract static class WithPartialDecode extends ArrayBytesCodec { - Array decodePartial( - StoreHandle handle, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata + public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; + public abstract ByteBuffer encode(Array shardArray) throws ZarrException; + + protected abstract Array decodePartial( + StoreHandle handle, long[] offset, int[] shape ) throws ZarrException; } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java index d8ce47a..6435463 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java @@ -1,15 +1,13 @@ package dev.zarr.zarrjava.v3.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; + import java.nio.ByteBuffer; -public interface BytesBytesCodec extends Codec { +public abstract class BytesBytesCodec extends Codec { - ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) - throws ZarrException; + protected abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; - ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) - throws ZarrException; + public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index 6ce7687..988dd1d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,9 +5,22 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public interface Codec { +public abstract class Codec { - long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException; + protected ArrayMetadata.CoreArrayMetadata arrayMetadata; + + protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + if (arrayMetadata == null) { + throw new ZarrException("arrayMetadata needs to get set in for every codec"); + } + return this.arrayMetadata; + } + + protected abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) + throws ZarrException; + + public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + this.arrayMetadata = arrayMetadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5e33d7c..bc59f4e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -62,13 +62,9 @@ public CodecBuilder withBlosc() { return withBlosc("zstd"); } - public CodecBuilder withTranspose(String order) { - try { + public CodecBuilder withTranspose(int[] order) { codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; + return this; } public CodecBuilder withBytes(Endian endian) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java index 9ece0f0..920a1f4 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java @@ -12,8 +12,10 @@ public class CodecPipeline { @Nonnull final Codec[] codecs; + public final CoreArrayMetadata arrayMetadata; - public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { + public CodecPipeline(@Nonnull Codec[] codecs, CoreArrayMetadata arrayMetadata) throws ZarrException { + this.arrayMetadata = arrayMetadata; long arrayBytesCodecCount = Arrays.stream(codecs).filter(c -> c instanceof ArrayBytesCodec) .count(); if (arrayBytesCodecCount != 1) { @@ -21,6 +23,7 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { "Exactly 1 ArrayBytesCodec is required. Found " + arrayBytesCodecCount + "."); } Codec prevCodec = null; + CoreArrayMetadata codecArrayMetadata = arrayMetadata; for (Codec codec : codecs) { if (prevCodec != null) { if (codec instanceof ArrayBytesCodec && prevCodec instanceof ArrayBytesCodec) { @@ -44,6 +47,8 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { prevCodec.getClass() + "'."); } } + codec.setCoreArrayMetadata(codecArrayMetadata); + codecArrayMetadata = codec.resolveArrayMetadata(); prevCodec = codec; } @@ -79,15 +84,14 @@ public boolean supportsPartialDecode() { @Nonnull public Array decodePartial( @Nonnull StoreHandle storeHandle, - long[] offset, int[] shape, - @Nonnull CoreArrayMetadata arrayMetadata + long[] offset, int[] shape ) throws ZarrException { if (!supportsPartialDecode()) { throw new ZarrException( "Partial decode is not supported for these codecs. " + Arrays.toString(codecs)); } Array chunkArray = ((ArrayBytesCodec.WithPartialDecode) getArrayBytesCodec()).decodePartial( - storeHandle, offset, shape, arrayMetadata); + storeHandle, offset, shape); if (chunkArray == null) { throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); } @@ -96,8 +100,7 @@ public Array decodePartial( @Nonnull public Array decode( - @Nonnull ByteBuffer chunkBytes, - @Nonnull CoreArrayMetadata arrayMetadata + @Nonnull ByteBuffer chunkBytes ) throws ZarrException { if (chunkBytes == null) { throw new ZarrException("chunkBytes is null. Ohh nooo."); @@ -106,7 +109,7 @@ public Array decode( BytesBytesCodec[] bytesBytesCodecs = getBytesBytesCodecs(); for (int i = bytesBytesCodecs.length - 1; i >= 0; --i) { BytesBytesCodec codec = bytesBytesCodecs[i]; - chunkBytes = codec.decode(chunkBytes, arrayMetadata); + chunkBytes = codec.decode(chunkBytes); } if (chunkBytes == null) { @@ -114,7 +117,7 @@ public Array decode( "chunkBytes is null. This is likely a bug in one of the codecs. " + Arrays.toString( getBytesBytesCodecs())); } - Array chunkArray = getArrayBytesCodec().decode(chunkBytes, arrayMetadata); + Array chunkArray = getArrayBytesCodec().decode(chunkBytes); if (chunkArray == null) { throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); } @@ -122,7 +125,7 @@ public Array decode( ArrayArrayCodec[] arrayArrayCodecs = getArrayArrayCodecs(); for (int i = arrayArrayCodecs.length - 1; i >= 0; --i) { ArrayArrayCodec codec = arrayArrayCodecs[i]; - chunkArray = codec.decode(chunkArray, arrayMetadata); + chunkArray = codec.decode(chunkArray); } if (chunkArray == null) { @@ -133,16 +136,16 @@ public Array decode( @Nonnull public ByteBuffer encode( - @Nonnull Array chunkArray, @Nonnull CoreArrayMetadata arrayMetadata + @Nonnull Array chunkArray ) throws ZarrException { for (ArrayArrayCodec codec : getArrayArrayCodecs()) { - chunkArray = codec.encode(chunkArray, arrayMetadata); + chunkArray = codec.encode(chunkArray); } - ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray, arrayMetadata); + ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray); for (BytesBytesCodec codec : getBytesBytesCodecs()) { - chunkBytes = codec.encode(chunkBytes, arrayMetadata); + chunkBytes = codec.encode(chunkBytes); } return chunkBytes; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index 2c3412b..1661304 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class BloscCodec implements BytesBytesCodec { +public class BloscCodec extends BytesBytesCodec { public final String name = "blosc"; @Nonnull @@ -33,7 +33,7 @@ public BloscCodec( } @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try { return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); @@ -43,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try { return ByteBuffer.wrap( diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index de9d1e7..1415da9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -11,7 +11,7 @@ import javax.annotation.Nonnull; import ucar.ma2.Array; -public class BytesCodec implements ArrayBytesCodec { +public class BytesCodec extends ArrayBytesCodec { public final String name = "bytes"; @Nonnull @@ -29,14 +29,14 @@ public BytesCodec(Endian endian) { } @Override - public Array decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) { + public Array decode(ByteBuffer chunkBytes) { chunkBytes.order(configuration.endian.getByteOrder()); return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, chunkBytes); } @Override - public ByteBuffer encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { + public ByteBuffer encode(Array chunkArray) { return chunkArray.getDataAsByteBuffer(configuration.endian.getByteOrder()); } @@ -72,7 +72,7 @@ public ByteOrder getByteOrder() { } } - public static final class Configuration { + public static final class Configuration{ @Nonnull public final BytesCodec.Endian endian; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 38be686..a1e3cb5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -9,17 +9,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class Crc32cCodec implements BytesBytesCodec { +public class Crc32cCodec extends BytesBytesCodec { public final String name = "crc32c"; @JsonCreator - public Crc32cCodec( - ) { - } + public Crc32cCodec(){} @Override - public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { ByteBuffer buffer = chunkBytes.slice(); buffer.order(ByteOrder.LITTLE_ENDIAN); @@ -45,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) { + public ByteBuffer encode(ByteBuffer chunkBytes) { return Utils.makeByteBuffer(chunkBytes.capacity() + 4, b -> { final CRC32C crc32c = new CRC32C(); crc32c.update(chunkBytes); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index 1054542..3ff5acd 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -16,7 +16,7 @@ import java.util.zip.GZIPOutputStream; import javax.annotation.Nonnull; -public class GzipCodec implements BytesBytesCodec { +public class GzipCodec extends BytesBytesCodec { public final String name = "gzip"; @Nonnull @@ -37,7 +37,7 @@ private void copy(InputStream inputStream, OutputStream outputStream) throws IOE } @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream( new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { @@ -50,7 +50,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream( outputStream)) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index f4237df..035fc7b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -22,15 +22,13 @@ import ucar.ma2.InvalidRangeException; -public class ShardingIndexedCodec implements ArrayBytesCodec, ArrayBytesCodec.WithPartialDecode { +public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode { public final String name = "sharding_indexed"; @Nonnull public final Configuration configuration; - @Nonnull - final CodecPipeline codecPipeline; - @Nonnull - final CodecPipeline indexCodecPipeline; + CodecPipeline codecPipeline; + CodecPipeline indexCodecPipeline; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ShardingIndexedCodec( @@ -38,8 +36,18 @@ public ShardingIndexedCodec( Configuration configuration ) throws ZarrException { this.configuration = configuration; - this.codecPipeline = new CodecPipeline(configuration.codecs); - this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs); + } + + @Override + public void setCoreArrayMetadata(CoreArrayMetadata arrayMetadata) throws ZarrException { + super.setCoreArrayMetadata(arrayMetadata); + final ArrayMetadata.CoreArrayMetadata shardMetadata = + new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), + configuration.chunkShape, arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); + this.codecPipeline = new CodecPipeline(configuration.codecs, shardMetadata); + this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs, getShardIndexArrayMetadata(getChunksPerShard(arrayMetadata))); } ArrayMetadata.CoreArrayMetadata getShardIndexArrayMetadata(int[] chunksPerShard) { @@ -85,20 +93,15 @@ void setValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int } @Override - public Array decode(ByteBuffer shardBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public Array decode(ByteBuffer shardBytes) throws ZarrException { return decodeInternal(new ByteBufferDataProvider(shardBytes), new long[arrayMetadata.ndim()], arrayMetadata.chunkShape, arrayMetadata); } @Override - public ByteBuffer encode(final Array shardArray, - final ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - final ArrayMetadata.CoreArrayMetadata shardMetadata = - new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), - configuration.chunkShape, arrayMetadata.dataType, - arrayMetadata.parsedFillValue - ); + public ByteBuffer encode(final Array shardArray) throws ZarrException { + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; final int[] chunksPerShard = getChunksPerShard(arrayMetadata); final int chunkCount = Arrays.stream(chunksPerShard) .reduce(1, (r, a) -> r * a); @@ -127,7 +130,7 @@ public ByteBuffer encode(final Array shardArray, setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, -1); setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, -1); } else { - final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, shardMetadata); + final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); synchronized (chunkBytesList) { int chunkByteOffset = chunkBytesList.stream() .mapToInt(ByteBuffer::capacity) @@ -150,7 +153,7 @@ public ByteBuffer encode(final Array shardArray, shardBytes.put(chunkBytes); } shardBytes.put( - indexCodecPipeline.encode(shardIndexArray, getShardIndexArrayMetadata(chunksPerShard))); + indexCodecPipeline.encode(shardIndexArray)); shardBytes.rewind(); return shardBytes; } @@ -172,25 +175,16 @@ private Array decodeInternal( DataProvider dataProvider, long[] offset, int[] shape, ArrayMetadata.CoreArrayMetadata arrayMetadata ) throws ZarrException { + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; + final Array outputArray = Array.factory(arrayMetadata.dataType.getMA2DataType(), shape); - final int[] chunksPerShard = getChunksPerShard(arrayMetadata); final int shardIndexByteLength = (int) getShardIndexSize(arrayMetadata); ByteBuffer shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength); if (shardIndexBytes == null) { throw new ZarrException("Could not read shard index."); } - final Array shardIndexArray = indexCodecPipeline.decode( - shardIndexBytes, - getShardIndexArrayMetadata(chunksPerShard) - ); - - final ArrayMetadata.CoreArrayMetadata shardMetadata = - new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), - configuration.chunkShape, arrayMetadata.dataType, - arrayMetadata.parsedFillValue - ); - + final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, shardMetadata.chunkShape, offset, shape); @@ -215,7 +209,7 @@ private Array decodeInternal( throw new ZarrException(String.format("Could not load byte data for chunk %s", Arrays.toString(chunkCoords))); } - chunkArray = codecPipeline.decode(chunkBytes, shardMetadata); + chunkArray = codecPipeline.decode(chunkBytes); } if (chunkArray == null) { chunkArray = shardMetadata.allocateFillValueChunk(); @@ -232,17 +226,13 @@ private Array decodeInternal( } @Override - public Array decodePartial( - StoreHandle chunkHandle, long[] offset, int[] shape, - ArrayMetadata.CoreArrayMetadata arrayMetadata - ) throws ZarrException { + public Array decodePartial(StoreHandle chunkHandle, long[] offset, int[] shape) throws ZarrException { if (Arrays.equals(shape, arrayMetadata.chunkShape)) { ByteBuffer chunkBytes = chunkHandle.read(); if (chunkBytes == null) { return arrayMetadata.allocateFillValueChunk(); } - return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, - arrayMetadata); + return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, arrayMetadata); } return decodeInternal(new StoreHandleDataProvider(chunkHandle), offset, shape, arrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index a65d9b5..4d614ae 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -5,62 +5,87 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.ArrayArrayCodec; -import javax.annotation.Nonnull; import ucar.ma2.Array; -public class TransposeCodec implements ArrayArrayCodec { +import javax.annotation.Nonnull; +import java.util.Arrays; + +import static dev.zarr.zarrjava.utils.Utils.inversePermutation; +import static dev.zarr.zarrjava.utils.Utils.isPermutation; - @Nonnull - public final String name = "transpose"; - @Nonnull - public final Configuration configuration; +public class TransposeCodec extends ArrayArrayCodec { - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public TransposeCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration - ) { - this.configuration = configuration; - } + @Nonnull + public final String name = "transpose"; + @Nonnull + public final Configuration configuration; - int[] reverseDims(int ndim) { - int[] dims = new int[ndim]; - for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - dims[dimIdx] = ndim - dimIdx - 1; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public TransposeCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; } - return dims; - } - @Override - public Array decode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { - if (configuration.order.equals("F")) { - chunkArray.permute(reverseDims(arrayMetadata.ndim())); + + @Override + public Array decode(Array chunkArray) throws ZarrException { + if (!isPermutation(configuration.order)){ + throw new ZarrException("Order is no permutation array"); + } + if (arrayMetadata.ndim() != configuration.order.length) { + throw new ZarrException("Array has not the same ndim as transpose codec order"); + } + chunkArray = chunkArray.permute(inversePermutation(configuration.order)); + return chunkArray; } - return chunkArray; - } - @Override - public Array encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { - if (configuration.order.equals("F")) { - chunkArray.permute(reverseDims(arrayMetadata.ndim())); + + + @Override + public Array encode(Array chunkArray) throws ZarrException { + if (!isPermutation(configuration.order)){ + throw new ZarrException("Order is no permutation array"); + } + if (arrayMetadata.ndim() != configuration.order.length) { + throw new ZarrException("Array has not the same ndim as transpose codec order"); + } + chunkArray = chunkArray.permute(configuration.order); + return chunkArray; } - return chunkArray; - } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength; - } + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength; + } - public static final class Configuration { + public static final class Configuration { + public final int[] order; - // TODO: order 'C' and 'F' are deprecated - // https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/v1.0.html#transpose-codec-v1 - public final int[] order; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "order") int[] order) { + this.order = order; + } + } - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "order") int[] order){ - this.order = order; + @Override + protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + super.resolveArrayMetadata(); + assert arrayMetadata.ndim() == configuration.order.length; + + int[] transposedChunkShape = new int[arrayMetadata.ndim()]; + Arrays.setAll(transposedChunkShape, i -> arrayMetadata.chunkShape[configuration.order[i]]); + + //only chunk shape gets transformed, the outer shape stays the same + long[] transposedArrayShape = new long[arrayMetadata.ndim()]; + Arrays.setAll(transposedArrayShape, i -> arrayMetadata.shape[i]/arrayMetadata.chunkShape[i]*transposedArrayShape[i]); + + return new ArrayMetadata.CoreArrayMetadata( + transposedArrayShape, + transposedChunkShape, + arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 39a69ee..6472aff 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -16,7 +16,7 @@ import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class ZstdCodec implements BytesBytesCodec { +public class ZstdCodec extends BytesBytesCodec { public final String name = "zstd"; @Nonnull @@ -37,7 +37,7 @@ private void copy(InputStream inputStream, OutputStream outputStream) throws IOE } @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdInputStream inputStream = new ZstdInputStream( new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { @@ -50,7 +50,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdOutputStream zstdStream = new ZstdOutputStream( outputStream, configuration.level).setChecksum( diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java new file mode 100644 index 0000000..49c4689 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -0,0 +1,31 @@ +package dev.zarr.zarrjava; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static dev.zarr.zarrjava.utils.Utils.inversePermutation; +import static dev.zarr.zarrjava.utils.Utils.isPermutation; +import static org.junit.Assert.assertFalse; + +public class TestUtils { + @Test + public void testIsPermutation(){ + assert isPermutation(new int[]{2, 1, 0}); + assert isPermutation(new int[]{4, 2, 1, 3, 0}); + assert !isPermutation(new int[]{0, 1, 2, 0}); + assert !isPermutation(new int[]{0, 1, 2, 3, 5}); + assert !isPermutation(new int[]{}); + } + + @Test + public void testInversePermutation(){ + Assertions.assertArrayEquals(new int[]{1, 0, 2}, inversePermutation(new int[]{1, 0, 2})); + Assertions.assertArrayEquals(new int[]{2, 0, 1}, inversePermutation(new int[]{1, 2, 0})); + Assertions.assertArrayEquals(new int[]{0, 3, 2, 4, 1}, inversePermutation(new int[]{0, 4, 2, 1, 3})); + Assertions.assertFalse(Arrays.equals(new int[]{2, 0, 1}, inversePermutation(new int[]{2, 0, 1}))); + } + +} + diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index e90ba28..07bb43b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -155,7 +155,7 @@ public void testWriteToZarrita(String codec) throws IOException, ZarrException, builder = builder.withCodecs(c -> c.withBytes("LITTLE")); break; case "transpose": - builder = builder.withCodecs(c -> c.withTranspose("F")); + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0})); break; case "sharding": builder = builder.withCodecs(c -> c.withSharding(new int[]{4, 4}, c1 -> c1.withBytes("LITTLE"))); @@ -223,7 +223,7 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException, builder = builder.withCodecs(c -> c.withBytes("LITTLE")); break; case "transpose": - builder = builder.withCodecs(c -> c.withTranspose("F")); + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); break; case "sharding": builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"))); @@ -253,7 +253,7 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException, public void testCodecTranspose() throws IOException, ZarrException, InterruptedException { ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); ArrayMetadata.CoreArrayMetadata metadata = new ArrayMetadata.CoreArrayMetadata( @@ -265,12 +265,16 @@ public void testCodecTranspose() throws IOException, ZarrException, InterruptedE TransposeCodec transposeCodecWrongOrder1 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 2})); TransposeCodec transposeCodecWrongOrder2 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3})); TransposeCodec transposeCodecWrongOrder3 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3, 0})); - - assertEquals(testDataTransposed120, transposeCodec.encode(testData, metadata)); - assertEquals(testData, transposeCodec.decode(testDataTransposed120, metadata)); - assertThrows(ZarrException.class, () -> transposeCodecWrongOrder1.encode(testData, metadata)); - assertThrows(ZarrException.class, () -> transposeCodecWrongOrder2.encode(testData, metadata)); - assertThrows(ZarrException.class, () -> transposeCodecWrongOrder3.encode(testData, metadata)); + transposeCodec.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder1.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder2.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder3.setCoreArrayMetadata(metadata); + + assert ucar.ma2.MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert ucar.ma2.MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder1.encode(testData)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder2.encode(testData)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder3.encode(testData)); } @Test diff --git a/src/test/java/dev/zarr/zarrjava/zarrita_read.py b/src/test/java/dev/zarr/zarrjava/zarrita_read.py index 5272521..f8c262e 100644 --- a/src/test/java/dev/zarr/zarrjava/zarrita_read.py +++ b/src/test/java/dev/zarr/zarrjava/zarrita_read.py @@ -14,7 +14,7 @@ elif codec_string == "bytes": codec = [zarrita.codecs.bytes_codec()] elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec("F"), zarrita.codecs.bytes_codec()] + codec = [zarrita.codecs.transpose_codec((1, 0)), zarrita.codecs.bytes_codec()] elif codec_string == "sharding": codec= zarrita.codecs.sharding_codec(chunk_shape=(4, 4), codecs=[zarrita.codecs.bytes_codec("little")]), elif codec_string == "crc32c": diff --git a/src/test/java/dev/zarr/zarrjava/zarrita_write.py b/src/test/java/dev/zarr/zarrjava/zarrita_write.py index bb306ca..b1976b7 100644 --- a/src/test/java/dev/zarr/zarrjava/zarrita_write.py +++ b/src/test/java/dev/zarr/zarrjava/zarrita_write.py @@ -14,7 +14,7 @@ elif codec_string == "bytes": codec = [zarrita.codecs.bytes_codec()] elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec("F"), zarrita.codecs.bytes_codec()] + codec = [zarrita.codecs.transpose_codec([0, 1]), zarrita.codecs.bytes_codec()] elif codec_string == "sharding": codec = [zarrita.codecs.sharding_codec(chunk_shape=(1, 2), codecs=[zarrita.codecs.bytes_codec()])] elif codec_string == "crc32c": From d30475a206f5de56982763fed01a79f57b2aa668 Mon Sep 17 00:00:00 2001 From: brokkoli71 <44113112+brokkoli71@users.noreply.github.com> Date: Fri, 24 May 2024 16:19:54 +0200 Subject: [PATCH 4/7] update java version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19dce7f..e913c2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v3 with: - java-version: '8' + java-version: '22' distribution: 'temurin' cache: maven From e50247997d15099646c6c57bc8ea885ebbeeb2ed Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 24 May 2024 16:33:47 +0200 Subject: [PATCH 5/7] remove wrong dependency --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 01e5bc2..676ec33 100644 --- a/pom.xml +++ b/pom.xml @@ -68,12 +68,6 @@ 4.13.1 test - - dev.zarr.zarrjava - zarr-java - 0.0.1-SNAPSHOT - test - From 292ecc656db6887309e20ad0452d37c816722eb4 Mon Sep 17 00:00:00 2001 From: brokkoli71 <44113112+brokkoli71@users.noreply.github.com> Date: Fri, 24 May 2024 16:36:38 +0200 Subject: [PATCH 6/7] set fail-fast: false --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e913c2d..fe32a53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: strategy: matrix: os: [ ubuntu, windows, macos ] + fail-fast: false runs-on: ${{ matrix.os }}-latest defaults: run: From 30bb41d717bc68e65d144eecf1d63b63faca4c5c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 24 May 2024 16:51:17 +0200 Subject: [PATCH 7/7] specify testSourceDirectory --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 676ec33..7a7e473 100644 --- a/pom.xml +++ b/pom.xml @@ -78,4 +78,7 @@ + + src/test/java/dev/zarr/zarrjava + \ No newline at end of file