diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 19dce7f..fe32a53 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,6 +12,7 @@ jobs:
strategy:
matrix:
os: [ ubuntu, windows, macos ]
+ fail-fast: false
runs-on: ${{ matrix.os }}-latest
defaults:
run:
@@ -22,7 +23,7 @@ jobs:
- name: Set up JDK
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: '22'
distribution: 'temurin'
cache: maven
diff --git a/pom.xml b/pom.xml
index 01e5bc2..7a7e473 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,12 +68,6 @@
4.13.1
test
-
- dev.zarr.zarrjava
- zarr-java
- 0.0.1-SNAPSHOT
- test
-
@@ -84,4 +78,7 @@
+
+ src/test/java/dev/zarr/zarrjava
+
\ No newline at end of file
diff --git a/src/main/java/dev/zarr/zarrjava/utils/Utils.java b/src/main/java/dev/zarr/zarrjava/utils/Utils.java
index 93b7cf1..3ea4bfd 100644
--- a/src/main/java/dev/zarr/zarrjava/utils/Utils.java
+++ b/src/main/java/dev/zarr/zarrjava/utils/Utils.java
@@ -77,4 +77,24 @@ public static T[] concatArrays(T[] array1, T[]... arrays) {
}
return result;
}
+
+ public static boolean isPermutation(int[] array) {
+ if (array.length==0){
+ return false;
+ }
+ int[] arange = new int[array.length];
+ Arrays.setAll(arange, i -> i);
+ int[] orderSorted = array.clone();
+ Arrays.sort(orderSorted);
+ return Arrays.equals(orderSorted, arange);
+ }
+
+ public static int[] inversePermutation(int[] origin){
+ assert isPermutation(origin);
+ int[] inverse = new int[origin.length];
+ for (int i = 0; i < origin.length; i++) {
+ inverse[origin[i]] = i;
+ }
+ return inverse;
+ }
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java
index 631ef70..85e0457 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/Array.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java
@@ -27,7 +27,7 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata)
throws IOException, ZarrException {
super(storeHandle);
this.metadata = arrayMetadata;
- this.codecPipeline = new CodecPipeline(arrayMetadata.codecs);
+ this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata);
}
/**
@@ -171,8 +171,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
if (codecPipeline.supportsPartialDecode()) {
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
- Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape,
- metadata.coreArrayMetadata);
+ Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
chunkProjection.outOffset, chunkProjection.shape
);
@@ -223,7 +222,7 @@ public ucar.ma2.Array readChunk(long[] chunkCoords)
return metadata.allocateFillValueChunk();
}
- return codecPipeline.decode(chunkBytes, metadata.coreArrayMetadata);
+ return codecPipeline.decode(chunkBytes);
}
/**
@@ -299,7 +298,7 @@ public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Zar
if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue)) {
chunkHandle.delete();
} else {
- ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, metadata.coreArrayMetadata);
+ ByteBuffer chunkBytes = codecPipeline.encode(chunkArray);
chunkHandle.set(chunkBytes);
}
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java
index 868183a..a488d30 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java
@@ -1,15 +1,14 @@
package dev.zarr.zarrjava.v3.codec;
import dev.zarr.zarrjava.ZarrException;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
import ucar.ma2.Array;
-public interface ArrayArrayCodec extends Codec {
+public abstract class ArrayArrayCodec extends Codec {
- Array encode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+ protected abstract Array encode(Array chunkArray)
throws ZarrException;
- Array decode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+ protected abstract Array decode(Array chunkArray)
throws ZarrException;
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java
index ad042ac..361ae61 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java
@@ -2,23 +2,24 @@
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.store.StoreHandle;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
import java.nio.ByteBuffer;
import ucar.ma2.Array;
-public interface ArrayBytesCodec extends Codec {
+public abstract class ArrayBytesCodec extends Codec {
- ByteBuffer encode(Array chunkArray, CoreArrayMetadata arrayMetadata)
+ protected abstract ByteBuffer encode(Array chunkArray)
throws ZarrException;
- Array decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
+ protected abstract Array decode(ByteBuffer chunkBytes)
throws ZarrException;
- interface WithPartialDecode extends ArrayBytesCodec {
+ public abstract static class WithPartialDecode extends ArrayBytesCodec {
- Array decodePartial(
- StoreHandle handle, long[] offset, int[] shape,
- CoreArrayMetadata arrayMetadata
+ public abstract Array decode(ByteBuffer shardBytes) throws ZarrException;
+ public abstract ByteBuffer encode(Array shardArray) throws ZarrException;
+
+ protected abstract Array decodePartial(
+ StoreHandle handle, long[] offset, int[] shape
) throws ZarrException;
}
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java
index d8ce47a..6435463 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java
@@ -1,15 +1,13 @@
package dev.zarr.zarrjava.v3.codec;
import dev.zarr.zarrjava.ZarrException;
-import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
+
import java.nio.ByteBuffer;
-public interface BytesBytesCodec extends Codec {
+public abstract class BytesBytesCodec extends Codec {
- ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
- throws ZarrException;
+ protected abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException;
- ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
- throws ZarrException;
+ public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException;
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java
index 6ce7687..988dd1d 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java
@@ -5,9 +5,22 @@
import dev.zarr.zarrjava.v3.ArrayMetadata;
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name")
-public interface Codec {
+public abstract class Codec {
- long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata)
- throws ZarrException;
+ protected ArrayMetadata.CoreArrayMetadata arrayMetadata;
+
+ protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException {
+ if (arrayMetadata == null) {
+ throw new ZarrException("arrayMetadata needs to get set in for every codec");
+ }
+ return this.arrayMetadata;
+ }
+
+ protected abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ throws ZarrException;
+
+ public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{
+ this.arrayMetadata = arrayMetadata;
+ }
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java
index 5e33d7c..bc59f4e 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java
@@ -62,13 +62,9 @@ public CodecBuilder withBlosc() {
return withBlosc("zstd");
}
- public CodecBuilder withTranspose(String order) {
- try {
+ public CodecBuilder withTranspose(int[] order) {
codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order)));
- } catch (ZarrException e) {
- throw new RuntimeException(e);
- }
- return this;
+ return this;
}
public CodecBuilder withBytes(Endian endian) {
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java
index 9ece0f0..920a1f4 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java
@@ -12,8 +12,10 @@ public class CodecPipeline {
@Nonnull
final Codec[] codecs;
+ public final CoreArrayMetadata arrayMetadata;
- public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException {
+ public CodecPipeline(@Nonnull Codec[] codecs, CoreArrayMetadata arrayMetadata) throws ZarrException {
+ this.arrayMetadata = arrayMetadata;
long arrayBytesCodecCount = Arrays.stream(codecs).filter(c -> c instanceof ArrayBytesCodec)
.count();
if (arrayBytesCodecCount != 1) {
@@ -21,6 +23,7 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException {
"Exactly 1 ArrayBytesCodec is required. Found " + arrayBytesCodecCount + ".");
}
Codec prevCodec = null;
+ CoreArrayMetadata codecArrayMetadata = arrayMetadata;
for (Codec codec : codecs) {
if (prevCodec != null) {
if (codec instanceof ArrayBytesCodec && prevCodec instanceof ArrayBytesCodec) {
@@ -44,6 +47,8 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException {
prevCodec.getClass() + "'.");
}
}
+ codec.setCoreArrayMetadata(codecArrayMetadata);
+ codecArrayMetadata = codec.resolveArrayMetadata();
prevCodec = codec;
}
@@ -79,15 +84,14 @@ public boolean supportsPartialDecode() {
@Nonnull
public Array decodePartial(
@Nonnull StoreHandle storeHandle,
- long[] offset, int[] shape,
- @Nonnull CoreArrayMetadata arrayMetadata
+ long[] offset, int[] shape
) throws ZarrException {
if (!supportsPartialDecode()) {
throw new ZarrException(
"Partial decode is not supported for these codecs. " + Arrays.toString(codecs));
}
Array chunkArray = ((ArrayBytesCodec.WithPartialDecode) getArrayBytesCodec()).decodePartial(
- storeHandle, offset, shape, arrayMetadata);
+ storeHandle, offset, shape);
if (chunkArray == null) {
throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs.");
}
@@ -96,8 +100,7 @@ public Array decodePartial(
@Nonnull
public Array decode(
- @Nonnull ByteBuffer chunkBytes,
- @Nonnull CoreArrayMetadata arrayMetadata
+ @Nonnull ByteBuffer chunkBytes
) throws ZarrException {
if (chunkBytes == null) {
throw new ZarrException("chunkBytes is null. Ohh nooo.");
@@ -106,7 +109,7 @@ public Array decode(
BytesBytesCodec[] bytesBytesCodecs = getBytesBytesCodecs();
for (int i = bytesBytesCodecs.length - 1; i >= 0; --i) {
BytesBytesCodec codec = bytesBytesCodecs[i];
- chunkBytes = codec.decode(chunkBytes, arrayMetadata);
+ chunkBytes = codec.decode(chunkBytes);
}
if (chunkBytes == null) {
@@ -114,7 +117,7 @@ public Array decode(
"chunkBytes is null. This is likely a bug in one of the codecs. " + Arrays.toString(
getBytesBytesCodecs()));
}
- Array chunkArray = getArrayBytesCodec().decode(chunkBytes, arrayMetadata);
+ Array chunkArray = getArrayBytesCodec().decode(chunkBytes);
if (chunkArray == null) {
throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs.");
}
@@ -122,7 +125,7 @@ public Array decode(
ArrayArrayCodec[] arrayArrayCodecs = getArrayArrayCodecs();
for (int i = arrayArrayCodecs.length - 1; i >= 0; --i) {
ArrayArrayCodec codec = arrayArrayCodecs[i];
- chunkArray = codec.decode(chunkArray, arrayMetadata);
+ chunkArray = codec.decode(chunkArray);
}
if (chunkArray == null) {
@@ -133,16 +136,16 @@ public Array decode(
@Nonnull
public ByteBuffer encode(
- @Nonnull Array chunkArray, @Nonnull CoreArrayMetadata arrayMetadata
+ @Nonnull Array chunkArray
) throws ZarrException {
for (ArrayArrayCodec codec : getArrayArrayCodecs()) {
- chunkArray = codec.encode(chunkArray, arrayMetadata);
+ chunkArray = codec.encode(chunkArray);
}
- ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray, arrayMetadata);
+ ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray);
for (BytesBytesCodec codec : getBytesBytesCodecs()) {
- chunkBytes = codec.encode(chunkBytes, arrayMetadata);
+ chunkBytes = codec.encode(chunkBytes);
}
return chunkBytes;
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java
index 2c3412b..1661304 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java
@@ -20,7 +20,7 @@
import java.nio.ByteBuffer;
import javax.annotation.Nonnull;
-public class BloscCodec implements BytesBytesCodec {
+public class BloscCodec extends BytesBytesCodec {
public final String name = "blosc";
@Nonnull
@@ -33,7 +33,7 @@ public BloscCodec(
}
@Override
- public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer decode(ByteBuffer chunkBytes)
throws ZarrException {
try {
return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes)));
@@ -43,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata
}
@Override
- public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer encode(ByteBuffer chunkBytes)
throws ZarrException {
try {
return ByteBuffer.wrap(
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java
index de9d1e7..1415da9 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java
@@ -11,7 +11,7 @@
import javax.annotation.Nonnull;
import ucar.ma2.Array;
-public class BytesCodec implements ArrayBytesCodec {
+public class BytesCodec extends ArrayBytesCodec {
public final String name = "bytes";
@Nonnull
@@ -29,14 +29,14 @@ public BytesCodec(Endian endian) {
}
@Override
- public Array decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) {
+ public Array decode(ByteBuffer chunkBytes) {
chunkBytes.order(configuration.endian.getByteOrder());
return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape,
chunkBytes);
}
@Override
- public ByteBuffer encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) {
+ public ByteBuffer encode(Array chunkArray) {
return chunkArray.getDataAsByteBuffer(configuration.endian.getByteOrder());
}
@@ -72,7 +72,7 @@ public ByteOrder getByteOrder() {
}
}
- public static final class Configuration {
+ public static final class Configuration{
@Nonnull
public final BytesCodec.Endian endian;
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java
index 38be686..a1e3cb5 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java
@@ -9,17 +9,15 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-public class Crc32cCodec implements BytesBytesCodec {
+public class Crc32cCodec extends BytesBytesCodec {
public final String name = "crc32c";
@JsonCreator
- public Crc32cCodec(
- ) {
- }
+ public Crc32cCodec(){}
@Override
- public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
+ public ByteBuffer decode(ByteBuffer chunkBytes)
throws ZarrException {
ByteBuffer buffer = chunkBytes.slice();
buffer.order(ByteOrder.LITTLE_ENDIAN);
@@ -45,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata)
}
@Override
- public ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) {
+ public ByteBuffer encode(ByteBuffer chunkBytes) {
return Utils.makeByteBuffer(chunkBytes.capacity() + 4, b -> {
final CRC32C crc32c = new CRC32C();
crc32c.update(chunkBytes);
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java
index 1054542..3ff5acd 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java
@@ -16,7 +16,7 @@
import java.util.zip.GZIPOutputStream;
import javax.annotation.Nonnull;
-public class GzipCodec implements BytesBytesCodec {
+public class GzipCodec extends BytesBytesCodec {
public final String name = "gzip";
@Nonnull
@@ -37,7 +37,7 @@ private void copy(InputStream inputStream, OutputStream outputStream) throws IOE
}
@Override
- public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer decode(ByteBuffer chunkBytes)
throws ZarrException {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream(
new ByteArrayInputStream(Utils.toArray(chunkBytes)))) {
@@ -50,7 +50,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata
}
@Override
- public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer encode(ByteBuffer chunkBytes)
throws ZarrException {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream(
outputStream)) {
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java
index f4237df..035fc7b 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java
@@ -22,15 +22,13 @@
import ucar.ma2.InvalidRangeException;
-public class ShardingIndexedCodec implements ArrayBytesCodec, ArrayBytesCodec.WithPartialDecode {
+public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode {
public final String name = "sharding_indexed";
@Nonnull
public final Configuration configuration;
- @Nonnull
- final CodecPipeline codecPipeline;
- @Nonnull
- final CodecPipeline indexCodecPipeline;
+ CodecPipeline codecPipeline;
+ CodecPipeline indexCodecPipeline;
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public ShardingIndexedCodec(
@@ -38,8 +36,18 @@ public ShardingIndexedCodec(
Configuration configuration
) throws ZarrException {
this.configuration = configuration;
- this.codecPipeline = new CodecPipeline(configuration.codecs);
- this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs);
+ }
+
+ @Override
+ public void setCoreArrayMetadata(CoreArrayMetadata arrayMetadata) throws ZarrException {
+ super.setCoreArrayMetadata(arrayMetadata);
+ final ArrayMetadata.CoreArrayMetadata shardMetadata =
+ new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape),
+ configuration.chunkShape, arrayMetadata.dataType,
+ arrayMetadata.parsedFillValue
+ );
+ this.codecPipeline = new CodecPipeline(configuration.codecs, shardMetadata);
+ this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs, getShardIndexArrayMetadata(getChunksPerShard(arrayMetadata)));
}
ArrayMetadata.CoreArrayMetadata getShardIndexArrayMetadata(int[] chunksPerShard) {
@@ -85,20 +93,15 @@ void setValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int
}
@Override
- public Array decode(ByteBuffer shardBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public Array decode(ByteBuffer shardBytes)
throws ZarrException {
return decodeInternal(new ByteBufferDataProvider(shardBytes), new long[arrayMetadata.ndim()],
arrayMetadata.chunkShape, arrayMetadata);
}
@Override
- public ByteBuffer encode(final Array shardArray,
- final ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
- final ArrayMetadata.CoreArrayMetadata shardMetadata =
- new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape),
- configuration.chunkShape, arrayMetadata.dataType,
- arrayMetadata.parsedFillValue
- );
+ public ByteBuffer encode(final Array shardArray) throws ZarrException {
+ final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata;
final int[] chunksPerShard = getChunksPerShard(arrayMetadata);
final int chunkCount = Arrays.stream(chunksPerShard)
.reduce(1, (r, a) -> r * a);
@@ -127,7 +130,7 @@ public ByteBuffer encode(final Array shardArray,
setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, -1);
setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, -1);
} else {
- final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, shardMetadata);
+ final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray);
synchronized (chunkBytesList) {
int chunkByteOffset = chunkBytesList.stream()
.mapToInt(ByteBuffer::capacity)
@@ -150,7 +153,7 @@ public ByteBuffer encode(final Array shardArray,
shardBytes.put(chunkBytes);
}
shardBytes.put(
- indexCodecPipeline.encode(shardIndexArray, getShardIndexArrayMetadata(chunksPerShard)));
+ indexCodecPipeline.encode(shardIndexArray));
shardBytes.rewind();
return shardBytes;
}
@@ -172,25 +175,16 @@ private Array decodeInternal(
DataProvider dataProvider, long[] offset, int[] shape,
ArrayMetadata.CoreArrayMetadata arrayMetadata
) throws ZarrException {
+ final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata;
+
final Array outputArray = Array.factory(arrayMetadata.dataType.getMA2DataType(), shape);
- final int[] chunksPerShard = getChunksPerShard(arrayMetadata);
final int shardIndexByteLength = (int) getShardIndexSize(arrayMetadata);
ByteBuffer shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength);
if (shardIndexBytes == null) {
throw new ZarrException("Could not read shard index.");
}
- final Array shardIndexArray = indexCodecPipeline.decode(
- shardIndexBytes,
- getShardIndexArrayMetadata(chunksPerShard)
- );
-
- final ArrayMetadata.CoreArrayMetadata shardMetadata =
- new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape),
- configuration.chunkShape, arrayMetadata.dataType,
- arrayMetadata.parsedFillValue
- );
-
+ final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes);
long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape,
shardMetadata.chunkShape, offset,
shape);
@@ -215,7 +209,7 @@ private Array decodeInternal(
throw new ZarrException(String.format("Could not load byte data for chunk %s",
Arrays.toString(chunkCoords)));
}
- chunkArray = codecPipeline.decode(chunkBytes, shardMetadata);
+ chunkArray = codecPipeline.decode(chunkBytes);
}
if (chunkArray == null) {
chunkArray = shardMetadata.allocateFillValueChunk();
@@ -232,17 +226,13 @@ private Array decodeInternal(
}
@Override
- public Array decodePartial(
- StoreHandle chunkHandle, long[] offset, int[] shape,
- ArrayMetadata.CoreArrayMetadata arrayMetadata
- ) throws ZarrException {
+ public Array decodePartial(StoreHandle chunkHandle, long[] offset, int[] shape) throws ZarrException {
if (Arrays.equals(shape, arrayMetadata.chunkShape)) {
ByteBuffer chunkBytes = chunkHandle.read();
if (chunkBytes == null) {
return arrayMetadata.allocateFillValueChunk();
}
- return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape,
- arrayMetadata);
+ return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, arrayMetadata);
}
return decodeInternal(new StoreHandleDataProvider(chunkHandle), offset, shape, arrayMetadata);
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java
index 69fb6fe..4d614ae 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java
@@ -5,64 +5,87 @@
import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.v3.ArrayMetadata;
import dev.zarr.zarrjava.v3.codec.ArrayArrayCodec;
-import javax.annotation.Nonnull;
import ucar.ma2.Array;
-public class TransposeCodec implements ArrayArrayCodec {
+import javax.annotation.Nonnull;
+import java.util.Arrays;
+
+import static dev.zarr.zarrjava.utils.Utils.inversePermutation;
+import static dev.zarr.zarrjava.utils.Utils.isPermutation;
- @Nonnull
- public final String name = "transpose";
- @Nonnull
- public final Configuration configuration;
+public class TransposeCodec extends ArrayArrayCodec {
- @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
- public TransposeCodec(
- @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration
- ) {
- this.configuration = configuration;
- }
+ @Nonnull
+ public final String name = "transpose";
+ @Nonnull
+ public final Configuration configuration;
- int[] reverseDims(int ndim) {
- int[] dims = new int[ndim];
- for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
- dims[dimIdx] = ndim - dimIdx - 1;
+ @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+ public TransposeCodec(
+ @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration
+ ) {
+ this.configuration = configuration;
}
- return dims;
- }
- @Override
- public Array decode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) {
- if (configuration.order.equals("F")) {
- chunkArray.permute(reverseDims(arrayMetadata.ndim()));
+
+ @Override
+ public Array decode(Array chunkArray) throws ZarrException {
+ if (!isPermutation(configuration.order)){
+ throw new ZarrException("Order is no permutation array");
+ }
+ if (arrayMetadata.ndim() != configuration.order.length) {
+ throw new ZarrException("Array has not the same ndim as transpose codec order");
+ }
+ chunkArray = chunkArray.permute(inversePermutation(configuration.order));
+ return chunkArray;
}
- return chunkArray;
- }
- @Override
- public Array encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) {
- if (configuration.order.equals("F")) {
- chunkArray.permute(reverseDims(arrayMetadata.ndim()));
+
+
+ @Override
+ public Array encode(Array chunkArray) throws ZarrException {
+ if (!isPermutation(configuration.order)){
+ throw new ZarrException("Order is no permutation array");
+ }
+ if (arrayMetadata.ndim() != configuration.order.length) {
+ throw new ZarrException("Array has not the same ndim as transpose codec order");
+ }
+ chunkArray = chunkArray.permute(configuration.order);
+ return chunkArray;
}
- return chunkArray;
- }
- @Override
- public long computeEncodedSize(long inputByteLength,
- ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
- return inputByteLength;
- }
+ @Override
+ public long computeEncodedSize(long inputByteLength,
+ ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
+ return inputByteLength;
+ }
- public static final class Configuration {
+ public static final class Configuration {
+ public final int[] order;
- public final String order;
+ @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+ public Configuration(@JsonProperty(value = "order") int[] order) {
+ this.order = order;
+ }
+ }
- @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
- public Configuration(@JsonProperty(value = "order", defaultValue = "C") String order)
- throws ZarrException {
- if (!order.equals("C") && !order.equals("F")) {
- throw new ZarrException("Only 'C' or 'F' are supported.");
- }
- this.order = order;
+ @Override
+ protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException {
+ super.resolveArrayMetadata();
+ assert arrayMetadata.ndim() == configuration.order.length;
+
+ int[] transposedChunkShape = new int[arrayMetadata.ndim()];
+ Arrays.setAll(transposedChunkShape, i -> arrayMetadata.chunkShape[configuration.order[i]]);
+
+ //only chunk shape gets transformed, the outer shape stays the same
+ long[] transposedArrayShape = new long[arrayMetadata.ndim()];
+ Arrays.setAll(transposedArrayShape, i -> arrayMetadata.shape[i]/arrayMetadata.chunkShape[i]*transposedArrayShape[i]);
+
+ return new ArrayMetadata.CoreArrayMetadata(
+ transposedArrayShape,
+ transposedChunkShape,
+ arrayMetadata.dataType,
+ arrayMetadata.parsedFillValue
+ );
}
- }
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java
index 39a69ee..6472aff 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java
@@ -16,7 +16,7 @@
import java.nio.ByteBuffer;
import javax.annotation.Nonnull;
-public class ZstdCodec implements BytesBytesCodec {
+public class ZstdCodec extends BytesBytesCodec {
public final String name = "zstd";
@Nonnull
@@ -37,7 +37,7 @@ private void copy(InputStream inputStream, OutputStream outputStream) throws IOE
}
@Override
- public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer decode(ByteBuffer chunkBytes)
throws ZarrException {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdInputStream inputStream = new ZstdInputStream(
new ByteArrayInputStream(Utils.toArray(chunkBytes)))) {
@@ -50,7 +50,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata
}
@Override
- public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata)
+ public ByteBuffer encode(ByteBuffer chunkBytes)
throws ZarrException {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdOutputStream zstdStream = new ZstdOutputStream(
outputStream, configuration.level).setChecksum(
diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java
new file mode 100644
index 0000000..49c4689
--- /dev/null
+++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java
@@ -0,0 +1,31 @@
+package dev.zarr.zarrjava;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static dev.zarr.zarrjava.utils.Utils.inversePermutation;
+import static dev.zarr.zarrjava.utils.Utils.isPermutation;
+import static org.junit.Assert.assertFalse;
+
+public class TestUtils {
+ @Test
+ public void testIsPermutation(){
+ assert isPermutation(new int[]{2, 1, 0});
+ assert isPermutation(new int[]{4, 2, 1, 3, 0});
+ assert !isPermutation(new int[]{0, 1, 2, 0});
+ assert !isPermutation(new int[]{0, 1, 2, 3, 5});
+ assert !isPermutation(new int[]{});
+ }
+
+ @Test
+ public void testInversePermutation(){
+ Assertions.assertArrayEquals(new int[]{1, 0, 2}, inversePermutation(new int[]{1, 0, 2}));
+ Assertions.assertArrayEquals(new int[]{2, 0, 1}, inversePermutation(new int[]{1, 2, 0}));
+ Assertions.assertArrayEquals(new int[]{0, 3, 2, 4, 1}, inversePermutation(new int[]{0, 4, 2, 1, 3}));
+ Assertions.assertFalse(Arrays.equals(new int[]{2, 0, 1}, inversePermutation(new int[]{2, 0, 1})));
+ }
+
+}
+
diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java
index e6d65b6..07bb43b 100644
--- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java
+++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java
@@ -10,6 +10,7 @@
import dev.zarr.zarrjava.store.StoreHandle;
import dev.zarr.zarrjava.utils.MultiArrayUtils;
import dev.zarr.zarrjava.v3.*;
+import dev.zarr.zarrjava.v3.codec.core.TransposeCodec;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
@@ -29,18 +30,15 @@
import java.util.Map;
import java.util.stream.Stream;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
public class ZarrTest {
final static Path TESTDATA = Paths.get("testdata");
final static Path TESTOUTPUT = Paths.get("testoutput");
-
//TODO: is the Path with / instead of \ readable in Windows?
final static Path ZARRITA_WRITE_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_write.py");
final static Path ZARRITA_READ_PATH = Paths.get("src/test/java/dev/zarr/zarrjava/zarrita_read.py");
-
final static String CONDA_ENVIRONMENT = "zarrita_env";
@BeforeAll
@@ -157,7 +155,7 @@ public void testWriteToZarrita(String codec) throws IOException, ZarrException,
builder = builder.withCodecs(c -> c.withBytes("LITTLE"));
break;
case "transpose":
- builder = builder.withCodecs(c -> c.withTranspose("F"));
+ builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0}));
break;
case "sharding":
builder = builder.withCodecs(c -> c.withSharding(new int[]{4, 4}, c1 -> c1.withBytes("LITTLE")));
@@ -225,7 +223,7 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException,
builder = builder.withCodecs(c -> c.withBytes("LITTLE"));
break;
case "transpose":
- builder = builder.withCodecs(c -> c.withTranspose("F"));
+ builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2}));
break;
case "sharding":
builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE")));
@@ -251,6 +249,34 @@ public void testCodecsWriteRead(String codec) throws IOException, ZarrException,
Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT));
}
+ @Test
+ public void testCodecTranspose() throws IOException, ZarrException, InterruptedException {
+ ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17});
+ ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{
+ 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17});
+
+ ArrayMetadata.CoreArrayMetadata metadata = new ArrayMetadata.CoreArrayMetadata(
+ new long[]{2, 3, 3},
+ new int[]{2, 3, 3},
+ DataType.UINT32,
+ null);
+ TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0}));
+ TransposeCodec transposeCodecWrongOrder1 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 2}));
+ TransposeCodec transposeCodecWrongOrder2 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3}));
+ TransposeCodec transposeCodecWrongOrder3 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3, 0}));
+ transposeCodec.setCoreArrayMetadata(metadata);
+ transposeCodecWrongOrder1.setCoreArrayMetadata(metadata);
+ transposeCodecWrongOrder2.setCoreArrayMetadata(metadata);
+ transposeCodecWrongOrder3.setCoreArrayMetadata(metadata);
+
+ assert ucar.ma2.MAMath.equals(testDataTransposed120, transposeCodec.encode(testData));
+ assert ucar.ma2.MAMath.equals(testData, transposeCodec.decode(testDataTransposed120));
+ assertThrows(ZarrException.class, () -> transposeCodecWrongOrder1.encode(testData));
+ assertThrows(ZarrException.class, () -> transposeCodecWrongOrder2.encode(testData));
+ assertThrows(ZarrException.class, () -> transposeCodecWrongOrder3.encode(testData));
+ }
+
@Test
public void testFileSystemStores() throws IOException, ZarrException {
FilesystemStore fsStore = new FilesystemStore(TESTDATA);
diff --git a/src/test/java/dev/zarr/zarrjava/zarrita_read.py b/src/test/java/dev/zarr/zarrjava/zarrita_read.py
index 5272521..f8c262e 100644
--- a/src/test/java/dev/zarr/zarrjava/zarrita_read.py
+++ b/src/test/java/dev/zarr/zarrjava/zarrita_read.py
@@ -14,7 +14,7 @@
elif codec_string == "bytes":
codec = [zarrita.codecs.bytes_codec()]
elif codec_string == "transpose":
- codec = [zarrita.codecs.transpose_codec("F"), zarrita.codecs.bytes_codec()]
+ codec = [zarrita.codecs.transpose_codec((1, 0)), zarrita.codecs.bytes_codec()]
elif codec_string == "sharding":
codec= zarrita.codecs.sharding_codec(chunk_shape=(4, 4), codecs=[zarrita.codecs.bytes_codec("little")]),
elif codec_string == "crc32c":
diff --git a/src/test/java/dev/zarr/zarrjava/zarrita_write.py b/src/test/java/dev/zarr/zarrjava/zarrita_write.py
index bb306ca..b1976b7 100644
--- a/src/test/java/dev/zarr/zarrjava/zarrita_write.py
+++ b/src/test/java/dev/zarr/zarrjava/zarrita_write.py
@@ -14,7 +14,7 @@
elif codec_string == "bytes":
codec = [zarrita.codecs.bytes_codec()]
elif codec_string == "transpose":
- codec = [zarrita.codecs.transpose_codec("F"), zarrita.codecs.bytes_codec()]
+ codec = [zarrita.codecs.transpose_codec([0, 1]), zarrita.codecs.bytes_codec()]
elif codec_string == "sharding":
codec = [zarrita.codecs.sharding_codec(chunk_shape=(1, 2), codecs=[zarrita.codecs.bytes_codec()])]
elif codec_string == "crc32c":