From 7f269ff2391a24278a6268c7acb59e1998f9579c Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Mon, 28 Jul 2025 09:34:01 +0100 Subject: [PATCH 01/17] Add support for zarr v3 --- build.gradle | 10 + .../omero/zarr/ZarrPixelBuffer.java | 98 +++---- .../omero/zarr/ZarrPixelsService.java | 29 ++- .../omero/zarr/model/ZArray.java | 18 ++ .../omero/zarr/model/ZArrayv2.java | 68 +++++ .../omero/zarr/model/ZArrayv3.java | 82 ++++++ .../omero/zarr/model/ZarrInfo.java | 239 ++++++++++++++++++ .../omero/zarr/model/ZarrPath.java | 13 + .../omero/zarr/model/ZarrPathv2.java | 28 ++ .../omero/zarr/model/ZarrPathv3.java | 28 ++ .../omero/zarr/TestZarrInfo.java | 55 ++++ .../omero/zarr/ZarrPixelBufferTest.java | 2 +- 12 files changed, 597 insertions(+), 73 deletions(-) create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java create mode 100644 src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java diff --git a/build.gradle b/build.gradle index 02f3a24..feff228 100644 --- a/build.gradle +++ b/build.gradle @@ -33,11 +33,21 @@ repositories { dependencies { api 'org.openmicroscopy:omero-blitz:5.8.3' + implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' implementation 'dev.zarr:jzarr:0.4.2' implementation 'org.lasersonlab:s3fs:2.2.3' implementation 'com.amazonaws:aws-java-sdk-s3:1.12.659' implementation 'org.apache.tika:tika-core:1.28.5' + implementation 'net.java.dev.jna:jna:5.10.0' + implementation 'ome:formats-gpl:7.3.1' + implementation 'info.picocli:picocli:4.7.5' + implementation 'com.univocity:univocity-parsers:2.8.4' + implementation 'dev.zarr:zarr-java:0.0.2' + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.3.14' + implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.3.14' + implementation 'org.apache.maven:maven-artifact:3.9.4' testImplementation 'info.picocli:picocli:4.7.5' testImplementation 'com.glencoesoftware:bioformats2raw:0.11.0' diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java index 4cd2bee..4ce44e4 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java @@ -55,8 +55,8 @@ public class ZarrPixelBuffer implements PixelBuffer { /** Reference to the pixels. */ private final Pixels pixels; - /** Root of the OME-NGFF multiscale we are operating on. */ - private final Path root; + /** Root of the OME-NGFF multiscale we are operating on */ + private final ZarrPath root; /** Requested resolution level. */ private int resolutionLevel; @@ -73,8 +73,8 @@ public class ZarrPixelBuffer implements PixelBuffer { /** Zarr attributes present on the root group. */ private final Map rootGroupAttributes; - /** Zarr array corresponding to the current resolution level. */ - private ZarrArray array; + /** Zarr array corresponding to the current resolution level */ + private ZArray array; /** * Mapping of Z plane indexes in full resolution to @@ -90,10 +90,10 @@ public class ZarrPixelBuffer implements PixelBuffer { /** Root path vs. metadata cache. */ private final - AsyncLoadingCache> zarrMetadataCache; + AsyncLoadingCache> zarrMetadataCache; - /** Array path vs. ZarrArray cache. */ - private final AsyncLoadingCache zarrArrayCache; + /** Array path vs. ZArray cache */ + private final AsyncLoadingCache zarrArrayCache; /** Supported axes, X and Y are essential. */ public enum Axis { @@ -109,10 +109,10 @@ public enum Axis { * @param pixels Pixels metadata for the pixel buffer * @param root The root of this buffer */ - public ZarrPixelBuffer(Pixels pixels, Path root, Integer maxPlaneWidth, + public ZarrPixelBuffer(Pixels pixels, ZarrPath root, Integer maxPlaneWidth, Integer maxPlaneHeight, - AsyncLoadingCache> zarrMetadataCache, - AsyncLoadingCache zarrArrayCache) + AsyncLoadingCache> zarrMetadataCache, + AsyncLoadingCache zarrArrayCache) throws IOException { log.info("Creating ZarrPixelBuffer"); this.pixels = pixels; @@ -158,36 +158,6 @@ public ZarrPixelBuffer(Pixels pixels, Path root, Integer maxPlaneWidth, }); } - /** - * Get Bio-Formats/OMERO pixels type for buffer. - * - * @return See above. - */ - public int getPixelsType() { - DataType dataType = array.getDataType(); - switch (dataType) { - case u1: - return FormatTools.UINT8; - case i1: - return FormatTools.INT8; - case u2: - return FormatTools.UINT16; - case i2: - return FormatTools.INT16; - case u4: - return FormatTools.UINT32; - case i4: - return FormatTools.INT32; - case f4: - return FormatTools.FLOAT; - case f8: - return FormatTools.DOUBLE; - default: - throw new IllegalArgumentException( - "Data type " + dataType + " not supported"); - } - } - /** * Calculates the pixel length of a given NumPy like "shape". * @@ -223,43 +193,43 @@ private void read(byte[] buffer, int[] shape, int[] offset) } try { ByteBuffer asByteBuffer = ByteBuffer.wrap(buffer); - DataType dataType = array.getDataType(); + int dataType = array.getPixelsType(); for (int z = 0; z < planes; z++) { if (axesOrder.containsKey(Axis.Z)) { offset[axesOrder.get(Axis.Z)] = zIndexMap.get(originalZIndex + z); } switch (dataType) { - case u1: - case i1: + case FormatTools.UINT8: + case FormatTools.INT8: array.read(buffer, shape, offset); break; - case u2: - case i2: + case FormatTools.UINT16: + case FormatTools.INT16: { short[] data = (short[]) array.read(shape, offset); asByteBuffer.asShortBuffer().put(data); break; } - case u4: - case i4: + case FormatTools.UINT32: + case FormatTools.INT32: { int[] data = (int[]) array.read(shape, offset); asByteBuffer.asIntBuffer().put(data); break; } - case i8: - { - long[] data = (long[]) array.read(shape, offset); - asByteBuffer.asLongBuffer().put(data); - break; - } - case f4: + // case FormatTools.INT64: + // { + // long[] data = (long[]) array.read(shape, offset); + // asByteBuffer.asLongBuffer().put(data); + // break; + // } + case FormatTools.FLOAT: { float[] data = (float[]) array.read(shape, offset); asByteBuffer.asFloatBuffer().put(data); break; } - case f8: + case FormatTools.DOUBLE: { double[] data = (double[]) array.read(shape, offset); asByteBuffer.asDoubleBuffer().put(data); @@ -284,7 +254,7 @@ private PixelData toPixelData(byte[] buffer) { return null; } PixelData d = new PixelData( - FormatTools.getPixelTypeString(getPixelsType()), + FormatTools.getPixelTypeString(array.getPixelsType()), ByteBuffer.wrap(buffer)); d.setOrder(ByteOrder.BIG_ENDIAN); return d; @@ -300,8 +270,8 @@ public int[][] getChunks() throws IOException { List> datasets = getDatasets(); List chunks = new ArrayList(); for (Map dataset : datasets) { - ZarrArray resolutionArray = ZarrArray.open( - root.resolve(dataset.get("path"))); + ZarrPath dsPath = root.resolve(dataset.get("path")); + ZArray resolutionArray = new ZArrayv2(ZarrArray.open((Path)dsPath.getPath())); int[] shape = resolutionArray.getChunks(); chunks.add(shape); } @@ -843,17 +813,17 @@ public byte[] calculateMessageDigest() throws IOException { @Override public int getByteWidth() { - return FormatTools.getBytesPerPixel(getPixelsType()); + return FormatTools.getBytesPerPixel(array.getPixelsType()); } @Override public boolean isSigned() { - return FormatTools.isSigned(getPixelsType()); + return FormatTools.isSigned(array.getPixelsType()); } @Override public boolean isFloat() { - return FormatTools.isFloatingPoint(getPixelsType()); + return FormatTools.isFloatingPoint(array.getPixelsType()); } @Override @@ -925,6 +895,10 @@ public int getResolutionLevel() { resolutionLevel - (resolutionLevels - 1)); } + public int getPixelsType() { + return array.getPixelsType(); + } + @Override public void setResolutionLevel(int resolutionLevel) { if (resolutionLevel >= resolutionLevels) { @@ -948,7 +922,7 @@ public void setResolutionLevel(int resolutionLevel) { array = zarrArrayCache.get( root.resolve(Integer.toString(this.resolutionLevel))).get(); - ZarrArray fullResolutionArray = zarrArrayCache.get( + ZArray fullResolutionArray = zarrArrayCache.get( root.resolve("0")).get(); if (axesOrder.containsKey(Axis.Z)) { diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java index d34e0c1..aff6553 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java @@ -78,10 +78,10 @@ public class ZarrPixelsService extends ome.io.nio.PixelsService { /** Root path vs. metadata cache. */ private final - AsyncLoadingCache> zarrMetadataCache; + AsyncLoadingCache> zarrMetadataCache; /** Array path vs. ZarrArray cache */ - private final AsyncLoadingCache zarrArrayCache; + private final AsyncLoadingCache zarrArrayCache; /** Default constructor. */ public ZarrPixelsService( @@ -113,12 +113,16 @@ public ZarrPixelsService( * @param path path to get Zarr metadata from * @return See above. */ - public static Map getZarrMetadata(Path path) + public static Map getZarrMetadata(ZarrPath path) throws IOException { // FIXME: Really should be ZarrUtils.readAttributes() to allow for // attribute retrieval from either a ZarrArray or ZarrGroup but ZarrPath // is package private at the moment. - return ZarrGroup.open(path).getAttributes(); + if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { + return ZarrGroup.open((Path)path.getPath()).getAttributes(); + } else { + throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); + } } /** @@ -127,8 +131,12 @@ public static Map getZarrMetadata(Path path) * @param path path to open a Zarr array from * @return See above. */ - public static ZarrArray getZarrArray(Path path) throws IOException { - return ZarrArray.open(path); + public static ZArray getZarrArray(ZarrPath path) throws IOException { + if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { + return new ZArrayv2(ZarrArray.open((Path)path.getPath())); + } else { + throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); + } } /** @@ -308,8 +316,9 @@ public ZarrPixelBuffer getLabelImagePixelBuffer(Mask mask) throw new IllegalArgumentException( "No root for Mask:" + mask.getId()); } + ZarrPath zarrPath = new ZarrPathv2(asPath(root)); return new ZarrPixelBuffer( - pixels, asPath(root), maxPlaneWidth, maxPlaneHeight, + pixels, zarrPath, maxPlaneWidth, maxPlaneHeight, zarrMetadataCache, zarrArrayCache); } @@ -335,11 +344,11 @@ protected ZarrPixelBuffer createOmeNgffPixelBuffer(Pixels pixels) { log.debug("No OME-NGFF root"); return null; } - Path root = asPath(uri); + ZarrInfo zarrInfo = new ZarrInfo(uri); log.info("OME-NGFF root is: " + uri); try { ZarrPixelBuffer v = new ZarrPixelBuffer( - pixels, root, maxPlaneWidth, maxPlaneHeight, + pixels, zarrInfo.getZarrPath(), maxPlaneWidth, maxPlaneHeight, zarrMetadataCache, zarrArrayCache); log.info("Using OME-NGFF pixel buffer"); return v; @@ -348,7 +357,7 @@ protected ZarrPixelBuffer createOmeNgffPixelBuffer(Pixels pixels) { "Getting OME-NGFF pixel buffer failed - " + "attempting to get local data", e); } - } catch (IOException e1) { + } catch (Exception e1) { log.debug( "Failed to find OME-NGFF metadata for Pixels:{}", pixels.getId()); diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java new file mode 100644 index 0000000..6090176 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java @@ -0,0 +1,18 @@ +package com.glencoesoftware.omero.zarr.model; + +import java.io.IOException; + +import ucar.ma2.InvalidRangeException; + +public interface ZArray { + public int[] getShape(); + + public int[] getChunks(); + + public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException; + + public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeException; + + public int getPixelsType(); + +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java new file mode 100644 index 0000000..b5c5131 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java @@ -0,0 +1,68 @@ +package com.glencoesoftware.omero.zarr.model; + +import java.io.IOException; + +import com.bc.zarr.DataType; +import com.bc.zarr.ZarrArray; + +import loci.formats.FormatTools; +import ucar.ma2.InvalidRangeException; + +public class ZArrayv2 implements ZArray { + + private ZarrArray array; + + public ZArrayv2(ZarrArray array) { + this.array = array; + } + + @Override + public int[] getShape() { + return array.getShape(); + } + + @Override + public int[] getChunks() { + return array.getChunks(); + } + + @Override + public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException { + array.read(buffer, shape, offset); + } + + @Override + public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeException { + return array.read(shape, offset); + } + + @Override + /** + * Get Bio-Formats/OMERO pixels type for buffer. + * @return See above. + */ + public int getPixelsType() { + DataType dataType = array.getDataType(); + switch (dataType) { + case u1: + return FormatTools.UINT8; + case i1: + return FormatTools.INT8; + case u2: + return FormatTools.UINT16; + case i2: + return FormatTools.INT16; + case u4: + return FormatTools.UINT32; + case i4: + return FormatTools.INT32; + case f4: + return FormatTools.FLOAT; + case f8: + return FormatTools.DOUBLE; + default: + throw new IllegalArgumentException( + "Data type " + dataType + " not supported"); + } + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java new file mode 100644 index 0000000..f35d42d --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java @@ -0,0 +1,82 @@ +package com.glencoesoftware.omero.zarr.model; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import loci.formats.FormatTools; +import ucar.ma2.InvalidRangeException; + +public class ZArrayv3 implements ZArray { + + private Array array; + + public ZArrayv3(Array array) { + this.array = array; + } + + @Override + public int[] getShape() { + int[] shape = new int[this.array.metadata.shape.length]; + for (int i = 0; i < this.array.metadata.shape.length; i++) { + shape[i] = (int) this.array.metadata.shape[i]; + } + return shape; + } + + @Override + public int[] getChunks() { + int[] chunks = new int[this.array.metadata.chunkShape().length]; + for (int i = 0; i < this.array.metadata.chunkShape().length; i++) { + chunks[i] = (int) this.array.metadata.chunkShape()[i]; + } + return chunks; + } + + @Override + public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException { + try { + ByteBuffer b = array.read(null, shape).getDataAsByteBuffer(); + System.arraycopy(b.array(), 0, buffer, 0, buffer.length); + } catch (ZarrException e) { + throw new IOException(e); + } + } + + @Override + public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeException { + try { + return array.read(null, shape).copyTo1DJavaArray(); + } catch (ZarrException e) { + throw new IOException(e); + } + } + + @Override + public int getPixelsType() { + DataType dataType = array.metadata.dataType; + switch (dataType) { + case UINT8: + return FormatTools.UINT8; + case INT8: + return FormatTools.INT8; + case UINT16: + return FormatTools.UINT16; + case INT16: + return FormatTools.INT16; + case UINT32: + return FormatTools.UINT32; + case INT32: + return FormatTools.INT32; + case FLOAT32: + return FormatTools.FLOAT; + case FLOAT64: + return FormatTools.DOUBLE; + default: + throw new IllegalArgumentException( + "Data type " + dataType + " not supported"); + } + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java new file mode 100644 index 0000000..03cae46 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java @@ -0,0 +1,239 @@ +package com.glencoesoftware.omero.zarr.model; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.slf4j.LoggerFactory; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.bc.zarr.ZarrGroup; +import com.google.common.base.Splitter; +import com.upplication.s3fs.OmeroS3FilesystemProvider; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.GroupMetadata; + + +/** + * To access the zarr data use: + * For zarr version 2, resp NGFF 0.4: asPath() + * For zarr version 3, resp NGFF > 0.5: asStoreHandle() + */ +public class ZarrInfo { + private static final org.slf4j.Logger log = + LoggerFactory.getLogger(ZarrInfo.class); + + public static final ComparableVersion ZARR_V2 = new ComparableVersion("2"); + public static final ComparableVersion ZARR_V3 = new ComparableVersion("3"); + public static final ComparableVersion NGFF_V0_4 = new ComparableVersion("0.4"); + + private ComparableVersion zarrVersion; + + private ComparableVersion ngffVersion; + + private String location; + + private boolean remote; + + public ZarrInfo(String location) { + this.location = location.endsWith("/") ? location.substring(0, location.length() - 1) : location; + checkProperties(); + log.info("Initialized ZarrPath: " + location); + log.info("Remote store: " + remote); + log.info("Zarr version: " + zarrVersion); + log.info("NGFF version: " + ngffVersion); + } + + /** + * Tries to check some properties of the zarr path, + * if it's remote or local, and the zarr and ngff versions. + */ + private void checkProperties() { + this.remote = location.toLowerCase().startsWith("http://") || + location.toLowerCase().startsWith("https://") || + location.toLowerCase().startsWith("s3://"); + + // checking for zarr v2 + try { + Map attr = ZarrGroup.open(asPath(location)).getAttributes(); + zarrVersion = new ComparableVersion("2"); // if that works it must be v2 + List tmp = (List) attr.get("multiscales"); + ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); + return; + } catch (Exception e) { + // fall through + } + + // checking for zarr v3 + try { + StoreHandle sh = asStoreHandle(); + GroupMetadata md = Group.open(sh).metadata; + if (md.attributes.containsKey("zarr_format")) { + zarrVersion = new ComparableVersion(md.attributes.get("zarr_format").toString()); + } else { + zarrVersion = new ComparableVersion("3"); + } + ngffVersion = new ComparableVersion(((Map) md.attributes.get("ome")).get("version").toString()); + } catch (Exception e) { + // fall through + } + + // set reasonable defaults + if (zarrVersion == null) { + if (ngffVersion != null && ngffVersion.compareTo(NGFF_V0_4) > 0) { + zarrVersion = new ComparableVersion("3"); + } else { + zarrVersion = new ComparableVersion("2"); + } + log.warn("No zarr version found, default to " + zarrVersion); + } + if (ngffVersion == null) { + if (zarrVersion != null && zarrVersion.compareTo(ZARR_V2) > 0) { + ngffVersion = new ComparableVersion("0.5"); + } else { + ngffVersion = new ComparableVersion("0.4"); + } + log.warn("No NGFF version found, default to " + ngffVersion); + } + } + + /** + * Gets the Zarr version. + * @return the Zarr version as a string + */ + public ComparableVersion getZarrVersion() { + return zarrVersion; + } + + /** + * Gets the NGFF version. + * @return the NGFF version as a string + */ + public ComparableVersion getNgffVersion() { + return ngffVersion; + } + + /** + * Gets the path. + * @return the path + */ + public String getLocation() { + return location; + } + + public ZarrPath getZarrPath() throws IOException { + if (zarrVersion.equals(ZARR_V2)) { + return new ZarrPathv2(asPath()); + } else { + return new ZarrPathv3(asStoreHandle()); + } + } + + /** + * Converts an NGFF root string to a path, initializing a {@link FileSystem} + * if required + * For zarr version 2, resp NGFF 0.4 + * @return Fully initialized path or null if the NGFF root + * directory has not been specified in configuration. + * @throws IOException + */ + public Path asPath() throws IOException { + if (location.isEmpty()) { + return null; + } + return asPath(location); + } + + public static Path asPath(String location) throws IOException { + try { + URI uri = new URI(location); + if ("s3".equals(uri.getScheme())) { + if (uri.getUserInfo() != null && !uri.getUserInfo().isEmpty()) { + throw new RuntimeException( + "Found unsupported user information in S3 URI." + + " If you are trying to pass S3 credentials, " + + "use either named profiles or instance credentials."); + } + String query = Optional.ofNullable(uri.getQuery()).orElse(""); + Map params = Splitter.on('&') + .trimResults() + .omitEmptyStrings() + .withKeyValueSeparator('=') + .split(query); + // drop initial "/" + String uriPath = uri.getPath().substring(1); + int first = uriPath.indexOf("/"); + String bucket = "/" + uriPath.substring(0, first); + String rest = uriPath.substring(first + 1); + // FIXME: We might want to support additional S3FS settings in + // the future. See: + // * https://github.com/lasersonlab/Amazon-S3-FileSystem-NIO2 + Map env = new HashMap(); + String profile = params.get("profile"); + if (profile != null) { + env.put("s3fs_credential_profile_name", profile); + } + String anonymous = + Optional.ofNullable(params.get("anonymous")) + .orElse("false"); + env.put("s3fs_anonymous", anonymous); + OmeroS3FilesystemProvider fsp = new OmeroS3FilesystemProvider(); + FileSystem fs = fsp.newFileSystem(uri, env); + return fs.getPath(bucket, rest); + } + } catch (URISyntaxException e) { + // Fall through + } + return Paths.get(location); + } + + /** + * Checks if this is a remote store. + * @return true if remote, false if local + */ + public boolean isRemote() { + return remote; + } + + public StoreHandle asStoreHandle() { + return asStoreHandle(location); + } + + /** + * Return a store handle. + * For zarr version 3, resp NGFF > 0.5 + * @return + */ + public static StoreHandle asStoreHandle(String location) { + //TODO: Properly parse the URI to get store/endpoint, bucket, etc. + + if (location.toLowerCase().startsWith("http://") || location.toLowerCase().startsWith("https://")) { + String store = location.substring(0, location.lastIndexOf("/")); + String zarr = location.substring(location.lastIndexOf("/")+1); + return new HttpStore(store).resolve(zarr); + } + else if (location.toLowerCase().startsWith("s3://")) { + // TODO: This def won't work like that (see comment above) + String store = location.substring(0, location.lastIndexOf("/")); + String zarr = location.substring(location.lastIndexOf("/")+1); + return new S3Store(AmazonS3ClientBuilder.standard().build(), store, zarr).resolve(""); + } + String store = location.substring(0, location.lastIndexOf("/")); + String zarr = location.substring(location.lastIndexOf("/")+1); + return new FilesystemStore(store).resolve(zarr); + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java new file mode 100644 index 0000000..540df88 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java @@ -0,0 +1,13 @@ +package com.glencoesoftware.omero.zarr.model; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +public interface ZarrPath { + + public Object getPath(); + + public ZarrPath resolve(String path); + + public ComparableVersion getVersion(); + +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java new file mode 100644 index 0000000..e6e87a3 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java @@ -0,0 +1,28 @@ +package com.glencoesoftware.omero.zarr.model; + +import java.nio.file.Path; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +public class ZarrPathv2 implements ZarrPath { + + private Path path; + + public ZarrPathv2(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + + @Override + public ZarrPath resolve(String path) { + return new ZarrPathv2(this.path.resolve(path)); + } + + @Override + public ComparableVersion getVersion() { + return ZarrInfo.ZARR_V2; + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java new file mode 100644 index 0000000..bc1df11 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java @@ -0,0 +1,28 @@ +package com.glencoesoftware.omero.zarr.model; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +import dev.zarr.zarrjava.store.StoreHandle; + +public class ZarrPathv3 implements ZarrPath { + + private StoreHandle path; + + public ZarrPathv3(StoreHandle path) { + this.path = path; + } + + public StoreHandle getPath() { + return path; + } + + @Override + public ZarrPath resolve(String path) { + return new ZarrPathv3(this.path.resolve(path)); + } + + @Override + public ComparableVersion getVersion() { + return ZarrInfo.ZARR_V3; + } +} diff --git a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java new file mode 100644 index 0000000..bddee6e --- /dev/null +++ b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java @@ -0,0 +1,55 @@ +package com.glencoesoftware.omero.zarr; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.maven.artifact.versioning.ComparableVersion; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.glencoesoftware.omero.zarr.model.ZarrInfo; + +public class TestZarrInfo { + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void testZarrInfo() throws IOException { + String v0_4_local = writeTestZarr(1,1,1,256,256,"uint8",1).toString()+"/0"; + String v0_5_http = "https://uk1s3.embassy.ebi.ac.uk/idr/share/ome2024-ngff-challenge/0.0.5/6001240_labels.zarr"; + + ZarrInfo zp = new ZarrInfo(v0_4_local); + Assert.assertFalse(zp.isRemote()); + Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); + + zp = new ZarrInfo(v0_5_http); + Assert.assertTrue(zp.isRemote()); + Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); + } + + public Path writeTestZarr( + int sizeT, + int sizeC, + int sizeZ, + int sizeY, + int sizeX, + String pixelType, + int resolutions) throws IOException { + Path input = ZarrPixelBufferTest.fake( + "sizeT", Integer.toString(sizeT), + "sizeC", Integer.toString(sizeC), + "sizeZ", Integer.toString(sizeZ), + "sizeY", Integer.toString(sizeY), + "sizeX", Integer.toString(sizeX), + "pixelType", pixelType, + "resolutions", Integer.toString(resolutions)); + Path output = tmpDir.getRoot().toPath().resolve("output.zarr"); + ZarrPixelBufferTest.assertBioFormats2Raw(input, output); + return output; + } +} diff --git a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java index 97e408e..28f31da 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java @@ -66,7 +66,7 @@ public ZarrPixelBuffer createPixelBuffer( Pixels pixels, Path path, Integer maxPlaneWidth, Integer maxPlaneHeight) throws IOException { return new ZarrPixelBuffer( - pixels, path, maxPlaneWidth, maxPlaneHeight, + pixels, new ZarrPathv2(path), maxPlaneWidth, maxPlaneHeight, Caffeine.newBuilder() .maximumSize(0) .buildAsync(ZarrPixelsService::getZarrMetadata), From 95b27c6345eec390e877a5fe1911cd8d787e6fe0 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Wed, 30 Jul 2025 13:04:34 +0100 Subject: [PATCH 02/17] Add toString to ZarrInfo --- .../com/glencoesoftware/omero/zarr/model/ZarrInfo.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java index 03cae46..a9b46c9 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java @@ -236,4 +236,14 @@ else if (location.toLowerCase().startsWith("s3://")) { String zarr = location.substring(location.lastIndexOf("/")+1); return new FilesystemStore(store).resolve(zarr); } + + @Override + public String toString() { + return "ZarrInfo{" + + "location='" + location + '\'' + + ", remote=" + remote + + ", zarrVersion=" + zarrVersion + + ", ngffVersion=" + ngffVersion + + '}'; + } } From bf12c7adc4fc14cd06cab9b97b05cd101a9c4e56 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Wed, 30 Jul 2025 13:05:03 +0100 Subject: [PATCH 03/17] Implement ZArrayv3 --- .../java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java index f35d42d..758f598 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java @@ -48,7 +48,11 @@ public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, I @Override public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeException { try { - return array.read(null, shape).copyTo1DJavaArray(); + long[] offsetLong = new long[offset.length]; + for (int i = 0; i < offset.length; i++) { + offsetLong[i] = offset[i]; + } + return array.read(offsetLong, shape).copyTo1DJavaArray(); } catch (ZarrException e) { throw new IOException(e); } From 72f2d335d9c781716751c4a78fd6a9a09ef81666 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Wed, 30 Jul 2025 13:05:45 +0100 Subject: [PATCH 04/17] Handle ZArrayv3 in PixelService --- .../omero/zarr/ZarrPixelsService.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java index aff6553..fda9ef7 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java @@ -120,6 +120,9 @@ public static Map getZarrMetadata(ZarrPath path) // is package private at the moment. if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { return ZarrGroup.open((Path)path.getPath()).getAttributes(); + } else if (path.getVersion().equals(ZarrInfo.ZARR_V3)) { + StoreHandle sh = ((StoreHandle)path.getPath()); + return Group.open(sh).metadata.attributes; } else { throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); } @@ -134,6 +137,15 @@ public static Map getZarrMetadata(ZarrPath path) public static ZArray getZarrArray(ZarrPath path) throws IOException { if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { return new ZArrayv2(ZarrArray.open((Path)path.getPath())); + } else if (path.getVersion().equals(ZarrInfo.ZARR_V3)) { + StoreHandle sh = ((StoreHandle)path.getPath()); + Array array; + try { + array = (Array) Group.open(sh).get(); + } catch (ZarrException e) { + throw new IOException(e); + } + return new ZArrayv3(array); } else { throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); } From eb52d459cab658ae0e0f4722286a7c6c92122b99 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 1 Aug 2025 11:43:10 +0100 Subject: [PATCH 05/17] Some refactoring --- .../omero/zarr/ZarrPixelBuffer.java | 8 +- .../omero/zarr/ZarrPixelsService.java | 29 ++---- .../omero/zarr/{model => compat}/ZArray.java | 2 +- .../zarr/{model => compat}/ZArrayv2.java | 4 +- .../zarr/{model => compat}/ZArrayv3.java | 10 ++- .../zarr/{model => compat}/ZarrInfo.java | 32 +++---- .../omero/zarr/compat/ZarrPath.java | 17 ++++ .../omero/zarr/compat/ZarrPathv2.java | 61 +++++++++++++ .../omero/zarr/compat/ZarrPathv3.java | 89 +++++++++++++++++++ .../omero/zarr/model/ZarrPath.java | 13 --- .../omero/zarr/model/ZarrPathv2.java | 28 ------ .../omero/zarr/model/ZarrPathv3.java | 28 ------ .../omero/zarr/TestZarrInfo.java | 2 +- .../omero/zarr/ZarrPixelBufferTest.java | 3 +- 14 files changed, 199 insertions(+), 127 deletions(-) rename src/main/java/com/glencoesoftware/omero/zarr/{model => compat}/ZArray.java (89%) rename src/main/java/com/glencoesoftware/omero/zarr/{model => compat}/ZArrayv2.java (94%) rename src/main/java/com/glencoesoftware/omero/zarr/{model => compat}/ZArrayv3.java (88%) rename src/main/java/com/glencoesoftware/omero/zarr/{model => compat}/ZarrInfo.java (92%) create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java create mode 100644 src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java delete mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java delete mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java delete mode 100644 src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java index 4ce44e4..cdbb753 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java @@ -27,7 +27,6 @@ import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -271,7 +270,7 @@ public int[][] getChunks() throws IOException { List chunks = new ArrayList(); for (Map dataset : datasets) { ZarrPath dsPath = root.resolve(dataset.get("path")); - ZArray resolutionArray = new ZArrayv2(ZarrArray.open((Path)dsPath.getPath())); + ZArray resolutionArray = dsPath.getArray(); //new ZArrayv2(ZarrArray.open((Path)dsPath.getPath())); int[] shape = resolutionArray.getChunks(); chunks.add(shape); } @@ -919,9 +918,8 @@ public void setResolutionLevel(int resolutionLevel) { zIndexMap.clear(); } try { - array = zarrArrayCache.get( - root.resolve(Integer.toString(this.resolutionLevel))).get(); - + ZarrPath p = root.resolve(Integer.toString(this.resolutionLevel)); + array = zarrArrayCache.get(p).get(); ZArray fullResolutionArray = zarrArrayCache.get( root.resolve("0")).get(); diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java index fda9ef7..3f5cfcc 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java @@ -118,14 +118,7 @@ public static Map getZarrMetadata(ZarrPath path) // FIXME: Really should be ZarrUtils.readAttributes() to allow for // attribute retrieval from either a ZarrArray or ZarrGroup but ZarrPath // is package private at the moment. - if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { - return ZarrGroup.open((Path)path.getPath()).getAttributes(); - } else if (path.getVersion().equals(ZarrInfo.ZARR_V3)) { - StoreHandle sh = ((StoreHandle)path.getPath()); - return Group.open(sh).metadata.attributes; - } else { - throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); - } + return path.getMetadata(); } /** @@ -135,20 +128,7 @@ public static Map getZarrMetadata(ZarrPath path) * @return See above. */ public static ZArray getZarrArray(ZarrPath path) throws IOException { - if (path.getVersion().equals(ZarrInfo.ZARR_V2)) { - return new ZArrayv2(ZarrArray.open((Path)path.getPath())); - } else if (path.getVersion().equals(ZarrInfo.ZARR_V3)) { - StoreHandle sh = ((StoreHandle)path.getPath()); - Array array; - try { - array = (Array) Group.open(sh).get(); - } catch (ZarrException e) { - throw new IOException(e); - } - return new ZArrayv3(array); - } else { - throw new RuntimeException("Unsupported Zarr version: " + path.getVersion()); - } + return path.getArray(); } /** @@ -328,7 +308,8 @@ public ZarrPixelBuffer getLabelImagePixelBuffer(Mask mask) throw new IllegalArgumentException( "No root for Mask:" + mask.getId()); } - ZarrPath zarrPath = new ZarrPathv2(asPath(root)); + ZarrInfo zarrInfo = new ZarrInfo(root); + ZarrPath zarrPath = zarrInfo.getZarrPath(); return new ZarrPixelBuffer( pixels, zarrPath, maxPlaneWidth, maxPlaneHeight, zarrMetadataCache, zarrArrayCache); @@ -357,7 +338,7 @@ protected ZarrPixelBuffer createOmeNgffPixelBuffer(Pixels pixels) { return null; } ZarrInfo zarrInfo = new ZarrInfo(uri); - log.info("OME-NGFF root is: " + uri); + log.info("OME-NGFF root is: " + zarrInfo); try { ZarrPixelBuffer v = new ZarrPixelBuffer( pixels, zarrInfo.getZarrPath(), maxPlaneWidth, maxPlaneHeight, diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java similarity index 89% rename from src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java rename to src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java index 6090176..eb47e4d 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArray.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java @@ -1,4 +1,4 @@ -package com.glencoesoftware.omero.zarr.model; +package com.glencoesoftware.omero.zarr.compat; import java.io.IOException; diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java similarity index 94% rename from src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java rename to src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java index b5c5131..fa6785d 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv2.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java @@ -1,4 +1,4 @@ -package com.glencoesoftware.omero.zarr.model; +package com.glencoesoftware.omero.zarr.compat; import java.io.IOException; @@ -8,7 +8,7 @@ import loci.formats.FormatTools; import ucar.ma2.InvalidRangeException; -public class ZArrayv2 implements ZArray { +class ZArrayv2 implements ZArray { private ZarrArray array; diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java similarity index 88% rename from src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java rename to src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java index 758f598..2ec04b2 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZArrayv3.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java @@ -1,4 +1,4 @@ -package com.glencoesoftware.omero.zarr.model; +package com.glencoesoftware.omero.zarr.compat; import java.io.IOException; import java.nio.ByteBuffer; @@ -9,7 +9,7 @@ import loci.formats.FormatTools; import ucar.ma2.InvalidRangeException; -public class ZArrayv3 implements ZArray { +class ZArrayv3 implements ZArray { private Array array; @@ -38,7 +38,11 @@ public int[] getChunks() { @Override public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException { try { - ByteBuffer b = array.read(null, shape).getDataAsByteBuffer(); + long[] offsetLong = new long[offset.length]; + for (int i = 0; i < offset.length; i++) { + offsetLong[i] = offset[i]; + } + ByteBuffer b = array.read(offsetLong, shape).getDataAsByteBuffer(); System.arraycopy(b.array(), 0, buffer, 0, buffer.length); } catch (ZarrException e) { throw new IOException(e); diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java similarity index 92% rename from src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java rename to src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java index a9b46c9..a4cb00e 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrInfo.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java @@ -1,4 +1,4 @@ -package com.glencoesoftware.omero.zarr.model; +package com.glencoesoftware.omero.zarr.compat; import java.io.IOException; import java.net.URI; @@ -29,9 +29,10 @@ /** - * To access the zarr data use: - * For zarr version 2, resp NGFF 0.4: asPath() - * For zarr version 3, resp NGFF > 0.5: asStoreHandle() + * Tries to determine some properties of the zarr path, + * if it's remote or local, and the zarr and ngff versions. + * + * To access the zarr metadata/array use getZarrPath(). */ public class ZarrInfo { private static final org.slf4j.Logger log = @@ -40,6 +41,7 @@ public class ZarrInfo { public static final ComparableVersion ZARR_V2 = new ComparableVersion("2"); public static final ComparableVersion ZARR_V3 = new ComparableVersion("3"); public static final ComparableVersion NGFF_V0_4 = new ComparableVersion("0.4"); + public static final ComparableVersion NGFF_V0_5 = new ComparableVersion("0.5"); private ComparableVersion zarrVersion; @@ -52,10 +54,6 @@ public class ZarrInfo { public ZarrInfo(String location) { this.location = location.endsWith("/") ? location.substring(0, location.length() - 1) : location; checkProperties(); - log.info("Initialized ZarrPath: " + location); - log.info("Remote store: " + remote); - log.info("Zarr version: " + zarrVersion); - log.info("NGFF version: " + ngffVersion); } /** @@ -143,6 +141,9 @@ public ZarrPath getZarrPath() throws IOException { } } + private Path asPath() throws IOException { + return asPath(location); + } /** * Converts an NGFF root string to a path, initializing a {@link FileSystem} * if required @@ -151,14 +152,7 @@ public ZarrPath getZarrPath() throws IOException { * directory has not been specified in configuration. * @throws IOException */ - public Path asPath() throws IOException { - if (location.isEmpty()) { - return null; - } - return asPath(location); - } - - public static Path asPath(String location) throws IOException { + private Path asPath(String location) throws IOException { try { URI uri = new URI(location); if ("s3".equals(uri.getScheme())) { @@ -209,16 +203,12 @@ public boolean isRemote() { return remote; } - public StoreHandle asStoreHandle() { - return asStoreHandle(location); - } - /** * Return a store handle. * For zarr version 3, resp NGFF > 0.5 * @return */ - public static StoreHandle asStoreHandle(String location) { + private StoreHandle asStoreHandle() { //TODO: Properly parse the URI to get store/endpoint, bucket, etc. if (location.toLowerCase().startsWith("http://") || location.toLowerCase().startsWith("https://")) { diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java new file mode 100644 index 0000000..ff6b03c --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java @@ -0,0 +1,17 @@ +package com.glencoesoftware.omero.zarr.compat; + +import java.io.IOException; +import java.util.Map; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +public interface ZarrPath { + + public ZarrPath resolve(String path); + + public ComparableVersion getVersion(); + + public Map getMetadata() throws IOException; + + public ZArray getArray() throws IOException; +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java new file mode 100644 index 0000000..b0b3039 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java @@ -0,0 +1,61 @@ +package com.glencoesoftware.omero.zarr.compat; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +import com.bc.zarr.ZarrArray; +import com.bc.zarr.ZarrGroup; + +class ZarrPathv2 implements ZarrPath { + + private Path path; + + public ZarrPathv2(Path path) { + this.path = path; + } + + @Override + public ZarrPath resolve(String path) { + return new ZarrPathv2(this.path.resolve(path)); + } + + @Override + public ComparableVersion getVersion() { + return ZarrInfo.ZARR_V2; + } + + @Override + public Map getMetadata() throws IOException { + return ZarrGroup.open(path).getAttributes(); + } + + @Override + public ZArray getArray() throws IOException { + return new ZArrayv2(ZarrArray.open(path)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ZarrPathv2 that = (ZarrPathv2) obj; + return path.toString().equals(that.path.toString()); + } + + @Override + public int hashCode() { + return path.toString().hashCode(); + } + + @Override + public String toString() { + return path.toString(); + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java new file mode 100644 index 0000000..f7ceca9 --- /dev/null +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java @@ -0,0 +1,89 @@ +package com.glencoesoftware.omero.zarr.compat; + +import java.io.IOException; +import java.util.Map; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.Group; + +class ZarrPathv3 implements ZarrPath { + + private StoreHandle path; + private Group group; + private String key = null; + + public ZarrPathv3(StoreHandle path) { + try { + this.path = path; + this.group = Group.open(path); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private ZarrPathv3(ZarrPathv3 parent, String resolvePath) { + // this can either resolve to another group or an array + this.path = parent.path.resolve(resolvePath); + try { + // it's a group + this.group = Group.open(this.path); + } catch (IOException e) { + // is not a group; points to array from parent group + // with resolvePath as key. + this.group = parent.group; + this.key = resolvePath; + } + } + + @Override + public ZarrPath resolve(String resolvePath) { + return new ZarrPathv3(this, resolvePath); + } + + @Override + public ComparableVersion getVersion() { + return ZarrInfo.ZARR_V3; + } + + @Override + public Map getMetadata() throws IOException { + return group.metadata.attributes; + } + + @Override + public ZArray getArray() throws IOException { + Array array; + try { + array = (Array) group.get(key); + } catch (ZarrException e) { + throw new IOException(e); + } + return new ZArrayv3(array); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ZarrPathv3 that = (ZarrPathv3) obj; + return path.toString().equals(that.path.toString()); + } + + @Override + public int hashCode() { + return path.toString().hashCode(); + } + + @Override + public String toString() { + return key == null ? "(Group) "+ path.toString() : "(Array) " + path.toString(); + } +} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java deleted file mode 100644 index 540df88..0000000 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPath.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.glencoesoftware.omero.zarr.model; - -import org.apache.maven.artifact.versioning.ComparableVersion; - -public interface ZarrPath { - - public Object getPath(); - - public ZarrPath resolve(String path); - - public ComparableVersion getVersion(); - -} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java deleted file mode 100644 index e6e87a3..0000000 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv2.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.glencoesoftware.omero.zarr.model; - -import java.nio.file.Path; - -import org.apache.maven.artifact.versioning.ComparableVersion; - -public class ZarrPathv2 implements ZarrPath { - - private Path path; - - public ZarrPathv2(Path path) { - this.path = path; - } - - public Path getPath() { - return path; - } - - @Override - public ZarrPath resolve(String path) { - return new ZarrPathv2(this.path.resolve(path)); - } - - @Override - public ComparableVersion getVersion() { - return ZarrInfo.ZARR_V2; - } -} diff --git a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java b/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java deleted file mode 100644 index bc1df11..0000000 --- a/src/main/java/com/glencoesoftware/omero/zarr/model/ZarrPathv3.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.glencoesoftware.omero.zarr.model; - -import org.apache.maven.artifact.versioning.ComparableVersion; - -import dev.zarr.zarrjava.store.StoreHandle; - -public class ZarrPathv3 implements ZarrPath { - - private StoreHandle path; - - public ZarrPathv3(StoreHandle path) { - this.path = path; - } - - public StoreHandle getPath() { - return path; - } - - @Override - public ZarrPath resolve(String path) { - return new ZarrPathv3(this.path.resolve(path)); - } - - @Override - public ComparableVersion getVersion() { - return ZarrInfo.ZARR_V3; - } -} diff --git a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java index bddee6e..67abb85 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java @@ -9,7 +9,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.glencoesoftware.omero.zarr.model.ZarrInfo; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo; public class TestZarrInfo { diff --git a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java index 28f31da..a79198d 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java @@ -65,8 +65,9 @@ public class ZarrPixelBufferTest { public ZarrPixelBuffer createPixelBuffer( Pixels pixels, Path path, Integer maxPlaneWidth, Integer maxPlaneHeight) throws IOException { + ZarrInfo zarrInfo = new ZarrInfo(path.toString()); return new ZarrPixelBuffer( - pixels, new ZarrPathv2(path), maxPlaneWidth, maxPlaneHeight, + pixels, zarrInfo.getZarrPath(), maxPlaneWidth, maxPlaneHeight, Caffeine.newBuilder() .maximumSize(0) .buildAsync(ZarrPixelsService::getZarrMetadata), From 7c6b1bb38fc6a8e28178e0a4b7baeb263acf5d73 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Thu, 18 Sep 2025 15:39:23 +0100 Subject: [PATCH 06/17] method moved to ZarrInfo --- .../omero/zarr/ZarrPixelsService.java | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java index 3f5cfcc..290bdba 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java @@ -131,61 +131,6 @@ public static ZArray getZarrArray(ZarrPath path) throws IOException { return path.getArray(); } - /** - * Converts an NGFF root string to a path, initializing a {@link FileSystem} - * if required. - * - * @param ngffDir NGFF directory root - * @return Fully initialized path or null if the NGFF root - * directory has not been specified in configuration. - */ - public static Path asPath(String ngffDir) throws IOException { - if (ngffDir.isEmpty()) { - return null; - } - - try { - URI uri = new URI(ngffDir); - if ("s3".equals(uri.getScheme())) { - if (uri.getUserInfo() != null && !uri.getUserInfo().isEmpty()) { - throw new RuntimeException( - "Found unsupported user information in S3 URI." - + " If you are trying to pass S3 credentials, " - + "use either named profiles or instance credentials."); - } - String query = Optional.ofNullable(uri.getQuery()).orElse(""); - Map params = Splitter.on('&') - .trimResults() - .omitEmptyStrings() - .withKeyValueSeparator('=') - .split(query); - // drop initial "/" - String uriPath = uri.getPath().substring(1); - int first = uriPath.indexOf("/"); - String bucket = "/" + uriPath.substring(0, first); - String rest = uriPath.substring(first + 1); - // FIXME: We might want to support additional S3FS settings in - // the future. See: - // * https://github.com/lasersonlab/Amazon-S3-FileSystem-NIO2 - Map env = new HashMap(); - String profile = params.get("profile"); - if (profile != null) { - env.put("s3fs_credential_profile_name", profile); - } - String anonymous = - Optional.ofNullable(params.get("anonymous")) - .orElse("false"); - env.put("s3fs_anonymous", anonymous); - OmeroS3FilesystemProvider fsp = new OmeroS3FilesystemProvider(); - FileSystem fs = fsp.newFileSystem(uri, env); - return fs.getPath(bucket, rest); - } - } catch (URISyntaxException e) { - // Fall through - } - return Paths.get(ngffDir); - } - /** * Retrieve {@link Mask} or {@link Image} URI stored under {@link ExternalInfo}. * From d7e43cc307bcb77c8a524526d77b8096bcb03a1f Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Thu, 18 Sep 2025 15:42:03 +0100 Subject: [PATCH 07/17] fixed some issues on ZarrInfo --- .../omero/zarr/compat/ZarrInfo.java | 144 +++++++++++------- 1 file changed, 85 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java index a4cb00e..d562340 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java @@ -1,11 +1,11 @@ package com.glencoesoftware.omero.zarr.compat; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -15,7 +15,6 @@ import org.apache.maven.artifact.versioning.ComparableVersion; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.bc.zarr.ZarrGroup; import com.google.common.base.Splitter; import com.upplication.s3fs.OmeroS3FilesystemProvider; @@ -49,9 +48,18 @@ public class ZarrInfo { private String location; - private boolean remote; + /** + * Enum representing different storage types for Zarr data. + */ + public enum StorageType { + FILE, + S3, + HTTP + } - public ZarrInfo(String location) { + private StorageType storageType; + + public ZarrInfo(String location) throws IOException { this.location = location.endsWith("/") ? location.substring(0, location.length() - 1) : location; checkProperties(); } @@ -59,18 +67,47 @@ public ZarrInfo(String location) { /** * Tries to check some properties of the zarr path, * if it's remote or local, and the zarr and ngff versions. + * @throws IOException */ - private void checkProperties() { - this.remote = location.toLowerCase().startsWith("http://") || - location.toLowerCase().startsWith("https://") || - location.toLowerCase().startsWith("s3://"); + private void checkProperties() throws IOException { + URI uri; + try { + uri = new URI(location); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI: " + location, e); + } + + if (uri.getScheme() == null || "file".equals(uri.getScheme().toLowerCase())) { + File test = new File(location); + if (!test.isDirectory()) { + throw new IOException("Not a directory: " + location); + } + if (!test.canRead()) { + throw new IOException("Cannot read directory: " + location); + } + storageType = StorageType.FILE; + } else { + String scheme = uri.getScheme().toLowerCase(); + if (scheme.equals("http") || scheme.equals("https")) { + storageType = StorageType.HTTP; + } else if (scheme.equals("s3")) { + storageType = StorageType.S3; + } else { + throw new IOException("Unsupported scheme: " + scheme); + } + } // checking for zarr v2 try { Map attr = ZarrGroup.open(asPath(location)).getAttributes(); zarrVersion = new ComparableVersion("2"); // if that works it must be v2 - List tmp = (List) attr.get("multiscales"); - ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); + try { + List tmp = (List) attr.get("multiscales"); + ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); + } catch (Exception e) { + log.debug("Failed to get ngff version from zarr, set to 0.4"); + ngffVersion = new ComparableVersion("0.4"); // if it's zarr v2 then we can actually safely assume it's ngff v0.4 + } return; } catch (Exception e) { // fall through @@ -80,32 +117,19 @@ private void checkProperties() { try { StoreHandle sh = asStoreHandle(); GroupMetadata md = Group.open(sh).metadata; - if (md.attributes.containsKey("zarr_format")) { - zarrVersion = new ComparableVersion(md.attributes.get("zarr_format").toString()); - } else { - zarrVersion = new ComparableVersion("3"); + zarrVersion = new ComparableVersion("3"); // if that works it should be v3 + try { + ngffVersion = new ComparableVersion(((Map) md.attributes.get("ome")).get("version").toString()); + } catch (Exception e) { + log.debug("Failed to get ngff version from zarr, set to 0.5"); + ngffVersion = new ComparableVersion("0.5"); } - ngffVersion = new ComparableVersion(((Map) md.attributes.get("ome")).get("version").toString()); } catch (Exception e) { // fall through } - // set reasonable defaults if (zarrVersion == null) { - if (ngffVersion != null && ngffVersion.compareTo(NGFF_V0_4) > 0) { - zarrVersion = new ComparableVersion("3"); - } else { - zarrVersion = new ComparableVersion("2"); - } - log.warn("No zarr version found, default to " + zarrVersion); - } - if (ngffVersion == null) { - if (zarrVersion != null && zarrVersion.compareTo(ZARR_V2) > 0) { - ngffVersion = new ComparableVersion("0.5"); - } else { - ngffVersion = new ComparableVersion("0.4"); - } - log.warn("No NGFF version found, default to " + ngffVersion); + throw new IOException("Failed to determine zarr version"); } } @@ -135,15 +159,12 @@ public String getLocation() { public ZarrPath getZarrPath() throws IOException { if (zarrVersion.equals(ZARR_V2)) { - return new ZarrPathv2(asPath()); + return new ZarrPathv2(asPath(location)); } else { return new ZarrPathv3(asStoreHandle()); } } - private Path asPath() throws IOException { - return asPath(location); - } /** * Converts an NGFF root string to a path, initializing a {@link FileSystem} * if required @@ -155,7 +176,12 @@ private Path asPath() throws IOException { private Path asPath(String location) throws IOException { try { URI uri = new URI(location); - if ("s3".equals(uri.getScheme())) { + String scheme = uri.getScheme() != null ? uri.getScheme().toLowerCase() : "file"; + if (scheme.startsWith("http")) { + String s3loc = location.replaceFirst("https?", "s3"); + return asPath(s3loc+"?anonymous=true"); + } + if (scheme.equals("s3")) { if (uri.getUserInfo() != null && !uri.getUserInfo().isEmpty()) { throw new RuntimeException( "Found unsupported user information in S3 URI." @@ -190,17 +216,9 @@ private Path asPath(String location) throws IOException { return fs.getPath(bucket, rest); } } catch (URISyntaxException e) { - // Fall through + // we made sure earlier that location is a valid URI } - return Paths.get(location); - } - - /** - * Checks if this is a remote store. - * @return true if remote, false if local - */ - public boolean isRemote() { - return remote; + return Path.of(location); } /** @@ -209,31 +227,39 @@ public boolean isRemote() { * @return */ private StoreHandle asStoreHandle() { - //TODO: Properly parse the URI to get store/endpoint, bucket, etc. - - if (location.toLowerCase().startsWith("http://") || location.toLowerCase().startsWith("https://")) { - String store = location.substring(0, location.lastIndexOf("/")); - String zarr = location.substring(location.lastIndexOf("/")+1); - return new HttpStore(store).resolve(zarr); - } - else if (location.toLowerCase().startsWith("s3://")) { - // TODO: This def won't work like that (see comment above) + try { + URI uri = new URI(location); + String scheme = uri.getScheme() != null ? uri.getScheme().toLowerCase() : "file"; String store = location.substring(0, location.lastIndexOf("/")); String zarr = location.substring(location.lastIndexOf("/")+1); - return new S3Store(AmazonS3ClientBuilder.standard().build(), store, zarr).resolve(""); + + if (scheme.startsWith("http")) { + return new HttpStore(store).resolve(zarr); + } + else if (scheme.startsWith("s3")) { + // TODO: implement S3Store + return new S3Store(null, null, null).resolve(zarr); + } + else { + return new FilesystemStore(store).resolve(zarr); + } + } catch (URISyntaxException e) { + // we checked earlier that location is a valid URI } - String store = location.substring(0, location.lastIndexOf("/")); - String zarr = location.substring(location.lastIndexOf("/")+1); - return new FilesystemStore(store).resolve(zarr); + return null; } @Override public String toString() { return "ZarrInfo{" + "location='" + location + '\'' + - ", remote=" + remote + + ", storageType=" + storageType + ", zarrVersion=" + zarrVersion + ", ngffVersion=" + ngffVersion + '}'; } + + public StorageType getStorageType() { + return storageType; + } } From 9696d3493cf2545d20ff9df288166c4b8009dd08 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Thu, 18 Sep 2025 15:42:17 +0100 Subject: [PATCH 08/17] added more tests for ZarrInfo --- .../omero/zarr/TestZarrInfo.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java index 67abb85..38f78e2 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java @@ -10,6 +10,7 @@ import org.junit.rules.TemporaryFolder; import com.glencoesoftware.omero.zarr.compat.ZarrInfo; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo.StorageType; public class TestZarrInfo { @@ -17,17 +18,35 @@ public class TestZarrInfo { public TemporaryFolder tmpDir = new TemporaryFolder(); @Test - public void testZarrInfo() throws IOException { + public void testLocalV2N04() throws IOException { String v0_4_local = writeTestZarr(1,1,1,256,256,"uint8",1).toString()+"/0"; - String v0_5_http = "https://uk1s3.embassy.ebi.ac.uk/idr/share/ome2024-ngff-challenge/0.0.5/6001240_labels.zarr"; ZarrInfo zp = new ZarrInfo(v0_4_local); - Assert.assertFalse(zp.isRemote()); + Assert.assertEquals(zp.getStorageType(), StorageType.FILE); Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); + } + + @Test + public void testHTTPV2N04() throws IOException { + ZarrInfo zp = new ZarrInfo("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0101A/13457227.zarr"); + Assert.assertEquals(zp.getStorageType(), StorageType.HTTP); + Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); + } - zp = new ZarrInfo(v0_5_http); - Assert.assertTrue(zp.isRemote()); + @Test + public void testS3V2N04() throws IOException { + ZarrInfo zp = new ZarrInfo("s3://s3.us-east-1.amazonaws.com/gs-public-zarr-archive/CMU-1.ome.zarr/0?anonymous=true"); + Assert.assertEquals(zp.getStorageType(), StorageType.S3); + Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); + } + + @Test + public void testHTTPV3N05() throws IOException { + ZarrInfo zp = new ZarrInfo("https://uk1s3.embassy.ebi.ac.uk/idr/share/ome2024-ngff-challenge/0.0.5/6001240_labels.zarr"); + Assert.assertEquals(zp.getStorageType(), StorageType.HTTP); Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); } From 6c16a97535a2cc34d4f5f3fa51936651a3bed972 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Thu, 18 Sep 2025 15:44:35 +0100 Subject: [PATCH 09/17] use latest zarr-java --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index feff228..1c35bdc 100644 --- a/build.gradle +++ b/build.gradle @@ -43,7 +43,7 @@ dependencies { implementation 'ome:formats-gpl:7.3.1' implementation 'info.picocli:picocli:4.7.5' implementation 'com.univocity:univocity-parsers:2.8.4' - implementation 'dev.zarr:zarr-java:0.0.2' + implementation 'dev.zarr:zarr-java:0.0.4' implementation 'javax.xml.bind:jaxb-api:2.3.0' implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.3.14' implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.3.14' From 6484a98693c6b86fef94ed2ccca1832afb3111cd Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 11:19:58 +0100 Subject: [PATCH 10/17] Fix for ngff challenge data --- .../com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java index cdbb753..3abe2ed 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java @@ -120,8 +120,13 @@ public ZarrPixelBuffer(Pixels pixels, ZarrPath root, Integer maxPlaneWidth, this.zarrArrayCache = zarrArrayCache; this.isRemote = root.toString().startsWith("s3://") ? true : false; try { - rootGroupAttributes = this.zarrMetadataCache.get(this.root).get(); - } catch (ExecutionException | InterruptedException e) { + Map tmp = this.zarrMetadataCache.get(this.root).get(); + if (tmp.containsKey("ome")) { // for ngff challenge data attr are often nested within "ome" key + rootGroupAttributes = (Map) tmp.get("ome"); + } else { + rootGroupAttributes = tmp; + } + } catch (ExecutionException|InterruptedException e) { throw new IOException(e); } if (!rootGroupAttributes.containsKey("multiscales")) { From ce80da19e76a1e08d680372a825bd748b23582d5 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 11:25:16 +0100 Subject: [PATCH 11/17] Handle s3 properly --- .../omero/zarr/compat/ZarrInfo.java | 82 ++++++++++++++----- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java index d562340..b1c2f69 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java @@ -6,15 +6,16 @@ import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.nio.file.Path; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.LoggerFactory; - import org.apache.maven.artifact.versioning.ComparableVersion; +import com.amazonaws.services.s3.AmazonS3; import com.bc.zarr.ZarrGroup; import com.google.common.base.Splitter; import com.upplication.s3fs.OmeroS3FilesystemProvider; @@ -88,7 +89,7 @@ private void checkProperties() throws IOException { storageType = StorageType.FILE; } else { String scheme = uri.getScheme().toLowerCase(); - if (scheme.equals("http") || scheme.equals("https")) { + if (scheme.startsWith("http")) { storageType = StorageType.HTTP; } else if (scheme.equals("s3")) { storageType = StorageType.S3; @@ -97,22 +98,6 @@ private void checkProperties() throws IOException { } } - // checking for zarr v2 - try { - Map attr = ZarrGroup.open(asPath(location)).getAttributes(); - zarrVersion = new ComparableVersion("2"); // if that works it must be v2 - try { - List tmp = (List) attr.get("multiscales"); - ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); - } catch (Exception e) { - log.debug("Failed to get ngff version from zarr, set to 0.4"); - ngffVersion = new ComparableVersion("0.4"); // if it's zarr v2 then we can actually safely assume it's ngff v0.4 - } - return; - } catch (Exception e) { - // fall through - } - // checking for zarr v3 try { StoreHandle sh = asStoreHandle(); @@ -124,11 +109,26 @@ private void checkProperties() throws IOException { log.debug("Failed to get ngff version from zarr, set to 0.5"); ngffVersion = new ComparableVersion("0.5"); } + return; } catch (Exception e) { + log.debug("Not zarr v3:", e); // fall through } - if (zarrVersion == null) { + // checking for zarr v2 + try { + Map attr = ZarrGroup.open(asPath(location)).getAttributes(); + zarrVersion = new ComparableVersion("2"); // if that works it must be v2 + try { + List tmp = (List) attr.get("multiscales"); + ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); + } catch (Exception e) { + log.debug("Failed to get ngff version from zarr, set to 0.4"); + ngffVersion = new ComparableVersion("0.4"); // if it's zarr v2 then we can actually safely assume it's ngff v0.4 + } + return; + } catch (Exception e) { + log.debug("Not zarr v2:", e); throw new IOException("Failed to determine zarr version"); } } @@ -232,13 +232,53 @@ private StoreHandle asStoreHandle() { String scheme = uri.getScheme() != null ? uri.getScheme().toLowerCase() : "file"; String store = location.substring(0, location.lastIndexOf("/")); String zarr = location.substring(location.lastIndexOf("/")+1); + zarr = zarr.replaceFirst("\\?.+", ""); if (scheme.startsWith("http")) { return new HttpStore(store).resolve(zarr); } else if (scheme.startsWith("s3")) { - // TODO: implement S3Store - return new S3Store(null, null, null).resolve(zarr); + String[] tmp = store.replaceFirst("s3://", "").split("/"); + String host = tmp[0]; + String bucket = tmp[1]; + String rest = String.join("/", Arrays.copyOfRange(tmp, 2, tmp.length)); + System.out.println("Host: " + host); + + // Use the OmeroS3FilesystemProvider to create AmazonS3 client + String query = Optional.ofNullable(uri.getQuery()).orElse(""); + Map params = Splitter.on('&') + .trimResults() + .omitEmptyStrings() + .withKeyValueSeparator('=') + .split(query); + Map env = new HashMap(); + String profile = params.get("profile"); + if (profile != null) { + env.put("s3fs_credential_profile_name", profile); + } + String anonymous = + Optional.ofNullable(params.get("anonymous")) + .orElse("false"); + env.put("s3fs_anonymous", anonymous); + OmeroS3FilesystemProvider fsp = new OmeroS3FilesystemProvider(); + AmazonS3 client = fsp.createAmazonS3(uri, env); + + // Create s3 client manually: + // AmazonS3 client = AmazonS3ClientBuilder.standard() + // .withEndpointConfiguration(new EndpointConfiguration(host, null)) + // .withCredentials(new AWSCredentialsProvider() { + // @Override + // public AWSCredentials getCredentials() { + // return new AnonymousAWSCredentials(); + // } + + // @Override + // public void refresh() { + // // do nothing + // } + // }) + // .build(); + return new S3Store(client, bucket, rest).resolve(zarr); } else { return new FilesystemStore(store).resolve(zarr); From 888cd3d5534b04eff790d4acf514c63fca28e2c5 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 11:25:57 +0100 Subject: [PATCH 12/17] Improve toString --- .../omero/zarr/compat/ZarrPathv3.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java index f7ceca9..0a2719e 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java @@ -5,10 +5,10 @@ import org.apache.maven.artifact.versioning.ComparableVersion; -import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.Node; class ZarrPathv3 implements ZarrPath { @@ -58,8 +58,9 @@ public Map getMetadata() throws IOException { public ZArray getArray() throws IOException { Array array; try { - array = (Array) group.get(key); - } catch (ZarrException e) { + Node node = group.get(key); + array = (Array) node; + } catch (Exception e) { throw new IOException(e); } return new ZArrayv3(array); @@ -74,16 +75,16 @@ public boolean equals(Object obj) { return false; } ZarrPathv3 that = (ZarrPathv3) obj; - return path.toString().equals(that.path.toString()); + return toString().equals(that.toString()); } @Override public int hashCode() { - return path.toString().hashCode(); + return toString().hashCode(); } @Override public String toString() { - return key == null ? "(Group) "+ path.toString() : "(Array) " + path.toString(); + return key == null ? "(Group) "+ path.toString() : "(Array) " + path.toString() + ":" + key; } } From 6d10eefc229dac0cedfc2f9ba7bbfe8599d2f969 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 11:26:47 +0100 Subject: [PATCH 13/17] Add public method to create S3 client --- .../com/upplication/s3fs/OmeroS3FilesystemProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java index ce232a3..e8b2e96 100644 --- a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java +++ b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java @@ -106,6 +106,12 @@ public S3FileSystem createFileSystem(URI uri, Properties props) { this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost()); } + public AmazonS3 createAmazonS3(URI uri, Map env) { + Properties props = getProperties(uri, env); + validateProperties(props); + return getAmazonS3(uri, props); + } + /** * Overridden, hybrid version of the implementation from * {@link S3FileSystemProvider#getAmazonS3Factory(Properties)}. Our From b2c51f57bf6c7269a633c3f37d15fdc21d8f2d23 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 11:28:52 +0100 Subject: [PATCH 14/17] Pin jackson libs to 2.14.2 --- build.gradle | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1c35bdc..7ea1b52 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,16 @@ plugins { group = 'com.glencoesoftware.omero' version = '0.6.0-SNAPSHOT' -sourceCompatibility = 1.11 -targetCompatibility = 1.11 +// Java 11 compatibility configuration +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +// Ensure UTF-8 encoding for compilation +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} repositories { mavenCentral() @@ -31,6 +39,26 @@ repositories { } } +configurations.all { + resolutionStrategy { + // Force all Jackson dependencies to version 2.14.2 + eachDependency { DependencyResolveDetails details -> + if (details.requested.group == 'com.fasterxml.jackson.core' || + details.requested.group == 'com.fasterxml.jackson.dataformat' || + details.requested.group == 'com.fasterxml.jackson.datatype' || + details.requested.group == 'com.fasterxml.jackson.module' || + details.requested.group == 'com.fasterxml.jackson.jr') { + details.useVersion '2.14.2' + } + } + + // Explicit force for common Jackson modules + force 'com.fasterxml.jackson.core:jackson-core:2.14.2' + force 'com.fasterxml.jackson.core:jackson-databind:2.14.2' + force 'com.fasterxml.jackson.core:jackson-annotations:2.14.2' + } +} + dependencies { api 'org.openmicroscopy:omero-blitz:5.8.3' @@ -49,6 +77,9 @@ dependencies { implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.3.14' implementation 'org.apache.maven:maven-artifact:3.9.4' + implementation 'com.fasterxml.jackson.core:jackson-core:2.14.2' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2' + testImplementation 'info.picocli:picocli:4.7.5' testImplementation 'com.glencoesoftware:bioformats2raw:0.11.0' } From 2e90f060b4024d491e75cbe2369c5a2930ec497d Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Fri, 26 Sep 2025 14:16:05 +0100 Subject: [PATCH 15/17] Fix imports --- .../omero/zarr/ZarrPixelBuffer.java | 5 +++-- .../omero/zarr/ZarrPixelsService.java | 14 +++----------- .../s3fs/OmeroS3FilesystemProvider.java | 1 + .../omero/zarr/ZarrPixelsServiceTest.java | 2 -- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java index 3abe2ed..c3700ac 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java @@ -18,10 +18,11 @@ package com.glencoesoftware.omero.zarr; -import com.bc.zarr.DataType; -import com.bc.zarr.ZarrArray; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.glencoesoftware.omero.zarr.compat.ZArray; +import com.glencoesoftware.omero.zarr.compat.ZarrPath; + import java.awt.Dimension; import java.io.IOException; import java.nio.BufferOverflowException; diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java index 290bdba..417f999 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelsService.java @@ -18,22 +18,14 @@ package com.glencoesoftware.omero.zarr; -import com.bc.zarr.ZarrArray; -import com.bc.zarr.ZarrGroup; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.common.base.Splitter; -import com.upplication.s3fs.OmeroS3FilesystemProvider; +import com.glencoesoftware.omero.zarr.compat.ZArray; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo; +import com.glencoesoftware.omero.zarr.compat.ZarrPath; import java.io.File; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; import java.util.Map; -import java.util.Optional; import ome.api.IQuery; import ome.conditions.LockTimeout; import ome.io.nio.BackOff; diff --git a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java index e8b2e96..26565ec 100644 --- a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java +++ b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java @@ -21,6 +21,7 @@ import static com.upplication.s3fs.AmazonS3Factory.ACCESS_KEY; import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY; +import com.amazonaws.services.s3.AmazonS3; import com.glencoesoftware.omero.zarr.OmeroAmazonS3ClientFactory; import com.glencoesoftware.omero.zarr.OmeroS3FileSystem; import com.glencoesoftware.omero.zarr.OmeroS3ReadOnlySeekableByteChannel; diff --git a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelsServiceTest.java b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelsServiceTest.java index 6d96c50..bf62c58 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelsServiceTest.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelsServiceTest.java @@ -18,11 +18,9 @@ package com.glencoesoftware.omero.zarr; -import static omero.rtypes.rdouble; import static omero.rtypes.rlong; import static omero.rtypes.rstring; -import com.glencoesoftware.omero.zarr.ZarrPixelsService; import java.io.File; import java.io.IOException; import java.nio.file.Files; From bbbadfd5caf17cf28d17f33a61e26cf8696015b8 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Tue, 30 Sep 2025 15:17:43 +0100 Subject: [PATCH 16/17] Fix formatting issues --- .../omero/zarr/ZarrPixelBuffer.java | 12 +- .../omero/zarr/compat/ZArray.java | 44 +- .../omero/zarr/compat/ZArrayv2.java | 15 +- .../omero/zarr/compat/ZArrayv3.java | 29 +- .../omero/zarr/compat/ZarrInfo.java | 166 ++++---- .../omero/zarr/compat/ZarrPath.java | 26 +- .../omero/zarr/compat/ZarrPathv2.java | 14 +- .../omero/zarr/compat/ZarrPathv3.java | 15 +- .../s3fs/OmeroS3FilesystemProvider.java | 3 + .../omero/zarr/ZarrPixelBufferTest.java | 402 +++++++----------- 10 files changed, 331 insertions(+), 395 deletions(-) diff --git a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java index c3700ac..b05b2e3 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/ZarrPixelBuffer.java @@ -22,7 +22,6 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.glencoesoftware.omero.zarr.compat.ZArray; import com.glencoesoftware.omero.zarr.compat.ZarrPath; - import java.awt.Dimension; import java.io.IOException; import java.nio.BufferOverflowException; @@ -55,7 +54,7 @@ public class ZarrPixelBuffer implements PixelBuffer { /** Reference to the pixels. */ private final Pixels pixels; - /** Root of the OME-NGFF multiscale we are operating on */ + /** Root of the OME-NGFF multiscale we are operating on. */ private final ZarrPath root; /** Requested resolution level. */ @@ -73,7 +72,7 @@ public class ZarrPixelBuffer implements PixelBuffer { /** Zarr attributes present on the root group. */ private final Map rootGroupAttributes; - /** Zarr array corresponding to the current resolution level */ + /** Zarr array corresponding to the current resolution level. */ private ZArray array; /** @@ -122,12 +121,13 @@ public ZarrPixelBuffer(Pixels pixels, ZarrPath root, Integer maxPlaneWidth, this.isRemote = root.toString().startsWith("s3://") ? true : false; try { Map tmp = this.zarrMetadataCache.get(this.root).get(); - if (tmp.containsKey("ome")) { // for ngff challenge data attr are often nested within "ome" key + if (tmp.containsKey("ome")) { + // for ngff challenge data attr are often nested within "ome" key rootGroupAttributes = (Map) tmp.get("ome"); } else { rootGroupAttributes = tmp; } - } catch (ExecutionException|InterruptedException e) { + } catch (ExecutionException | InterruptedException e) { throw new IOException(e); } if (!rootGroupAttributes.containsKey("multiscales")) { @@ -276,7 +276,7 @@ public int[][] getChunks() throws IOException { List chunks = new ArrayList(); for (Map dataset : datasets) { ZarrPath dsPath = root.resolve(dataset.get("path")); - ZArray resolutionArray = dsPath.getArray(); //new ZArrayv2(ZarrArray.open((Path)dsPath.getPath())); + ZArray resolutionArray = dsPath.getArray(); int[] shape = resolutionArray.getChunks(); chunks.add(shape); } diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java index eb47e4d..9e8cc2f 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArray.java @@ -1,18 +1,56 @@ package com.glencoesoftware.omero.zarr.compat; import java.io.IOException; - import ucar.ma2.InvalidRangeException; +/** + * Interface for representing a Zarr array with methods for accessing array properties and reading + * data from the array. + */ public interface ZArray { + + /** + * Gets the shape (dimensions) of the array. + * + * @return an array of integers representing the size of each dimension + */ public int[] getShape(); + /** + * Gets the chunk size. + * + * @return an array of integers representing the chunk size. + */ public int[] getChunks(); - public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException; - + /** + * Reads data from the array into a provided byte buffer. + * + * @param buffer the byte buffer to read data into + * @param shape the shape of the data to read + * @param offset the offset position to start reading from + * @throws IOException if an I/O error occurs during reading + * @throws InvalidRangeException if the specified range is invalid + */ + public void read(byte[] buffer, int[] shape, int[] offset) + throws IOException, InvalidRangeException; + + /** + * Reads data from the array and returns it as an Object (short[], int[], etc.). + * + * @param shape the shape of the data to read + * @param offset the offset position to start reading from + * @return the data read from the array as an Object + * @throws IOException if an I/O error occurs during reading + * @throws InvalidRangeException if the specified range is invalid + */ public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeException; + /** + * Gets the pixel type identifier for this array. + * + * @return an integer representing the pixel type + */ public int getPixelsType(); } diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java index fa6785d..1dae765 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv2.java @@ -1,15 +1,13 @@ package com.glencoesoftware.omero.zarr.compat; -import java.io.IOException; - import com.bc.zarr.DataType; import com.bc.zarr.ZarrArray; - +import java.io.IOException; import loci.formats.FormatTools; import ucar.ma2.InvalidRangeException; class ZArrayv2 implements ZArray { - + private ZarrArray array; public ZArrayv2(ZarrArray array) { @@ -27,7 +25,8 @@ public int[] getChunks() { } @Override - public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException { + public void read(byte[] buffer, int[] shape, int[] offset) + throws IOException, InvalidRangeException { array.read(buffer, shape, offset); } @@ -36,11 +35,12 @@ public Object read(int[] shape, int[] offset) throws IOException, InvalidRangeEx return array.read(shape, offset); } - @Override /** * Get Bio-Formats/OMERO pixels type for buffer. + * * @return See above. */ + @Override public int getPixelsType() { DataType dataType = array.getDataType(); switch (dataType) { @@ -61,8 +61,7 @@ public int getPixelsType() { case f8: return FormatTools.DOUBLE; default: - throw new IllegalArgumentException( - "Data type " + dataType + " not supported"); + throw new IllegalArgumentException("Data type " + dataType + " not supported"); } } } diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java index 2ec04b2..2dc52a3 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZArrayv3.java @@ -1,11 +1,10 @@ package com.glencoesoftware.omero.zarr.compat; -import java.io.IOException; -import java.nio.ByteBuffer; - import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.DataType; +import java.io.IOException; +import java.nio.ByteBuffer; import loci.formats.FormatTools; import ucar.ma2.InvalidRangeException; @@ -36,17 +35,18 @@ public int[] getChunks() { } @Override - public void read(byte[] buffer, int[] shape, int[] offset) throws IOException, InvalidRangeException { - try { - long[] offsetLong = new long[offset.length]; - for (int i = 0; i < offset.length; i++) { - offsetLong[i] = offset[i]; + public void read(byte[] buffer, int[] shape, int[] offset) + throws IOException, InvalidRangeException { + try { + long[] offsetLong = new long[offset.length]; + for (int i = 0; i < offset.length; i++) { + offsetLong[i] = offset[i]; + } + ByteBuffer b = array.read(offsetLong, shape).getDataAsByteBuffer(); + System.arraycopy(b.array(), 0, buffer, 0, buffer.length); + } catch (ZarrException e) { + throw new IOException(e); } - ByteBuffer b = array.read(offsetLong, shape).getDataAsByteBuffer(); - System.arraycopy(b.array(), 0, buffer, 0, buffer.length); - } catch (ZarrException e) { - throw new IOException(e); - } } @Override @@ -83,8 +83,7 @@ public int getPixelsType() { case FLOAT64: return FormatTools.DOUBLE; default: - throw new IllegalArgumentException( - "Data type " + dataType + " not supported"); + throw new IllegalArgumentException("Data type " + dataType + " not supported"); } } } diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java index b1c2f69..ee23b7c 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrInfo.java @@ -1,5 +1,15 @@ package com.glencoesoftware.omero.zarr.compat; +import com.amazonaws.services.s3.AmazonS3; +import com.bc.zarr.ZarrGroup; +import com.google.common.base.Splitter; +import com.upplication.s3fs.OmeroS3FilesystemProvider; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.GroupMetadata; import java.io.File; import java.io.IOException; import java.net.URI; @@ -11,32 +21,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; - -import org.slf4j.LoggerFactory; import org.apache.maven.artifact.versioning.ComparableVersion; - -import com.amazonaws.services.s3.AmazonS3; -import com.bc.zarr.ZarrGroup; -import com.google.common.base.Splitter; -import com.upplication.s3fs.OmeroS3FilesystemProvider; - -import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.HttpStore; -import dev.zarr.zarrjava.store.S3Store; -import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.Group; -import dev.zarr.zarrjava.v3.GroupMetadata; - +import org.slf4j.LoggerFactory; /** - * Tries to determine some properties of the zarr path, - * if it's remote or local, and the zarr and ngff versions. - * - * To access the zarr metadata/array use getZarrPath(). + * Tries to determine some properties of the zarr path, if it's remote or local, and the zarr and + * ngff versions. To access the zarr metadata/array use getZarrPath(). */ public class ZarrInfo { - private static final org.slf4j.Logger log = - LoggerFactory.getLogger(ZarrInfo.class); + private static final org.slf4j.Logger log = LoggerFactory.getLogger(ZarrInfo.class); public static final ComparableVersion ZARR_V2 = new ComparableVersion("2"); public static final ComparableVersion ZARR_V3 = new ComparableVersion("3"); @@ -53,22 +46,28 @@ public class ZarrInfo { * Enum representing different storage types for Zarr data. */ public enum StorageType { - FILE, - S3, - HTTP + FILE, S3, HTTP } private StorageType storageType; + /** + * Create a new ZarrInfo. + * + * @param location the location of the zarr + * @throws IOException if the zarr can't be read + */ public ZarrInfo(String location) throws IOException { - this.location = location.endsWith("/") ? location.substring(0, location.length() - 1) : location; + this.location = location.endsWith("/") ? location.substring(0, location.length() - 1) + : location; checkProperties(); } /** - * Tries to check some properties of the zarr path, - * if it's remote or local, and the zarr and ngff versions. - * @throws IOException + * Tries to check some properties of the zarr path, if it's remote or local, and the zarr and + * ngff versions. + * + * @throws IOException If the zarr can't be read */ private void checkProperties() throws IOException { URI uri; @@ -102,9 +101,10 @@ private void checkProperties() throws IOException { try { StoreHandle sh = asStoreHandle(); GroupMetadata md = Group.open(sh).metadata; - zarrVersion = new ComparableVersion("3"); // if that works it should be v3 + zarrVersion = new ComparableVersion("3"); // if that works it should be v3 try { - ngffVersion = new ComparableVersion(((Map) md.attributes.get("ome")).get("version").toString()); + ngffVersion = new ComparableVersion( + ((Map) md.attributes.get("ome")).get("version").toString()); } catch (Exception e) { log.debug("Failed to get ngff version from zarr, set to 0.5"); ngffVersion = new ComparableVersion("0.5"); @@ -121,10 +121,12 @@ private void checkProperties() throws IOException { zarrVersion = new ComparableVersion("2"); // if that works it must be v2 try { List tmp = (List) attr.get("multiscales"); - ngffVersion = new ComparableVersion(((Map) tmp.get(0)).get("version").toString()); + ngffVersion = new ComparableVersion( + ((Map) tmp.get(0)).get("version").toString()); } catch (Exception e) { log.debug("Failed to get ngff version from zarr, set to 0.4"); - ngffVersion = new ComparableVersion("0.4"); // if it's zarr v2 then we can actually safely assume it's ngff v0.4 + ngffVersion = new ComparableVersion("0.4"); + // if it's zarr v2 then we can actually safely assume it's ngff v0.4 } return; } catch (Exception e) { @@ -135,6 +137,7 @@ private void checkProperties() throws IOException { /** * Gets the Zarr version. + * * @return the Zarr version as a string */ public ComparableVersion getZarrVersion() { @@ -143,6 +146,7 @@ public ComparableVersion getZarrVersion() { /** * Gets the NGFF version. + * * @return the NGFF version as a string */ public ComparableVersion getNgffVersion() { @@ -151,12 +155,19 @@ public ComparableVersion getNgffVersion() { /** * Gets the path. + * * @return the path */ public String getLocation() { return location; } + /** + * Gets the Zarr path. + * + * @return the Zarr path + * @throws IOException If the zarr can't be read + */ public ZarrPath getZarrPath() throws IOException { if (zarrVersion.equals(ZARR_V2)) { return new ZarrPathv2(asPath(location)); @@ -166,12 +177,12 @@ public ZarrPath getZarrPath() throws IOException { } /** - * Converts an NGFF root string to a path, initializing a {@link FileSystem} - * if required - * For zarr version 2, resp NGFF 0.4 - * @return Fully initialized path or null if the NGFF root - * directory has not been specified in configuration. - * @throws IOException + * Converts an NGFF root string to a path, initializing a {@link FileSystem} if required For + * zarr version 2, resp NGFF 0.4 + * + * @return Fully initialized path or null if the NGFF root directory has not been + * specified in configuration. + * @throws IOException If the path can't be read */ private Path asPath(String location) throws IOException { try { @@ -179,37 +190,31 @@ private Path asPath(String location) throws IOException { String scheme = uri.getScheme() != null ? uri.getScheme().toLowerCase() : "file"; if (scheme.startsWith("http")) { String s3loc = location.replaceFirst("https?", "s3"); - return asPath(s3loc+"?anonymous=true"); + return asPath(s3loc + "?anonymous=true"); } if (scheme.equals("s3")) { if (uri.getUserInfo() != null && !uri.getUserInfo().isEmpty()) { - throw new RuntimeException( - "Found unsupported user information in S3 URI." + throw new RuntimeException("Found unsupported user information in S3 URI." + " If you are trying to pass S3 credentials, " + "use either named profiles or instance credentials."); } String query = Optional.ofNullable(uri.getQuery()).orElse(""); - Map params = Splitter.on('&') - .trimResults() - .omitEmptyStrings() - .withKeyValueSeparator('=') - .split(query); + Map params = Splitter.on('&').trimResults().omitEmptyStrings() + .withKeyValueSeparator('=').split(query); // drop initial "/" String uriPath = uri.getPath().substring(1); int first = uriPath.indexOf("/"); String bucket = "/" + uriPath.substring(0, first); String rest = uriPath.substring(first + 1); // FIXME: We might want to support additional S3FS settings in - // the future. See: - // * https://github.com/lasersonlab/Amazon-S3-FileSystem-NIO2 + // the future. See: + // * https://github.com/lasersonlab/Amazon-S3-FileSystem-NIO2 Map env = new HashMap(); String profile = params.get("profile"); if (profile != null) { env.put("s3fs_credential_profile_name", profile); } - String anonymous = - Optional.ofNullable(params.get("anonymous")) - .orElse("false"); + String anonymous = Optional.ofNullable(params.get("anonymous")).orElse("false"); env.put("s3fs_anonymous", anonymous); OmeroS3FilesystemProvider fsp = new OmeroS3FilesystemProvider(); FileSystem fs = fsp.newFileSystem(uri, env); @@ -222,43 +227,36 @@ private Path asPath(String location) throws IOException { } /** - * Return a store handle. - * For zarr version 3, resp NGFF > 0.5 - * @return + * Return a store handle. For zarr version 3, resp NGFF > 0.5 + * + * @return the store handle */ private StoreHandle asStoreHandle() { try { URI uri = new URI(location); String scheme = uri.getScheme() != null ? uri.getScheme().toLowerCase() : "file"; - String store = location.substring(0, location.lastIndexOf("/")); - String zarr = location.substring(location.lastIndexOf("/")+1); - zarr = zarr.replaceFirst("\\?.+", ""); + String locationNoParams = location.replaceAll("/?\\?.+", ""); + String store = locationNoParams.substring(0, location.lastIndexOf("/")); + String zarr = locationNoParams.substring(location.lastIndexOf("/") + 1); if (scheme.startsWith("http")) { return new HttpStore(store).resolve(zarr); - } - else if (scheme.startsWith("s3")) { + } else if (scheme.startsWith("s3")) { String[] tmp = store.replaceFirst("s3://", "").split("/"); - String host = tmp[0]; + // String host = tmp[0]; String bucket = tmp[1]; String rest = String.join("/", Arrays.copyOfRange(tmp, 2, tmp.length)); - System.out.println("Host: " + host); // Use the OmeroS3FilesystemProvider to create AmazonS3 client String query = Optional.ofNullable(uri.getQuery()).orElse(""); - Map params = Splitter.on('&') - .trimResults() - .omitEmptyStrings() - .withKeyValueSeparator('=') - .split(query); + Map params = Splitter.on('&').trimResults().omitEmptyStrings() + .withKeyValueSeparator('=').split(query); Map env = new HashMap(); String profile = params.get("profile"); if (profile != null) { env.put("s3fs_credential_profile_name", profile); } - String anonymous = - Optional.ofNullable(params.get("anonymous")) - .orElse("false"); + String anonymous = Optional.ofNullable(params.get("anonymous")).orElse("false"); env.put("s3fs_anonymous", anonymous); OmeroS3FilesystemProvider fsp = new OmeroS3FilesystemProvider(); AmazonS3 client = fsp.createAmazonS3(uri, env); @@ -267,20 +265,18 @@ else if (scheme.startsWith("s3")) { // AmazonS3 client = AmazonS3ClientBuilder.standard() // .withEndpointConfiguration(new EndpointConfiguration(host, null)) // .withCredentials(new AWSCredentialsProvider() { - // @Override - // public AWSCredentials getCredentials() { - // return new AnonymousAWSCredentials(); - // } + // @Override + // public AWSCredentials getCredentials() { + // return new AnonymousAWSCredentials(); + // } - // @Override - // public void refresh() { - // // do nothing - // } - // }) - // .build(); + // @Override + // public void refresh() { + // // do nothing + // } + // }).build(); return new S3Store(client, bucket, rest).resolve(zarr); - } - else { + } else { return new FilesystemStore(store).resolve(zarr); } } catch (URISyntaxException e) { @@ -291,12 +287,8 @@ else if (scheme.startsWith("s3")) { @Override public String toString() { - return "ZarrInfo{" + - "location='" + location + '\'' + - ", storageType=" + storageType + - ", zarrVersion=" + zarrVersion + - ", ngffVersion=" + ngffVersion + - '}'; + return "ZarrInfo{" + "location='" + location + '\'' + ", storageType=" + storageType + + ", zarrVersion=" + zarrVersion + ", ngffVersion=" + ngffVersion + '}'; } public StorageType getStorageType() { diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java index ff6b03c..a27c057 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPath.java @@ -3,15 +3,33 @@ import java.io.IOException; import java.util.Map; -import org.apache.maven.artifact.versioning.ComparableVersion; - +/** + * Interface for representing a path within a Zarr hierarchy. Provides methods for navigating the + * Zarr structure, accessing metadata, and retrieving array information. + */ public interface ZarrPath { + /** + * Resolves a relative path against this ZarrPath. + * + * @param path the relative path to resolve + * @return a new ZarrPath representing the resolved path + */ public ZarrPath resolve(String path); - public ComparableVersion getVersion(); - + /** + * Retrieves the metadata associated with this Zarr path. + * + * @return a map containing the metadata key-value pairs + * @throws IOException if an I/O error occurs while reading the metadata + */ public Map getMetadata() throws IOException; + /** + * Gets the Zarr array associated with this path. + * + * @return the ZArray object representing the array at this path + * @throws IOException if an I/O error occurs while accessing the array + */ public ZArray getArray() throws IOException; } diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java index b0b3039..2079ee8 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv2.java @@ -1,14 +1,11 @@ package com.glencoesoftware.omero.zarr.compat; +import com.bc.zarr.ZarrArray; +import com.bc.zarr.ZarrGroup; import java.io.IOException; import java.nio.file.Path; import java.util.Map; -import org.apache.maven.artifact.versioning.ComparableVersion; - -import com.bc.zarr.ZarrArray; -import com.bc.zarr.ZarrGroup; - class ZarrPathv2 implements ZarrPath { private Path path; @@ -22,11 +19,6 @@ public ZarrPath resolve(String path) { return new ZarrPathv2(this.path.resolve(path)); } - @Override - public ComparableVersion getVersion() { - return ZarrInfo.ZARR_V2; - } - @Override public Map getMetadata() throws IOException { return ZarrGroup.open(path).getAttributes(); @@ -34,7 +26,7 @@ public Map getMetadata() throws IOException { @Override public ZArray getArray() throws IOException { - return new ZArrayv2(ZarrArray.open(path)); + return new ZArrayv2(ZarrArray.open(path)); } @Override diff --git a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java index 0a2719e..b3690ef 100644 --- a/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java +++ b/src/main/java/com/glencoesoftware/omero/zarr/compat/ZarrPathv3.java @@ -1,14 +1,11 @@ package com.glencoesoftware.omero.zarr.compat; -import java.io.IOException; -import java.util.Map; - -import org.apache.maven.artifact.versioning.ComparableVersion; - import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.Group; import dev.zarr.zarrjava.v3.Node; +import java.io.IOException; +import java.util.Map; class ZarrPathv3 implements ZarrPath { @@ -44,11 +41,6 @@ public ZarrPath resolve(String resolvePath) { return new ZarrPathv3(this, resolvePath); } - @Override - public ComparableVersion getVersion() { - return ZarrInfo.ZARR_V3; - } - @Override public Map getMetadata() throws IOException { return group.metadata.attributes; @@ -85,6 +77,7 @@ public int hashCode() { @Override public String toString() { - return key == null ? "(Group) "+ path.toString() : "(Array) " + path.toString() + ":" + key; + return key == null ? "(Group) " + path.toString() + : "(Array) " + path.toString() + ":" + key; } } diff --git a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java index 26565ec..b5c756c 100644 --- a/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java +++ b/src/main/java/com/upplication/s3fs/OmeroS3FilesystemProvider.java @@ -107,6 +107,9 @@ public S3FileSystem createFileSystem(URI uri, Properties props) { this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost()); } + /** + * Create an Amazon S3 client from the given URI and environment. + */ public AmazonS3 createAmazonS3(URI uri, Map env) { Properties props = getProperties(uri, env); validateProperties(props); diff --git a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java index a79198d..be7a68d 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/ZarrPixelBufferTest.java @@ -19,11 +19,11 @@ package com.glencoesoftware.omero.zarr; import com.bc.zarr.ZarrArray; -import com.bc.zarr.ZarrGroup; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.Caffeine; import com.glencoesoftware.bioformats2raw.Converter; import com.glencoesoftware.omero.zarr.ZarrPixelBuffer.Axis; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo; import java.awt.Dimension; import java.io.File; import java.io.IOException; @@ -44,7 +44,6 @@ import ome.io.nio.DimensionsOutOfBoundsException; import ome.model.core.Pixels; import ome.model.enums.DimensionOrder; -import ome.model.enums.PixelsType; import ome.util.PixelData; import org.junit.Assert; import org.junit.Rule; @@ -62,19 +61,12 @@ public class ZarrPixelBufferTest { public TemporaryFolder tmpDir = new TemporaryFolder(); /** Constructor. */ - public ZarrPixelBuffer createPixelBuffer( - Pixels pixels, Path path, - Integer maxPlaneWidth, Integer maxPlaneHeight) throws IOException { + public ZarrPixelBuffer createPixelBuffer(Pixels pixels, Path path, Integer maxPlaneWidth, + Integer maxPlaneHeight) throws IOException { ZarrInfo zarrInfo = new ZarrInfo(path.toString()); - return new ZarrPixelBuffer( - pixels, zarrInfo.getZarrPath(), maxPlaneWidth, maxPlaneHeight, - Caffeine.newBuilder() - .maximumSize(0) - .buildAsync(ZarrPixelsService::getZarrMetadata), - Caffeine.newBuilder() - .maximumSize(0) - .buildAsync(ZarrPixelsService::getZarrArray) - ); + return new ZarrPixelBuffer(pixels, zarrInfo.getZarrPath(), maxPlaneWidth, maxPlaneHeight, + Caffeine.newBuilder().maximumSize(0).buildAsync(ZarrPixelsService::getZarrMetadata), + Caffeine.newBuilder().maximumSize(0).buildAsync(ZarrPixelsService::getZarrArray)); } /** @@ -82,10 +74,10 @@ public ZarrPixelBuffer createPixelBuffer( * * @param additionalArgs CLI arguments as needed beyond "input output" */ - void assertBioFormats2Raw(Path input, Path output, String... additionalArgs) - throws IOException { + static void assertBioFormats2Raw(Path input, Path output, String... additionalArgs) + throws IOException { List args = new ArrayList( - Arrays.asList(new String[] { "--compression", "null" })); + Arrays.asList(new String[] { "--compression", "null" })); for (String arg : additionalArgs) { args.add(arg); } @@ -94,10 +86,9 @@ void assertBioFormats2Raw(Path input, Path output, String... additionalArgs) try { Converter converter = new Converter(); CommandLine cli = new CommandLine(converter); - cli.execute(args.toArray(new String[]{})); + cli.execute(args.toArray(new String[] {})); Assert.assertTrue(Files.exists(output.resolve(".zattrs"))); - Assert.assertTrue(Files.exists( - output.resolve("OME").resolve("METADATA.ome.xml"))); + Assert.assertTrue(Files.exists(output.resolve("OME").resolve("METADATA.ome.xml"))); } catch (RuntimeException rt) { throw rt; } catch (Throwable t) { @@ -121,21 +112,21 @@ static Path fake(Map options) { /** * Create a Bio-Formats fake INI file to use for testing. * - * @param options map of the options to assign as part of the fake filename - * from the allowed keys - * @param series map of the integer series index and options map (same format - * as options to add to the fake INI content - * @see fake file specification + * @param options map of the options to assign as part of the fake filename from the allowed + * keys + * @param series map of the integer series index and options map (same format as + * options to add to the fake INI content + * @see fake + * file specification * @return path to the fake INI file that has been created */ - static Path fake(Map options, - Map> series) { + static Path fake(Map options, Map> series) { return fake(options, series, null); } - static Path fake(Map options, - Map> series, - Map originalMetadata) { + static Path fake(Map options, Map> series, + Map originalMetadata) { StringBuilder sb = new StringBuilder(); sb.append("image"); if (options != null) { @@ -152,8 +143,7 @@ static Path fake(Map options, if (originalMetadata != null) { lines.add("[GlobalMetadata]"); for (String key : originalMetadata.keySet()) { - lines.add(String.format( - "%s=%s", key, originalMetadata.get(key))); + lines.add(String.format("%s=%s", key, originalMetadata.get(key))); } } if (series != null) { @@ -161,8 +151,7 @@ static Path fake(Map options, Map seriesOptions = series.get(s); lines.add(String.format("[series_%d]", s)); for (String key : seriesOptions.keySet()) { - lines.add(String.format( - "%s=%s", key, seriesOptions.get(key))); + lines.add(String.format("%s=%s", key, seriesOptions.get(key))); } } } @@ -171,7 +160,7 @@ static Path fake(Map options, String iniPath = iniAsFile.getAbsolutePath(); String fakePath = iniPath.substring(0, iniPath.length() - 4); Path fake = Paths.get(fakePath); - Files.write(fake, new byte[]{}); + Files.write(fake, new byte[] {}); Files.write(ini, lines); iniAsFile.deleteOnExit(); File fakeAsFile = fake.toFile(); @@ -183,22 +172,12 @@ static Path fake(Map options, } /** Write Zarr multiscales attributes. */ - public Path writeTestZarr( - int sizeT, - int sizeC, - int sizeZ, - int sizeY, - int sizeX, - String pixelType, - String... options) throws IOException { - - Path input = fake( - "sizeT", Integer.toString(sizeT), - "sizeC", Integer.toString(sizeC), - "sizeZ", Integer.toString(sizeZ), - "sizeY", Integer.toString(sizeY), - "sizeX", Integer.toString(sizeX), - "pixelType", pixelType); + public Path writeTestZarr(int sizeT, int sizeC, int sizeZ, int sizeY, int sizeX, + String pixelType, String... options) throws IOException { + + Path input = fake("sizeT", Integer.toString(sizeT), "sizeC", Integer.toString(sizeC), + "sizeZ", Integer.toString(sizeZ), "sizeY", Integer.toString(sizeY), "sizeX", + Integer.toString(sizeX), "pixelType", pixelType); Path output = tmpDir.getRoot().toPath().resolve("output.zarr"); assertBioFormats2Raw(input, output, options); return output; @@ -213,17 +192,12 @@ public void testGetChunks() throws IOException { int sizeX = 2048; int resolutions = 3; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", - "--resolutions", String.valueOf(resolutions)); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--resolutions", + String.valueOf(resolutions)); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { int[][] chunks = zpbuf.getChunks(); - int[][] expectedChunks = new int[][] { - new int[] {1, 1, 1, 512, 1024}, - new int[] {1, 1, 1, 256, 1024}, - new int[] {1, 1, 1, 128, 512} - }; + int[][] expectedChunks = new int[][] { new int[] { 1, 1, 1, 512, 1024 }, + new int[] { 1, 1, 1, 256, 1024 }, new int[] { 1, 1, 1, 128, 512 } }; Assert.assertEquals(chunks, expectedChunks); } } @@ -238,19 +212,13 @@ public void testGet3DChunks() throws IOException { int resolutions = 4; int chunkDepth = 16; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", - "--resolutions", String.valueOf(resolutions), - "--chunk-depth", String.valueOf(chunkDepth)); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--resolutions", + String.valueOf(resolutions), "--chunk-depth", String.valueOf(chunkDepth)); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { int[][] chunks = zpbuf.getChunks(); - int[][] expectedChunks = new int[][] { - new int[] {1, 1, chunkDepth, 512, 1024}, - new int[] {1, 1, chunkDepth, 256, 1024}, - new int[] {1, 1, chunkDepth, 128, 512}, - new int[] {1, 1, chunkDepth, 64, 256} - }; + int[][] expectedChunks = new int[][] { new int[] { 1, 1, chunkDepth, 512, 1024 }, + new int[] { 1, 1, chunkDepth, 256, 1024 }, new int[] { 1, 1, chunkDepth, 128, 512 }, + new int[] { 1, 1, chunkDepth, 64, 256 } }; Assert.assertEquals(chunks, expectedChunks); } } @@ -264,25 +232,22 @@ public void testGetDatasets() throws IOException { int sizeX = 2048; int resolutions = 3; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", - "--resolutions", String.valueOf(resolutions)); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--resolutions", + String.valueOf(resolutions)); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { List> datasets = zpbuf.getDatasets(); Assert.assertEquals(datasets.size(), resolutions); for (int i = 0; i < datasets.size(); i++) { Assert.assertEquals(datasets.get(i).get("path"), Integer.toString(i)); - List> transformations = - new ArrayList>(); + List> transformations = new ArrayList>(); Map scale = new HashMap(); scale.put("type", "scale"); - scale.put("scale", Arrays.asList( - new Double[] {1.0, 1.0, 1.0, Math.pow(2, i), Math.pow(2, i)})); + scale.put("scale", + Arrays.asList(new Double[] { 1.0, 1.0, 1.0, Math.pow(2, i), Math.pow(2, i) })); transformations.add(scale); - Assert.assertEquals( - datasets.get(i).get("coordinateTransformations"), transformations); + Assert.assertEquals(datasets.get(i).get("coordinateTransformations"), + transformations); } } } @@ -296,15 +261,13 @@ public void testGetResolutionDescriptions() throws IOException { int sizeX = 2048; int resolutions = 3; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", - "--resolutions", String.valueOf(resolutions)); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--resolutions", + String.valueOf(resolutions)); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { List> expected = new ArrayList>(); - expected.add(Arrays.asList(new Integer[] {2048, 512})); - expected.add(Arrays.asList(new Integer[] {1024, 256})); - expected.add(Arrays.asList(new Integer[] {512, 128})); + expected.add(Arrays.asList(new Integer[] { 2048, 512 })); + expected.add(Arrays.asList(new Integer[] { 1024, 256 })); + expected.add(Arrays.asList(new Integer[] { 512, 128 })); Assert.assertEquals(resolutions, zpbuf.getResolutionLevels()); Assert.assertEquals(expected, zpbuf.getResolutionDescriptions()); @@ -344,9 +307,9 @@ public void testGetTile() throws IOException, InvalidRangeException { for (int i = 0; i < length; i++) { data[i] = i; } - test.write(data, new int[] {sizeT, sizeC, sizeZ, sizeY, sizeX}, new int[] {0, 0, 0, 0, 0}); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + test.write(data, new int[] { sizeT, sizeC, sizeZ, sizeY, sizeX }, + new int[] { 0, 0, 0, 0, 0 }); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { PixelData pixelData = zpbuf.getTile(0, 0, 0, 0, 0, 2, 2); ByteBuffer bb = pixelData.getData(); bb.order(ByteOrder.BIG_ENDIAN); @@ -374,15 +337,12 @@ private Array asArray(byte[] storage, int[] shape) { return Array.factory(DataType.INT, shape, asInt); } - private byte[] getStack( - byte[] timepoint, int c, int sizeC, int sizeZ, int sizeX, - int sizeY) { + private byte[] getStack(byte[] timepoint, int c, int sizeC, int sizeZ, int sizeX, int sizeY) { return getStack(timepoint, c, sizeC, sizeZ, sizeX, sizeY, "TCZYX"); } - private byte[] getStack( - byte[] timepoint, int c, int sizeC, int sizeZ, int sizeX, - int sizeY, String order) { + private byte[] getStack(byte[] timepoint, int c, int sizeC, int sizeZ, int sizeX, int sizeY, + String order) { // XXX: Is not data type agnostic, expects signed 32-bit integer pixels int[] shape = new int[4]; String shapeorder = order.replace("T", ""); @@ -391,22 +351,19 @@ private byte[] getStack( shape[shapeorder.indexOf('Y')] = sizeY; shape[shapeorder.indexOf('X')] = sizeX; int bytesPerPixel = 4; - int size = IntStream.of(new int[] {sizeZ, sizeY, sizeX, bytesPerPixel}) - .reduce(1, Math::multiplyExact); + int size = IntStream.of(new int[] { sizeZ, sizeY, sizeX, bytesPerPixel }).reduce(1, + Math::multiplyExact); Array array = asArray(timepoint, shape).slice(shapeorder.indexOf('C'), c); byte[] asBytes = new byte[size]; - ByteBuffer.wrap(asBytes).asIntBuffer() - .put((int[]) array.copyTo1DJavaArray()); + ByteBuffer.wrap(asBytes).asIntBuffer().put((int[]) array.copyTo1DJavaArray()); return asBytes; } - private byte[] getPlane( - byte[] stack, int z, int sizeZ, int sizeX, int sizeY) { + private byte[] getPlane(byte[] stack, int z, int sizeZ, int sizeX, int sizeY) { return getPlane(stack, z, sizeZ, sizeX, sizeY, "TCZYX"); } - private byte[] getPlane( - byte[] stack, int z, int sizeZ, int sizeX, int sizeY, String order) { + private byte[] getPlane(byte[] stack, int z, int sizeZ, int sizeX, int sizeY, String order) { // XXX: Is not data type agnostic, expects signed 32-bit integer pixels String shapeorder = order.replace("T", "").replace("C", ""); int[] shape = new int[3]; @@ -414,33 +371,27 @@ private byte[] getPlane( shape[shapeorder.indexOf('Y')] = sizeY; shape[shapeorder.indexOf('X')] = sizeX; int bytesPerPixel = 4; - int size = IntStream.of(new int[] {sizeY, sizeX, bytesPerPixel}) - .reduce(1, Math::multiplyExact); + int size = IntStream.of(new int[] { sizeY, sizeX, bytesPerPixel }).reduce(1, + Math::multiplyExact); Array array = asArray(stack, shape).slice(shapeorder.indexOf('Z'), z); byte[] asBytes = new byte[size]; - ByteBuffer.wrap(asBytes).asIntBuffer() - .put((int[]) array.copyTo1DJavaArray()); + ByteBuffer.wrap(asBytes).asIntBuffer().put((int[]) array.copyTo1DJavaArray()); return asBytes; } - private void assertPixels( - ZarrPixelBuffer zpbuf, int sizeX, int sizeY, int sizeZ, int sizeC, int sizeT) - throws IOException { + private void assertPixels(ZarrPixelBuffer zpbuf, int sizeX, int sizeY, int sizeZ, int sizeC, + int sizeT) throws IOException { for (int t = 0; t < sizeT; t++) { for (int c = 0; c < sizeC; c++) { for (int z = 0; z < sizeZ; z++) { byte[] plane = zpbuf.getPlane(z, c, t).getData().array(); - int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels( - plane, zpbuf.getPixelsType(), false); - int planeNumber = FormatTools.getIndex( - DimensionOrder.VALUE_XYZCT, - sizeZ, sizeC, sizeT, sizeZ * sizeC * sizeT, - z, c, t); - Assert.assertArrayEquals( - Arrays.toString(seriesPlaneNumberZCT), - new int[] {0, planeNumber, z, c, t}, - seriesPlaneNumberZCT); + int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels(plane, + zpbuf.getPixelsType(), false); + int planeNumber = FormatTools.getIndex(DimensionOrder.VALUE_XYZCT, sizeZ, sizeC, + sizeT, sizeZ * sizeC * sizeT, z, c, t); + Assert.assertArrayEquals(Arrays.toString(seriesPlaneNumberZCT), + new int[] { 0, planeNumber, z, c, t }, seriesPlaneNumberZCT); } } } @@ -472,19 +423,16 @@ private void assertAxes(ZarrPixelBuffer zpbuf, String order) { private byte[] getCol(byte[] plane, int x, int sizeX, int sizeY) { // XXX: Is not data type agnostic, expects signed 32-bit integer pixels int bytesPerPixel = 4; - int[] shape = new int[] {sizeY, sizeX}; - int size = IntStream.of(new int[] {sizeY, bytesPerPixel}) - .reduce(1, Math::multiplyExact); + int[] shape = new int[] { sizeY, sizeX }; + int size = IntStream.of(new int[] { sizeY, bytesPerPixel }).reduce(1, Math::multiplyExact); Array array = asArray(plane, shape).slice(1, x); byte[] asBytes = new byte[size]; - ByteBuffer.wrap(asBytes).asIntBuffer() - .put((int[]) array.copyTo1DJavaArray()); + ByteBuffer.wrap(asBytes).asIntBuffer().put((int[]) array.copyTo1DJavaArray()); return asBytes; } @Test - public void testGetTimepointStackPlaneRowCol() - throws IOException, InvalidRangeException { + public void testGetTimepointStackPlaneRowCol() throws IOException, InvalidRangeException { int sizeT = 2; int sizeC = 3; int sizeZ = 4; @@ -492,42 +440,33 @@ public void testGetTimepointStackPlaneRowCol() int sizeX = 2048; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "int32"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 2048, 2048)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 2048, 2048)) { for (int t = 0; t < sizeT; t++) { // Assert timepoint byte[] timepoint = zpbuf.getTimepoint(t).getData().array(); for (int c = 0; c < sizeC; c++) { // Assert stack byte[] stack = zpbuf.getStack(c, t).getData().array(); - byte[] stackFromTimepoint = - getStack(timepoint, c, sizeC, sizeZ, sizeX, sizeY); + byte[] stackFromTimepoint = getStack(timepoint, c, sizeC, sizeZ, sizeX, sizeY); Assert.assertArrayEquals(stack, stackFromTimepoint); for (int z = 0; z < sizeZ; z++) { // Assert plane - byte[] plane = - zpbuf.getPlane(z, c, t).getData().array(); - byte[] planeFromStack = - getPlane(stack, z, sizeZ, sizeX, sizeY); + byte[] plane = zpbuf.getPlane(z, c, t).getData().array(); + byte[] planeFromStack = getPlane(stack, z, sizeZ, sizeX, sizeY); Assert.assertArrayEquals(plane, planeFromStack); - int[] seriesPlaneNumberZCT = - FakeReader.readSpecialPixels( - plane, zpbuf.getPixelsType(), false); - int planeNumber = FormatTools.getIndex( - DimensionOrder.VALUE_XYZCT, - sizeZ, sizeC, sizeT, sizeZ * sizeC * sizeT, - z, c, t); - Assert.assertArrayEquals( - Arrays.toString(seriesPlaneNumberZCT), - new int[] {0, planeNumber, z, c, t}, - seriesPlaneNumberZCT); + int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels(plane, + zpbuf.getPixelsType(), false); + int planeNumber = FormatTools.getIndex(DimensionOrder.VALUE_XYZCT, sizeZ, + sizeC, sizeT, sizeZ * sizeC * sizeT, z, c, t); + Assert.assertArrayEquals(Arrays.toString(seriesPlaneNumberZCT), + new int[] { 0, planeNumber, z, c, t }, seriesPlaneNumberZCT); // Assert row int y = sizeY / 2; int rowSize = zpbuf.getRowSize(); int rowOffset = y * rowSize; byte[] row = zpbuf.getRow(y, z, c, t).getData().array(); - byte[] rowExpected = Arrays.copyOfRange( - plane, rowOffset, rowOffset + rowSize); + byte[] rowExpected = Arrays.copyOfRange(plane, rowOffset, + rowOffset + rowSize); Assert.assertArrayEquals(rowExpected, row); // Assert column int x = sizeX / 2; @@ -541,8 +480,7 @@ public void testGetTimepointStackPlaneRowCol() } @Test(expected = DimensionsOutOfBoundsException.class) - public void testGetTileLargerThanImage() - throws IOException, InvalidRangeException { + public void testGetTileLargerThanImage() throws IOException, InvalidRangeException { int sizeT = 2; int sizeC = 3; int sizeZ = 4; @@ -556,9 +494,9 @@ public void testGetTileLargerThanImage() for (int i = 0; i < length; i++) { data[i] = i; } - test.write(data, new int[] {sizeT, sizeC, sizeZ, sizeY, sizeX}, new int[] {0, 0, 0, 0, 0}); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + test.write(data, new int[] { sizeT, sizeC, sizeZ, sizeY, sizeX }, + new int[] { 0, 0, 0, 0, 0 }); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.setResolutionLevel(0); PixelData pixelData = zpbuf.getTile(0, 0, 0, 0, 0, 10, 10); ByteBuffer bb = pixelData.getData(); @@ -572,8 +510,7 @@ public void testGetTileLargerThanImage() } @Test(expected = IllegalArgumentException.class) - public void testTileIntegerOverflow() - throws IOException, InvalidRangeException { + public void testTileIntegerOverflow() throws IOException, InvalidRangeException { int sizeT = 1; int sizeC = 3; int sizeZ = 1; @@ -585,15 +522,13 @@ public void testTileIntegerOverflow() // Hack the .zarray so we can appear as though we have more data than // we actually have written above. ObjectMapper mapper = new ObjectMapper(); - HashMap arrayAttrs = mapper.readValue( - Files.readAllBytes(output.resolve("0/0/.zarray")), - HashMap.class); + HashMap arrayAttrs = mapper + .readValue(Files.readAllBytes(output.resolve("0/0/.zarray")), HashMap.class); List shape = (List) arrayAttrs.get("shape"); shape.set(3, 50000); shape.set(4, 50000); mapper.writeValue(output.resolve("0/0/.zarray").toFile(), arrayAttrs); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 32, 32)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 32, 32)) { zpbuf.getTile(0, 0, 0, 0, 0, 50000, 50000); } } @@ -608,8 +543,7 @@ public void testTileExceedsMinMax() throws IOException { Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 32, 32)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 32, 32)) { Assert.assertNull(zpbuf.getTile(0, 0, 0, 0, 0, 32, 33)); // Throws exception zpbuf.getTile(0, 0, 0, -1, 0, 1, 1); @@ -625,8 +559,7 @@ public void testCheckBoundsValidZeros() throws IOException { int sizeX = 2048; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.checkBounds(0, 0, 0, 0, 0); } } @@ -640,8 +573,7 @@ public void testCheckBoundsValidEnd() throws IOException { int sizeX = 2048; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.checkBounds(2047, 511, 2, 1, 0); } } @@ -655,8 +587,7 @@ public void testCheckBoundsOutOfRange() throws IOException { int sizeX = 2048; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.checkBounds(2048, 511, 2, 1, 0); } } @@ -668,11 +599,9 @@ public void testCheckBounds() throws IOException { int sizeZ = 3; int sizeY = 512; int sizeX = 2048; - Pixels pixels = new Pixels( - null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); + Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.checkBounds(-1, 0, 0, 0, 0); } } @@ -686,8 +615,7 @@ public void testGetTileSize() throws IOException { int sizeX = 2048; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { Dimension tileSize = zpbuf.getTileSize(); Assert.assertEquals(1024, tileSize.getWidth(), 0.1); Assert.assertEquals(1024, tileSize.getHeight(), 0.1); @@ -705,8 +633,7 @@ public void testUint16() throws IOException { int bytesPerPixel = 2; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { Assert.assertEquals(FormatTools.UINT16, zpbuf.getPixelsType()); Assert.assertEquals(false, zpbuf.isSigned()); Assert.assertEquals(false, zpbuf.isFloat()); @@ -725,8 +652,7 @@ public void testFloat() throws IOException { int bytesPerPixel = 4; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "float"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { Assert.assertEquals(FormatTools.FLOAT, zpbuf.getPixelsType()); Assert.assertEquals(true, zpbuf.isSigned()); Assert.assertEquals(true, zpbuf.isFloat()); @@ -744,32 +670,22 @@ public void testSizes() throws IOException { int bytesPerPixel = 2; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { // Plane size - Assert.assertEquals( - sizeX * sizeY * bytesPerPixel, - zpbuf.getPlaneSize().longValue()); + Assert.assertEquals(sizeX * sizeY * bytesPerPixel, zpbuf.getPlaneSize().longValue()); // Stack size - Assert.assertEquals( - sizeZ * sizeX * sizeY * bytesPerPixel, - zpbuf.getStackSize().longValue()); + Assert.assertEquals(sizeZ * sizeX * sizeY * bytesPerPixel, + zpbuf.getStackSize().longValue()); // Timepoint size - Assert.assertEquals( - sizeC * sizeZ * sizeX * sizeY * bytesPerPixel, - zpbuf.getTimepointSize().longValue()); + Assert.assertEquals(sizeC * sizeZ * sizeX * sizeY * bytesPerPixel, + zpbuf.getTimepointSize().longValue()); // Total size - Assert.assertEquals( - sizeT * sizeC * sizeZ * sizeX * sizeY * bytesPerPixel, - zpbuf.getTotalSize().longValue()); + Assert.assertEquals(sizeT * sizeC * sizeZ * sizeX * sizeY * bytesPerPixel, + zpbuf.getTotalSize().longValue()); // Column size - Assert.assertEquals( - sizeY * bytesPerPixel, - zpbuf.getColSize().longValue()); + Assert.assertEquals(sizeY * bytesPerPixel, zpbuf.getColSize().longValue()); // Row size - Assert.assertEquals( - sizeX * bytesPerPixel, - zpbuf.getRowSize().longValue()); + Assert.assertEquals(sizeX * bytesPerPixel, zpbuf.getRowSize().longValue()); } } @@ -782,11 +698,9 @@ public void testSetResolutionLevelOutOfBounds() throws IOException { int sizeX = 2048; int resolutions = 3; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", - "--resolutions", String.valueOf(resolutions)); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--resolutions", + String.valueOf(resolutions)); + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), 1024, 1024)) { zpbuf.setResolutionLevel(resolutions); } } @@ -800,23 +714,20 @@ public void testDownsampledZ() throws IOException { int sizeX = 2048; int resolutions = 3; Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint8", - "--resolutions", String.valueOf(resolutions)); + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint8", "--resolutions", + String.valueOf(resolutions)); // Hack the .zarray to hide Z sections in lower resolutions for (int r = 1; r < resolutions; r++) { ObjectMapper mapper = new ObjectMapper(); HashMap arrayAttrs = mapper.readValue( - Files.readAllBytes(output.resolve("0/" + r + "/.zarray")), - HashMap.class); + Files.readAllBytes(output.resolve("0/" + r + "/.zarray")), HashMap.class); List shape = (List) arrayAttrs.get("shape"); shape.set(2, sizeZ / (int) Math.pow(2, r)); mapper.writeValue(output.resolve("0/" + r + "/.zarray").toFile(), arrayAttrs); } - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { // get the last Z section, for each resolution level for (int r = 0; r < resolutions; r++) { zpbuf.setResolutionLevel(r); @@ -827,8 +738,7 @@ public void testDownsampledZ() throws IOException { } @Test - public void testReadDataNonDefaultAxes() - throws IOException, InvalidRangeException { + public void testReadDataNonDefaultAxes() throws IOException, InvalidRangeException { // Pretty much the same as testGetTimepointStackPlaneRowCol() // but testing a different axes order. int sizeT = 2; @@ -837,37 +747,34 @@ public void testReadDataNonDefaultAxes() int sizeY = 1024; int sizeX = 2048; String order = DimensionOrder.VALUE_XYCTZ; // Default XYZCT - String revOrder = new StringBuilder(order).reverse().toString(); - Pixels pixels = new Pixels( - null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", new DimensionOrder(order)); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "int32", "--dimension-order", order); - - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { + String revOrder = new StringBuilder(order).reverse().toString(); + Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", + new DimensionOrder(order)); + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "int32", "--dimension-order", + order); + + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { for (int t = 0; t < sizeT; t++) { // Assert timepoint byte[] timepoint = zpbuf.getTimepoint(t).getData().array(); for (int c = 0; c < sizeC; c++) { // Assert stack byte[] stack = zpbuf.getStack(c, t).getData().array(); - byte[] stackFromTimepoint = - getStack(timepoint, c, sizeC, sizeZ, sizeX, sizeY, revOrder); + byte[] stackFromTimepoint = getStack(timepoint, c, sizeC, sizeZ, sizeX, sizeY, + revOrder); Assert.assertArrayEquals(stack, stackFromTimepoint); for (int z = 0; z < sizeZ; z++) { // Assert plane - byte[] plane = - zpbuf.getPlane(z, c, t).getData().array(); - byte[] planeFromStack = - getPlane(stack, z, sizeZ, sizeX, sizeY, revOrder); + byte[] plane = zpbuf.getPlane(z, c, t).getData().array(); + byte[] planeFromStack = getPlane(stack, z, sizeZ, sizeX, sizeY, revOrder); Assert.assertArrayEquals(plane, planeFromStack); // Assert row int y = sizeY / 2; int rowSize = zpbuf.getRowSize(); int rowOffset = y * rowSize; byte[] row = zpbuf.getRow(y, z, c, t).getData().array(); - byte[] rowExpected = Arrays.copyOfRange( - plane, rowOffset, rowOffset + rowSize); + byte[] rowExpected = Arrays.copyOfRange(plane, rowOffset, + rowOffset + rowSize); Assert.assertArrayEquals(rowExpected, row); // Assert column int x = sizeX / 2; @@ -881,8 +788,7 @@ public void testReadDataNonDefaultAxes() } @Test - public void testDefaultOrder() - throws IOException, InvalidRangeException { + public void testDefaultOrder() throws IOException, InvalidRangeException { // Check that if access are not in the file it defaults to TCZYX order when no axes found int sizeT = 1; @@ -893,8 +799,7 @@ public void testDefaultOrder() Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint8"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { Assert.assertEquals(sizeT, zpbuf.getSizeT()); Assert.assertEquals(sizeC, zpbuf.getSizeC()); Assert.assertEquals(sizeZ, zpbuf.getSizeZ()); @@ -980,13 +885,12 @@ private void testOrder(String order) throws IOException, InvalidRangeException { int sizeZ = 4; int sizeY = 256; int sizeX = 512; - Pixels pixels = new Pixels( - null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", new DimensionOrder(order)); - Path output = writeTestZarr( - sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--dimension-order", order); + Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", + new DimensionOrder(order)); + Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", + "--dimension-order", order); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { Assert.assertEquals(sizeT, zpbuf.getSizeT()); Assert.assertEquals(sizeC, zpbuf.getSizeC()); Assert.assertEquals(sizeZ, zpbuf.getSizeZ()); @@ -997,14 +901,12 @@ private void testOrder(String order) throws IOException, InvalidRangeException { } } - private void testCompactDimensions( - int sizeX, int sizeY, int sizeZ, int sizeC, int sizeT, String order) - throws IOException, InvalidRangeException { - + private void testCompactDimensions(int sizeX, int sizeY, int sizeZ, int sizeC, int sizeT, + String order) throws IOException, InvalidRangeException { + Pixels pixels = new Pixels(null, null, sizeX, sizeY, sizeZ, sizeC, sizeT, "", null); Path output = writeTestZarr(sizeT, sizeC, sizeZ, sizeY, sizeX, "uint16", "--compact"); - try (ZarrPixelBuffer zpbuf = - createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { + try (ZarrPixelBuffer zpbuf = createPixelBuffer(pixels, output.resolve("0"), sizeX, sizeY)) { assertAxes(zpbuf, order); assertPixels(zpbuf, sizeX, sizeY, sizeZ, sizeC, sizeT); } From af7496fb303cfe319d88d4bdcd0ce53dcc5aeed0 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Tue, 30 Sep 2025 15:17:57 +0100 Subject: [PATCH 17/17] Add more tests --- .../omero/zarr/TestZarrInfo.java | 149 ++++++++++++++---- 1 file changed, 117 insertions(+), 32 deletions(-) diff --git a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java index 38f78e2..bddcb04 100644 --- a/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java +++ b/src/test/java/com/glencoesoftware/omero/zarr/TestZarrInfo.java @@ -1,72 +1,157 @@ package com.glencoesoftware.omero.zarr; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo; +import com.glencoesoftware.omero.zarr.compat.ZarrInfo.StorageType; import java.io.IOException; import java.nio.file.Path; - import org.apache.maven.artifact.versioning.ComparableVersion; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.glencoesoftware.omero.zarr.compat.ZarrInfo; -import com.glencoesoftware.omero.zarr.compat.ZarrInfo.StorageType; - +/** + * Test ZarrInfo. + */ public class TestZarrInfo { - + @Rule public TemporaryFolder tmpDir = new TemporaryFolder(); + /** + * Test local storage, zarr v2. + * + * @throws IOException If an I/O error occurs. + */ @Test - public void testLocalV2N04() throws IOException { - String v0_4_local = writeTestZarr(1,1,1,256,256,"uint8",1).toString()+"/0"; - - ZarrInfo zp = new ZarrInfo(v0_4_local); + public void testLocalV2() throws IOException { + String path = writeTestZarr(1, 1, 1, 256, 256, "uint8", 1).toString() + "/0"; + + ZarrInfo zp = new ZarrInfo(path); + System.out.println("testLocalV2: " + zp); Assert.assertEquals(zp.getStorageType(), StorageType.FILE); Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); } + /** + * Test local storage, zarr v3. + * + * @throws IOException If an I/O error occurs. + */ @Test - public void testHTTPV2N04() throws IOException { - ZarrInfo zp = new ZarrInfo("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0101A/13457227.zarr"); + public void testLocalV3() throws IOException { + // TODO: implement + // String path = writeTestZarr(1, 1, 1, 256, 256, "uint8", 1).toString() + "/0"; + // ZarrInfo zp = new ZarrInfo(path); + // Assert.assertEquals(zp.getStorageType(), StorageType.FILE); + // Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); + // Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); + } + + /** + * Test HTTP storage, zarr v2. + * + * @throws IOException If an I/O error occurs. + */ + @Test + public void testHTTPV2() throws IOException { + ZarrInfo zp = new ZarrInfo("https://s3.ltd.ovh/public/cat_z2.ome.zarr/0"); + System.out.println("testHTTPV2: " + zp); Assert.assertEquals(zp.getStorageType(), StorageType.HTTP); Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); } + /** + * Test HTTP storage, zarr v3. + * + * @throws IOException If an I/O error occurs. + */ @Test - public void testS3V2N04() throws IOException { - ZarrInfo zp = new ZarrInfo("s3://s3.us-east-1.amazonaws.com/gs-public-zarr-archive/CMU-1.ome.zarr/0?anonymous=true"); + public void testHTTPV3() throws IOException { + ZarrInfo zp = new ZarrInfo("https://s3.ltd.ovh/public/cat_z3.ome.zarr/0"); + System.out.println("testHTTPV3: " + zp); + Assert.assertEquals(zp.getStorageType(), StorageType.HTTP); + Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); + } + + /** + * Test public S3 storage, zarr v2. + * + * @throws IOException If an I/O error occurs. + */ + @Test + public void testS3V2Public() throws IOException { + ZarrInfo zp = new ZarrInfo("s3://s3.ltd.ovh/public/cat_z2.ome.zarr/0?anonymous=true"); + System.out.println("testS3V2Public: " + zp); Assert.assertEquals(zp.getStorageType(), StorageType.S3); Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); } + /** + * Test public S3 storage, zarr v3. + * + * @throws IOException If an I/O error occurs. + */ @Test - public void testHTTPV3N05() throws IOException { - ZarrInfo zp = new ZarrInfo("https://uk1s3.embassy.ebi.ac.uk/idr/share/ome2024-ngff-challenge/0.0.5/6001240_labels.zarr"); - Assert.assertEquals(zp.getStorageType(), StorageType.HTTP); + public void testS3V3Public() throws IOException { + ZarrInfo zp = new ZarrInfo("s3://s3.ltd.ovh/public/cat_z3.ome.zarr/0?anonymous=true"); + System.out.println("testS3V3Public: " + zp); + Assert.assertEquals(zp.getStorageType(), StorageType.S3); + Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); + } + + /** + * Test private S3 storage, zarr v2. + * + * @throws IOException If an I/O error occurs. + */ + @Test + public void testS3V2Private() throws IOException { + ZarrInfo zp = new ZarrInfo("s3://s3.ltd.ovh/private/cat_z2.ome.zarr/0?profile=ltd"); + System.out.println("testS3V2Private: " + zp); + Assert.assertEquals(zp.getStorageType(), StorageType.S3); + Assert.assertEquals(new ComparableVersion("2"), zp.getZarrVersion()); + Assert.assertEquals(new ComparableVersion("0.4"), zp.getNgffVersion()); + } + + /** + * Test private S3 storage, zarr v3. + * + * @throws IOException If an I/O error occurs. + */ + @Test + public void testS3V3Private() throws IOException { + ZarrInfo zp = new ZarrInfo("s3://s3.ltd.ovh/private/cat_z3.ome.zarr/0?profile=ltd"); + System.out.println("testS3V3Private: " + zp); + Assert.assertEquals(zp.getStorageType(), StorageType.S3); Assert.assertEquals(new ComparableVersion("3"), zp.getZarrVersion()); Assert.assertEquals(new ComparableVersion("0.5"), zp.getNgffVersion()); } - public Path writeTestZarr( - int sizeT, - int sizeC, - int sizeZ, - int sizeY, - int sizeX, - String pixelType, - int resolutions) throws IOException { - Path input = ZarrPixelBufferTest.fake( - "sizeT", Integer.toString(sizeT), - "sizeC", Integer.toString(sizeC), - "sizeZ", Integer.toString(sizeZ), - "sizeY", Integer.toString(sizeY), - "sizeX", Integer.toString(sizeX), - "pixelType", pixelType, - "resolutions", Integer.toString(resolutions)); + /** + * Write a test Zarr file. + * + * @param sizeT Number of time points. + * @param sizeC Number of channels. + * @param sizeZ Number of z-sections. + * @param sizeY Number of rows. + * @param sizeX Number of columns. + * @param pixelType Pixel type. + * @param resolutions Number of resolutions. + * @return Path to the test Zarr file. + * @throws IOException If an I/O error occurs. + */ + public Path writeTestZarr(int sizeT, int sizeC, int sizeZ, int sizeY, int sizeX, + String pixelType, int resolutions) throws IOException { + Path input = ZarrPixelBufferTest.fake("sizeT", Integer.toString(sizeT), "sizeC", + Integer.toString(sizeC), "sizeZ", Integer.toString(sizeZ), "sizeY", + Integer.toString(sizeY), "sizeX", Integer.toString(sizeX), "pixelType", pixelType, + "resolutions", Integer.toString(resolutions)); Path output = tmpDir.getRoot().toPath().resolve("output.zarr"); ZarrPixelBufferTest.assertBioFormats2Raw(input, output); return output;