Merge pull request #25 from chris-allan/zarr-only

Switch to Zarr only

Author: chris-allan

README.md

@@ -6,30 +6,37 @@ Python tool that uses Philips' SDK to write slides in an intermediary raw format
 
 ## Requirements
 
+* Python 3.6+
 * Philips iSyntax SDK (https://openpathology.philips.com)
 
 The iSyntax SDK __must__ be downloaded separately from Philips and the
 relevant license agreement agreed to before any conversion can take place.
 
+As of version 0.4.0, which has a Python 3.6+ requirement, the supported
+iSyntax SDK versions and environments are as follows:
+
+* iSyntax SDK 1.2.1 (CentOS 7, Ubuntu 18.04, Windows 10 64-bit)
+* iSyntax SDK 2.0 (CentOS 8, Ubuntu 18.04, Windows 10 64-bit)
+
 ## Usage
 
 Basic usage is:
 
-    isyntax2raw write_tiles /path/to/input.isyntax /path/to/tile/directory
+    isyntax2raw write_tiles /path/to/input.isyntax /path/to/directory.zarr
 
 Please see `isyntax2raw write_tiles --help` for detailed information.
 
 Output tile width and height can optionally be specified; default values are
 detailed in `--help`.
 
 A directory structure containing the pyramid tiles at all resolutions and
-macro/label images will be created.  The default format is N5.  Additional
+macro/label images will be created.  The default format is Zarr.  Additional
 metadata is written to a JSON file.  Be mindful of available disk space, as
 larger .isyntax files can result in >20 GB of tiles.
 
-Use of a n5 (the default) or zarr `--file_type` will result in losslessly
-compressed output.  These are the only formats that are currently
-supported by the downstream `raw2ometiff`.
+Use of the Zarr file type will result in losslessly compressed output.  This
+is the only format currently supported by the downstream `raw2ometiff` (as of
+version 0.3.0).
 
 ## Background color
 

appveyor.yml

@@ -1,7 +1,6 @@
 image: ubuntu1804
 
 install:
-  - sudo apt-get update
   - sudo apt-get install -y python3-setuptools python3-wheel twine
 
 build: off

isyntax2raw/__init__.py

@@ -25,12 +25,13 @@
 
 from PIL import Image
 from kajiki import PackageLoader
+from zarr.storage import FSStore
 
 
 log = logging.getLogger(__name__)
 
-# version of the N5/Zarr layout
-LAYOUT_VERSION = 1
+# version of the Zarr layout
+LAYOUT_VERSION = 3
 
 
 class MaxQueuePool(object):
@@ -79,20 +80,20 @@ def __exit__(self, exception_type, exception_value, traceback):
 class WriteTiles(object):
 
     def __init__(
-        self, tile_width, tile_height, resolutions, file_type, max_workers,
-        batch_size, input_path, output_path, fill_color
+        self, tile_width, tile_height, resolutions, max_workers,
+        batch_size, fill_color, nested, input_path, output_path
     ):
         self.tile_width = tile_width
         self.tile_height = tile_height
         self.resolutions = resolutions
-        self.file_type = file_type
         self.max_workers = max_workers
         self.batch_size = batch_size
+        self.fill_color = fill_color
+        self.nested = nested
         self.input_path = input_path
         self.slide_directory = output_path
-        self.fill_color = fill_color
 
-        os.makedirs(self.slide_directory, exist_ok=True)
+        os.makedirs(os.path.join(self.slide_directory, "OME"), exist_ok=True)
 
         render_context = softwarerendercontext.SoftwareRenderContext()
         render_backend = softwarerenderbackend.SoftwareRenderBackend()
@@ -378,7 +379,9 @@ def wait_any(self, regions):
 
     def write_metadata(self):
         '''write metadata to a JSON file'''
-        metadata_file = os.path.join(self.slide_directory, "METADATA.json")
+        metadata_file = os.path.join(
+            self.slide_directory, "OME", "METADATA.json"
+        )
 
         with open(metadata_file, "w", encoding="utf-8") as f:
             metadata = self.get_metadata()
@@ -419,7 +422,9 @@ def write_metadata(self):
         loader = PackageLoader()
         template = loader.import_("isyntax2raw.resources.ome_template")
         xml = template(xml_values).render()
-        ome_xml_file = os.path.join(self.slide_directory, "METADATA.ome.xml")
+        ome_xml_file = os.path.join(
+            self.slide_directory, "OME", "METADATA.ome.xml"
+        )
         with open(ome_xml_file, "w", encoding="utf-8") as omexml:
             omexml.write(xml)
 
@@ -474,12 +479,15 @@ def write_image_type(self, image_type, series):
             log.info("wrote %s image" % image_type)
 
     def create_tile_directory(self, series, resolution, width, height):
-        tile_directory = os.path.join(
-            self.slide_directory, "data.%s" % self.file_type
+        dimension_separator = '/'
+        if not self.nested:
+            dimension_separator = '.'
+        self.zarr_store = FSStore(
+            self.slide_directory,
+            dimension_separator=dimension_separator,
+            normalize_keys=True,
+            auto_mkdir=True
         )
-        self.zarr_store = zarr.DirectoryStore(tile_directory)
-        if self.file_type == "n5":
-            self.zarr_store = zarr.N5Store(tile_directory)
         self.zarr_group = zarr.group(store=self.zarr_store)
         self.zarr_group.attrs['bioformats2raw.layout'] = LAYOUT_VERSION
 
@@ -520,7 +528,7 @@ def write_tile(
             x_end = x_start + tile_width
             y_end = y_start + tile_height
             try:
-                # N5/Zarr has a single n-dimensional array representation on
+                # Zarr has a single n-dimensional array representation on
                 # disk (not interleaved RGB)
                 pixels = self.make_planar(pixels, tile_width, tile_height)
                 z = self.zarr_group["0/%d" % resolution]

isyntax2raw/cli/isyntax2raw.py

@@ -31,11 +31,6 @@ def cli():
     "--resolutions", type=int,
     help="number of pyramid resolutions to generate [default: all]"
 )
-@click.option(
-    "--file_type", type=click.Choice(['n5', 'zarr']), default="n5",
-    show_default=True,
-    help="tile file extension"
-)
 @click.option(
     "--max_workers", default=4, type=int,
     show_default=True,
@@ -50,15 +45,19 @@ def cli():
     show_default=True,
     help="background color for missing tiles (0-255)"
 )
+@click.option(
+    "--nested/--no-nested", default=True, show_default=True,
+    help="Whether to use '/' as the chunk path separator"
+)
 @click.option(
     "--debug", is_flag=True,
     help="enable debugging",
 )
 @click.argument("input_path")
 @click.argument("output_path")
 def write_tiles(
-    tile_width, tile_height, resolutions, file_type, max_workers, batch_size,
-    input_path, output_path, fill_color, debug
+    tile_width, tile_height, resolutions, max_workers, batch_size,
+    fill_color, nested, debug, input_path, output_path
 ):
     level = logging.INFO
     if debug:
@@ -69,8 +68,8 @@ def write_tiles(
                "(%(thread)10s) %(message)s"
     )
     with WriteTiles(
-        tile_width, tile_height, resolutions, file_type, max_workers,
-        batch_size, input_path, output_path, fill_color
+        tile_width, tile_height, resolutions, max_workers,
+        batch_size, fill_color, nested, input_path, output_path
     ) as wt:
         wt.write_metadata()
         wt.write_label_image()

setup.py

@@ -62,6 +62,7 @@ def read(fname):
 
 setup(name='isyntax2raw',
       version=version.getVersion(),
+      python_requires='>=3.6',
       description='iSyntax to raw format converter',
       long_description=read('README.md'),
       long_description_content_type='text/markdown',
@@ -82,8 +83,9 @@ def read(fname):
           'click==7.0',
           'pillow>=7.1.0',
           'numpy==1.17.3',
-          'zarr==2.4.0',
+          'zarr==2.8.1',
           'kajiki==0.8.2',
+          'fsspec>=0.9.0',
       ],
       tests_require=[
           'flake8',